import{S as Co,i as zo,s as Oo,e as n,k as p,w as c,t as s,M as No,c as l,d as t,m as h,a as o,x as m,h as r,b as d,G as a,g as f,y as u,L as Bo,q as _,o as y,B as g,v as Lo}from"../chunks/vendor-hf-doc-builder.js";import{I as me}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as v}from"../chunks/CodeBlock-hf-doc-builder.js";function Ho(nl){let W,dt,F,K,Sa,ue,gs,Pa,bs,pt,b,vs,Fe,ws,ks,Ve,$s,js,Ye,Ds,Es,Ge,xs,Is,Me,As,qs,Re,Ss,Ps,ht,V,Q,Ta,_e,Ts,Ca,Cs,ct,S,zs,Je,Os,Ns,za,Bs,Ls,mt,ye,ut,E,Hs,Ue,Ws,Fs,Oa,Vs,Ys,Na,Gs,Ms,_t,Ke,Rs,yt,ge,gt,w,Js,be,Us,Ks,ve,Qs,Xs,we,Zs,er,Qe,ar,tr,bt,X,sr,Ba,rr,nr,vt,Y,Z,La,ke,lr,Ha,or,wt,ee,ir,Xe,fr,dr,kt,$e,$t,x,pr,Wa,hr,cr,Fa,mr,ur,Va,_r,yr,jt,je,Dt,G,ae,Ya,De,gr,Ga,br,Et,P,vr,Ze,wr,kr,ea,$r,jr,xt,Ee,It,aa,Dr,At,te,Er,Ma,xr,Ir,qt,xe,St,k,Ar,ta,qr,Sr,sa,Pr,Tr,ra,Cr,zr,na,Or,Nr,Pt,M,se,Ra,Ie,Br,Ja,Lr,Tt,I,Hr,la,Wr,Fr,oa,Vr,Yr,Ua,Gr,Mr,Ct,Ae,zt,$,Rr,Ka,Jr,Ur,ia,Kr,Qr,Qa,Xr,Zr,Xa,en,an,Ot,fa,tn,Nt,qe,Bt,R,re,Za,Se,sn,et,rn,Lt,j,nn,da,ln,on,pa,fn,dn,at,pn,hn,tt,cn,mn,Ht,Pe,Wt,T,un,st,_n,yn,ha,gn,bn,Ft,Te,Vt,ne,vn,ca,wn,kn,Yt,Ce,Gt,J,le,rt,ze,$n,nt,jn,Mt,C,Dn,ma,En,xn,lt,In,An,Rt,A,qn,ua,Sn,Pn,_a,Tn,Cn,ya,zn,On,Jt,Oe,Ut,z,Nn,ga,Bn,Ln,ba,Hn,Wn,Kt,Ne,Qt,oe,Fn,ot,Vn,Yn,Xt,Be,Zt,U,ie,it,Le,Gn,ft,Mn,es,q,Rn,va,Jn,Un,wa,Kn,Qn,ka,Xn,Zn,as,He,ts,O,el,$a,al,tl,ja,sl,rl,ss,We,rs;return ue=new me({}),_e=new me({}),ye=new v({props:{code:`from datasets import load_dataset

imagenet = load_dataset("imagenet-1k", split="train")  # downloads the full dataset
print(imagenet[0])`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

imagenet = load_dataset(<span class="hljs-string">&quot;imagenet-1k&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)  <span class="hljs-comment"># downloads the full dataset</span>
<span class="hljs-built_in">print</span>(imagenet[<span class="hljs-number">0</span>])`}}),ge=new v({props:{code:`from datasets import load_dataset

imagenet = load_dataset("imagenet-1k", split="train", streaming=True)  # will start loading the data when iterated over
for example in imagenet:
    print(example)
    break`,highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

imagenet = load_dataset(<span class="hljs-string">&quot;imagenet-1k&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>)  <span class="hljs-comment"># will start loading the data when iterated over</span>
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> imagenet:
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`}}),ke=new me({}),$e=new v({props:{code:`my_dataset = Dataset.from_dict({"col_1": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]})
print(my_dataset[0])`,highlighted:`my_dataset = Dataset.from_dict({<span class="hljs-string">&quot;col_1&quot;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>]})
<span class="hljs-built_in">print</span>(my_dataset[<span class="hljs-number">0</span>])`}}),je=new v({props:{code:`def my_generator(n):
    for i in range(n):
        yield {"col_1": i}

my_iterable_dataset = IterableDataset.from_generator(my_generator, gen_kwargs={"n": 10})
for example in my_iterable_dataset:
    print(example)
    break`,highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">my_generator</span>(<span class="hljs-params">n</span>):
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n):
        <span class="hljs-keyword">yield</span> {<span class="hljs-string">&quot;col_1&quot;</span>: i}

my_iterable_dataset = IterableDataset.from_generator(my_generator, gen_kwargs={<span class="hljs-string">&quot;n&quot;</span>: <span class="hljs-number">10</span>})
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`}}),De=new me({}),Ee=new v({props:{code:`data_files = {"train": ["path/to/data.csv"]}
my_dataset = load_dataset("csv", data_files=data_files, split="train")
print(my_dataset[0])`,highlighted:`data_files = {<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;path/to/data.csv&quot;</span>]}
my_dataset = load_dataset(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-built_in">print</span>(my_dataset[<span class="hljs-number">0</span>])`}}),xe=new v({props:{code:`data_files = {"train": ["path/to/data.csv"]}
my_iterable_dataset = load_dataset("csv", data_files=data_files, split="train", streaming=True)
for example in my_iterable_dataset:  # this reads the CSV file progressively as you iterate over the dataset
    print(example)
    break`,highlighted:`data_files = {<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;path/to/data.csv&quot;</span>]}
my_iterable_dataset = load_dataset(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>)
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:  <span class="hljs-comment"># this reads the CSV file progressively as you iterate over the dataset</span>
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`}}),Ie=new me({}),Ae=new v({props:{code:`my_dataset = my_dataset.map(process_fn)  # process_fn is applied on all the examples of the dataset
print(my_dataset[0])`,highlighted:`my_dataset = my_dataset.<span class="hljs-built_in">map</span>(process_fn)  <span class="hljs-comment"># process_fn is applied on all the examples of the dataset</span>
<span class="hljs-built_in">print</span>(my_dataset[<span class="hljs-number">0</span>])`}}),qe=new v({props:{code:`my_iterable_dataset = my_iterable_dataset.map(process_fn_1)
my_iterable_dataset = my_iterable_dataset.filter(filter_fn)
my_iterable_dataset = my_iterable_dataset.map(process_fn_2)

# process_fn_1, filter_fn and process_fn_2 are applied on-the-fly when iterating over the dataset
for example in my_iterable_dataset:  
    print(example)
    break`,highlighted:`my_iterable_dataset = my_iterable_dataset.<span class="hljs-built_in">map</span>(process_fn_1)
my_iterable_dataset = my_iterable_dataset.<span class="hljs-built_in">filter</span>(filter_fn)
my_iterable_dataset = my_iterable_dataset.<span class="hljs-built_in">map</span>(process_fn_2)

<span class="hljs-comment"># process_fn_1, filter_fn and process_fn_2 are applied on-the-fly when iterating over the dataset</span>
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:  
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`}}),Se=new me({}),Pe=new v({props:{code:`my_dataset = my_dataset.shuffle(seed=42)
print(my_dataset[0])`,highlighted:`my_dataset = my_dataset.shuffle(seed=<span class="hljs-number">42</span>)
<span class="hljs-built_in">print</span>(my_dataset[<span class="hljs-number">0</span>])`}}),Te=new v({props:{code:`my_iterable_dataset = my_iterable_dataset.shuffle(seed=42, buffer_size=100)
for example in my_iterable_dataset:
    print(example)
    break`,highlighted:`my_iterable_dataset = my_iterable_dataset.shuffle(seed=<span class="hljs-number">42</span>, buffer_size=<span class="hljs-number">100</span>)
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`}}),Ce=new v({props:{code:`# Stream from the internet
my_iterable_dataset = load_dataset("c4", "en", split="train", streaming=True)
my_iterable_dataset.n_shards  # 1024

# Stream from local files
data_files = {"train": [f"path/to/data_{i}.csv" for i in range(1024)]}
my_iterable_dataset = load_dataset("csv", data_files=data_files, split="train", streaming=True)
my_iterable_dataset.n_shards  # 1024

# From a generator function
def my_generator(n, sources):
    for source in sources:
        for example_id_for_current_source in range(n):
            yield {"example_id": f"{source}_{example_id_for_current_source}"}

gen_kwargs = {"n": 10, "sources": [f"path/to/data_{i}" for i in range(1024)]}
my_iterable_dataset = IterableDataset.from_generator(my_generator, gen_kwargs=gen_kwargs)
my_iterable_dataset.n_shards  # 1024`,highlighted:`<span class="hljs-comment"># Stream from the internet</span>
my_iterable_dataset = load_dataset(<span class="hljs-string">&quot;c4&quot;</span>, <span class="hljs-string">&quot;en&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>)
my_iterable_dataset.n_shards  <span class="hljs-comment"># 1024</span>

<span class="hljs-comment"># Stream from local files</span>
data_files = {<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">f&quot;path/to/data_<span class="hljs-subst">{i}</span>.csv&quot;</span> <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1024</span>)]}
my_iterable_dataset = load_dataset(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>)
my_iterable_dataset.n_shards  <span class="hljs-comment"># 1024</span>

<span class="hljs-comment"># From a generator function</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">my_generator</span>(<span class="hljs-params">n, sources</span>):
    <span class="hljs-keyword">for</span> source <span class="hljs-keyword">in</span> sources:
        <span class="hljs-keyword">for</span> example_id_for_current_source <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n):
            <span class="hljs-keyword">yield</span> {<span class="hljs-string">&quot;example_id&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{source}</span>_<span class="hljs-subst">{example_id_for_current_source}</span>&quot;</span>}

gen_kwargs = {<span class="hljs-string">&quot;n&quot;</span>: <span class="hljs-number">10</span>, <span class="hljs-string">&quot;sources&quot;</span>: [<span class="hljs-string">f&quot;path/to/data_<span class="hljs-subst">{i}</span>&quot;</span> <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1024</span>)]}
my_iterable_dataset = IterableDataset.from_generator(my_generator, gen_kwargs=gen_kwargs)
my_iterable_dataset.n_shards  <span class="hljs-comment"># 1024</span>`}}),ze=new me({}),Oe=new v({props:{code:`my_dataset[0]  # fast
my_dataset = my_dataset.shuffle(seed=42)
my_dataset[0]  # up to 10x slower
my_dataset = my_dataset.flatten_indices()  # rewrite the shuffled dataset on disk as contiguous chunks of data
my_dataset[0]  # fast again`,highlighted:`my_dataset[<span class="hljs-number">0</span>]  <span class="hljs-comment"># fast</span>
my_dataset = my_dataset.shuffle(seed=<span class="hljs-number">42</span>)
my_dataset[<span class="hljs-number">0</span>]  <span class="hljs-comment"># up to 10x slower</span>
my_dataset = my_dataset.flatten_indices()  <span class="hljs-comment"># rewrite the shuffled dataset on disk as contiguous chunks of data</span>
my_dataset[<span class="hljs-number">0</span>]  <span class="hljs-comment"># fast again</span>`}}),Ne=new v({props:{code:`for example in enumerate(my_iterable_dataset):  # fast
    pass

shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=42, buffer_size=100)

for example in enumerate(shuffled_iterable_dataset):  # as fast as before
    pass

shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=1337, buffer_size=100)  # reshuffling using another seed is instantaneous

for example in enumerate(shuffled_iterable_dataset):  # still as fast as before
    pass`,highlighted:`<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(my_iterable_dataset):  <span class="hljs-comment"># fast</span>
    <span class="hljs-keyword">pass</span>

shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=<span class="hljs-number">42</span>, buffer_size=<span class="hljs-number">100</span>)

<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(shuffled_iterable_dataset):  <span class="hljs-comment"># as fast as before</span>
    <span class="hljs-keyword">pass</span>

shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=<span class="hljs-number">1337</span>, buffer_size=<span class="hljs-number">100</span>)  <span class="hljs-comment"># reshuffling using another seed is instantaneous</span>

<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(shuffled_iterable_dataset):  <span class="hljs-comment"># still as fast as before</span>
    <span class="hljs-keyword">pass</span>`}}),Be=new v({props:{code:`for epoch in range(n_epochs):
    my_iterable_dataset.set_epoch(epoch)
    for example in my_iterable_dataset:  # fast + reshuffled at each epoch using \`effective_seed = seed + epoch\`
        pass`,highlighted:`<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_epochs):
    my_iterable_dataset.set_epoch(epoch)
    <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:  <span class="hljs-comment"># fast + reshuffled at each epoch using \`effective_seed = seed + epoch\`</span>
        <span class="hljs-keyword">pass</span>`}}),Le=new me({}),He=new v({props:{code:"my_iterable_dataset = my_dataset.to_iterable_dataset()",highlighted:"my_iterable_dataset = my_dataset.to_iterable_dataset()"}}),We=new v({props:{code:`my_iterable_dataset = my_dataset.to_iterable_dataset(num_shards=1024)
my_iterable_dataset.n_shards  # 1024`,highlighted:`my_iterable_dataset = my_dataset.to_iterable_dataset(num_shards=<span class="hljs-number">1024</span>)
my_iterable_dataset.n_shards  <span class="hljs-comment"># 1024</span>`}}),{c(){W=n("meta"),dt=p(),F=n("h1"),K=n("a"),Sa=n("span"),c(ue.$$.fragment),gs=p(),Pa=n("span"),bs=s("Differences between Dataset and IterableDataset"),pt=p(),b=n("p"),vs=s("There are two types of dataset objects, a "),Fe=n("a"),ws=s("Dataset"),ks=s(" and an "),Ve=n("a"),$s=s("IterableDataset"),js=s(`.
Whichever type of dataset you choose to use or create depends on the size of the dataset.
In general, an `),Ye=n("a"),Ds=s("IterableDataset"),Es=s(" is ideal for big datasets (think hundreds of GBs!) due to its lazy behavior and speed advantages, while a "),Ge=n("a"),xs=s("Dataset"),Is=s(` is great for everything else.
This page will compare the differences between a `),Me=n("a"),As=s("Dataset"),qs=s(" and an "),Re=n("a"),Ss=s("IterableDataset"),Ps=s(" to help you pick the right dataset object for you."),ht=p(),V=n("h2"),Q=n("a"),Ta=n("span"),c(_e.$$.fragment),Ts=p(),Ca=n("span"),Cs=s("Downloading and streaming"),ct=p(),S=n("p"),zs=s("When you have a regular "),Je=n("a"),Os=s("Dataset"),Ns=s(", you can access it using "),za=n("code"),Bs=s("my_dataset[0]"),Ls=s(`. This provides random access to the rows.
Such datasets are also called \u201Cmap-style\u201D datasets.
For example you can download ImageNet-1k like this and access any row:`),mt=p(),c(ye.$$.fragment),ut=p(),E=n("p"),Hs=s(`But one caveat is that you must have the entire dataset stored on your disk or in memory, which blocks you from accessing datasets bigger than the disk.
Because it can become inconvenient for big datasets, there exists another type of dataset, the `),Ue=n("a"),Ws=s("IterableDataset"),Fs=s(`.
When you have an `),Oa=n("code"),Vs=s("IterableDataset"),Ys=s(", you can access it using a "),Na=n("code"),Gs=s("for"),Ms=s(` loop to load the data progressively as you iterate over the dataset.
This way, only a small fraction of examples is loaded in memory, and you don\u2019t write anything on disk.`),_t=p(),Ke=n("p"),Rs=s("For example, you can stream the ImageNet-1k dataset without downloading it on disk:"),yt=p(),c(ge.$$.fragment),gt=p(),w=n("p"),Js=s(`Streaming can read online data without writing any file to disk.
For example, you can stream datasets made out of multiple shards, each of which is hundreds of gigabytes like `),be=n("a"),Us=s("C4"),Ks=s(", "),ve=n("a"),Qs=s("OSCAR"),Xs=s(" or "),we=n("a"),Zs=s("LAION-2B"),er=s(`.
Learn more about how to stream a dataset in the `),Qe=n("a"),ar=s("Dataset Streaming Guide"),tr=s("."),bt=p(),X=n("p"),sr=s("This is not the only difference though, because the \u201Clazy\u201D behavior of an "),Ba=n("code"),rr=s("IterableDataset"),nr=s(" is also present when it comes to dataset creation and processing."),vt=p(),Y=n("h2"),Z=n("a"),La=n("span"),c(ke.$$.fragment),lr=p(),Ha=n("span"),or=s("Creating map-style datasets and iterable datasets"),wt=p(),ee=n("p"),ir=s("You can create a "),Xe=n("a"),fr=s("Dataset"),dr=s(" using lists or dictionaries, and the data is entirely converted to Arrow so you can easily access any row:"),kt=p(),c($e.$$.fragment),$t=p(),x=n("p"),pr=s("To create an "),Wa=n("code"),hr=s("IterableDataset"),cr=s(` on the other hand, you must provide a \u201Clazy\u201D way to load the data.
In Python, we generally use generator functions. These functions `),Fa=n("code"),mr=s("yield"),ur=s(" one example at a time, which means you can\u2019t access a row by slicing it like a regular "),Va=n("code"),_r=s("Dataset"),yr=s(":"),jt=p(),c(je.$$.fragment),Dt=p(),G=n("h2"),ae=n("a"),Ya=n("span"),c(De.$$.fragment),gr=p(),Ga=n("span"),br=s("Loading local files entirely and progressively"),Et=p(),P=n("p"),vr=s("It is possible to convert local or remote data files to an Arrow "),Ze=n("a"),wr=s("Dataset"),kr=s(" using "),ea=n("a"),$r=s("load_dataset()"),jr=s(":"),xt=p(),c(Ee.$$.fragment),It=p(),aa=n("p"),Dr=s("However, this requires a conversion step from CSV to Arrow format, which takes time and disk space if your dataset is big."),At=p(),te=n("p"),Er=s("To save disk space and skip the conversion step, you can define an "),Ma=n("code"),xr=s("IterableDataset"),Ir=s(` by streaming from the local files directly.
This way, the data is read progressively from the local files as you iterate over the dataset:`),qt=p(),c(xe.$$.fragment),St=p(),k=n("p"),Ar=s(`Many file formats are supported, like CSV, JSONL, and Parquet, as well as image and audio files.
You can find more information in the corresponding guides for loading `),ta=n("a"),qr=s("tabular"),Sr=s(", "),sa=n("a"),Pr=s("text"),Tr=s(", "),ra=n("a"),Cr=s("vision"),zr=s(", and "),na=n("a"),Or=s("audio"),Nr=s(" datasets."),Pt=p(),M=n("h2"),se=n("a"),Ra=n("span"),c(Ie.$$.fragment),Br=p(),Ja=n("span"),Lr=s("Eager data processing and lazy data processing"),Tt=p(),I=n("p"),Hr=s("When you process a "),la=n("a"),Wr=s("Dataset"),Fr=s(" object using "),oa=n("a"),Vr=s("Dataset.map()"),Yr=s(`, the entire dataset is processed immediately and returned.
This is similar to how `),Ua=n("code"),Gr=s("pandas"),Mr=s(" works for example."),Ct=p(),c(Ae.$$.fragment),zt=p(),$=n("p"),Rr=s("On the other hand, due to the \u201Clazy\u201D nature of an "),Ka=n("code"),Jr=s("IterableDataset"),Ur=s(", calling "),ia=n("a"),Kr=s("IterableDataset.map()"),Qr=s(" does not apply your "),Qa=n("code"),Xr=s("map"),Zr=s(` function over the full dataset.
Instead, your `),Xa=n("code"),en=s("map"),an=s(" function is applied on-the-fly."),Ot=p(),fa=n("p"),tn=s("Because of that, you can chain multiple processing steps and they will all run at once when you start iterating over the dataset:"),Nt=p(),c(qe.$$.fragment),Bt=p(),R=n("h2"),re=n("a"),Za=n("span"),c(Se.$$.fragment),sn=p(),et=n("span"),rn=s("Exact and fast approximate shuffling"),Lt=p(),j=n("p"),nn=s("When you shuffle a "),da=n("a"),ln=s("Dataset"),on=s(" using "),pa=n("a"),fn=s("Dataset.shuffle()"),dn=s(`, you apply an exact shuffling of the dataset.
It works by taking a list of indices `),at=n("code"),pn=s("[0, 1, 2, ... len(my_dataset) - 1]"),hn=s(` and shuffling this list.
Then, accessing `),tt=n("code"),cn=s("my_dataset[0]"),mn=s(" returns the row and index defined by the first element of the indices mapping that has been shuffled:"),Ht=p(),c(Pe.$$.fragment),Wt=p(),T=n("p"),un=s("Since we don\u2019t have random access to the rows in the case of an "),st=n("code"),_n=s("IterableDataset"),yn=s(`, we can\u2019t use a shuffled list of indices and access a row at an arbitrary position.
This prevents the use of exact shuffling.
Instead, a fast approximate shuffling is used in `),ha=n("a"),gn=s("IterableDataset.shuffle()"),bn=s(`.
It uses a shuffle buffer to sample random examples iteratively from the dataset.
Since the dataset is still read iteratively, it provides excellent speed performance:`),Ft=p(),c(Te.$$.fragment),Vt=p(),ne=n("p"),vn=s("But using a shuffle buffer is not enough to provide a satisfactory shuffling for machine learning model training. So "),ca=n("a"),wn=s("IterableDataset.shuffle()"),kn=s(" also shuffles the dataset shards if your dataset is made of multiple files or sources:"),Yt=p(),c(Ce.$$.fragment),Gt=p(),J=n("h2"),le=n("a"),rt=n("span"),c(ze.$$.fragment),$n=p(),nt=n("span"),jn=s("Speed differences"),Mt=p(),C=n("p"),Dn=s("Regular "),ma=n("a"),En=s("Dataset"),xn=s(` objects are based on Arrow which provides fast random access to the rows.
Thanks to memory mapping and the fact that Arrow is an in-memory format, reading data from disk doesn\u2019t do expensive system calls and deserialization.
It provides even faster data loading when iterating using a `),lt=n("code"),In=s("for"),An=s(" loop by iterating on contiguous Arrow record batches."),Rt=p(),A=n("p"),qn=s("However as soon as your "),ua=n("a"),Sn=s("Dataset"),Pn=s(" has an indices mapping (via "),_a=n("a"),Tn=s("Dataset.shuffle()"),Cn=s(` for example), the speed can become 10x slower.
This is because there is an extra step to get the row index to read using the indices mapping, and most importantly, you aren\u2019t reading contiguous chunks of data anymore.
To restore the speed, you\u2019d need to rewrite the entire dataset on your disk again using `),ya=n("a"),zn=s("Dataset.flatten_indices()"),On=s(`, which removes the indices mapping.
This may take a lot of time depending of the size of your dataset though:`),Jt=p(),c(Oe.$$.fragment),Ut=p(),z=n("p"),Nn=s("In this case, we recommend switching to an "),ga=n("a"),Bn=s("IterableDataset"),Ln=s(" and leveraging its fast approximate shuffling method "),ba=n("a"),Hn=s("IterableDataset.shuffle()"),Wn=s(`.
It only shuffles the shards order and adds a shuffle buffer to your dataset, which keeps the speed of your dataset optimal.
You can also reshuffle the dataset easily:`),Kt=p(),c(Ne.$$.fragment),Qt=p(),oe=n("p"),Fn=s("If you\u2019re using your dataset on multiple epochs, the effective seed to shuffle the shards order in the shuffle buffer is "),ot=n("code"),Vn=s("seed + epoch"),Yn=s(`.
It makes it easy to reshuffle a dataset betwen epochs:`),Xt=p(),c(Be.$$.fragment),Zt=p(),U=n("h2"),ie=n("a"),it=n("span"),c(Le.$$.fragment),Gn=p(),ft=n("span"),Mn=s("Switch from map-style to iterable"),es=p(),q=n("p"),Rn=s("If you want to benefit from the \u201Clazy\u201D behavior of an "),va=n("a"),Jn=s("IterableDataset"),Un=s(" or their speed advantages, you can switch your map-style "),wa=n("a"),Kn=s("Dataset"),Qn=s(" to an "),ka=n("a"),Xn=s("IterableDataset"),Zn=s(":"),as=p(),c(He.$$.fragment),ts=p(),O=n("p"),el=s("If you want to shuffle your dataset or "),$a=n("a"),al=s("use it with a PyTorch DataLoader"),tl=s(", we recommend generating a shared "),ja=n("a"),sl=s("IterableDataset"),rl=s(":"),ss=p(),c(We.$$.fragment),this.h()},l(e){const i=No('[data-svelte="svelte-1phssyn"]',document.head);W=l(i,"META",{name:!0,content:!0}),i.forEach(t),dt=h(e),F=l(e,"H1",{class:!0});var ns=o(F);K=l(ns,"A",{id:!0,class:!0,href:!0});var ll=o(K);Sa=l(ll,"SPAN",{});var ol=o(Sa);m(ue.$$.fragment,ol),ol.forEach(t),ll.forEach(t),gs=h(ns),Pa=l(ns,"SPAN",{});var il=o(Pa);bs=r(il,"Differences between Dataset and IterableDataset"),il.forEach(t),ns.forEach(t),pt=h(e),b=l(e,"P",{});var D=o(b);vs=r(D,"There are two types of dataset objects, a "),Fe=l(D,"A",{href:!0});var fl=o(Fe);ws=r(fl,"Dataset"),fl.forEach(t),ks=r(D," and an "),Ve=l(D,"A",{href:!0});var dl=o(Ve);$s=r(dl,"IterableDataset"),dl.forEach(t),js=r(D,`.
Whichever type of dataset you choose to use or create depends on the size of the dataset.
In general, an `),Ye=l(D,"A",{href:!0});var pl=o(Ye);Ds=r(pl,"IterableDataset"),pl.forEach(t),Es=r(D," is ideal for big datasets (think hundreds of GBs!) due to its lazy behavior and speed advantages, while a "),Ge=l(D,"A",{href:!0});var hl=o(Ge);xs=r(hl,"Dataset"),hl.forEach(t),Is=r(D,` is great for everything else.
This page will compare the differences between a `),Me=l(D,"A",{href:!0});var cl=o(Me);As=r(cl,"Dataset"),cl.forEach(t),qs=r(D," and an "),Re=l(D,"A",{href:!0});var ml=o(Re);Ss=r(ml,"IterableDataset"),ml.forEach(t),Ps=r(D," to help you pick the right dataset object for you."),D.forEach(t),ht=h(e),V=l(e,"H2",{class:!0});var ls=o(V);Q=l(ls,"A",{id:!0,class:!0,href:!0});var ul=o(Q);Ta=l(ul,"SPAN",{});var _l=o(Ta);m(_e.$$.fragment,_l),_l.forEach(t),ul.forEach(t),Ts=h(ls),Ca=l(ls,"SPAN",{});var yl=o(Ca);Cs=r(yl,"Downloading and streaming"),yl.forEach(t),ls.forEach(t),ct=h(e),S=l(e,"P",{});var Da=o(S);zs=r(Da,"When you have a regular "),Je=l(Da,"A",{href:!0});var gl=o(Je);Os=r(gl,"Dataset"),gl.forEach(t),Ns=r(Da,", you can access it using "),za=l(Da,"CODE",{});var bl=o(za);Bs=r(bl,"my_dataset[0]"),bl.forEach(t),Ls=r(Da,`. This provides random access to the rows.
Such datasets are also called \u201Cmap-style\u201D datasets.
For example you can download ImageNet-1k like this and access any row:`),Da.forEach(t),mt=h(e),m(ye.$$.fragment,e),ut=h(e),E=l(e,"P",{});var fe=o(E);Hs=r(fe,`But one caveat is that you must have the entire dataset stored on your disk or in memory, which blocks you from accessing datasets bigger than the disk.
Because it can become inconvenient for big datasets, there exists another type of dataset, the `),Ue=l(fe,"A",{href:!0});var vl=o(Ue);Ws=r(vl,"IterableDataset"),vl.forEach(t),Fs=r(fe,`.
When you have an `),Oa=l(fe,"CODE",{});var wl=o(Oa);Vs=r(wl,"IterableDataset"),wl.forEach(t),Ys=r(fe,", you can access it using a "),Na=l(fe,"CODE",{});var kl=o(Na);Gs=r(kl,"for"),kl.forEach(t),Ms=r(fe,` loop to load the data progressively as you iterate over the dataset.
This way, only a small fraction of examples is loaded in memory, and you don\u2019t write anything on disk.`),fe.forEach(t),_t=h(e),Ke=l(e,"P",{});var $l=o(Ke);Rs=r($l,"For example, you can stream the ImageNet-1k dataset without downloading it on disk:"),$l.forEach(t),yt=h(e),m(ge.$$.fragment,e),gt=h(e),w=l(e,"P",{});var N=o(w);Js=r(N,`Streaming can read online data without writing any file to disk.
For example, you can stream datasets made out of multiple shards, each of which is hundreds of gigabytes like `),be=l(N,"A",{href:!0,rel:!0});var jl=o(be);Us=r(jl,"C4"),jl.forEach(t),Ks=r(N,", "),ve=l(N,"A",{href:!0,rel:!0});var Dl=o(ve);Qs=r(Dl,"OSCAR"),Dl.forEach(t),Xs=r(N," or "),we=l(N,"A",{href:!0,rel:!0});var El=o(we);Zs=r(El,"LAION-2B"),El.forEach(t),er=r(N,`.
Learn more about how to stream a dataset in the `),Qe=l(N,"A",{href:!0});var xl=o(Qe);ar=r(xl,"Dataset Streaming Guide"),xl.forEach(t),tr=r(N,"."),N.forEach(t),bt=h(e),X=l(e,"P",{});var os=o(X);sr=r(os,"This is not the only difference though, because the \u201Clazy\u201D behavior of an "),Ba=l(os,"CODE",{});var Il=o(Ba);rr=r(Il,"IterableDataset"),Il.forEach(t),nr=r(os," is also present when it comes to dataset creation and processing."),os.forEach(t),vt=h(e),Y=l(e,"H2",{class:!0});var is=o(Y);Z=l(is,"A",{id:!0,class:!0,href:!0});var Al=o(Z);La=l(Al,"SPAN",{});var ql=o(La);m(ke.$$.fragment,ql),ql.forEach(t),Al.forEach(t),lr=h(is),Ha=l(is,"SPAN",{});var Sl=o(Ha);or=r(Sl,"Creating map-style datasets and iterable datasets"),Sl.forEach(t),is.forEach(t),wt=h(e),ee=l(e,"P",{});var fs=o(ee);ir=r(fs,"You can create a "),Xe=l(fs,"A",{href:!0});var Pl=o(Xe);fr=r(Pl,"Dataset"),Pl.forEach(t),dr=r(fs," using lists or dictionaries, and the data is entirely converted to Arrow so you can easily access any row:"),fs.forEach(t),kt=h(e),m($e.$$.fragment,e),$t=h(e),x=l(e,"P",{});var de=o(x);pr=r(de,"To create an "),Wa=l(de,"CODE",{});var Tl=o(Wa);hr=r(Tl,"IterableDataset"),Tl.forEach(t),cr=r(de,` on the other hand, you must provide a \u201Clazy\u201D way to load the data.
In Python, we generally use generator functions. These functions `),Fa=l(de,"CODE",{});var Cl=o(Fa);mr=r(Cl,"yield"),Cl.forEach(t),ur=r(de," one example at a time, which means you can\u2019t access a row by slicing it like a regular "),Va=l(de,"CODE",{});var zl=o(Va);_r=r(zl,"Dataset"),zl.forEach(t),yr=r(de,":"),de.forEach(t),jt=h(e),m(je.$$.fragment,e),Dt=h(e),G=l(e,"H2",{class:!0});var ds=o(G);ae=l(ds,"A",{id:!0,class:!0,href:!0});var Ol=o(ae);Ya=l(Ol,"SPAN",{});var Nl=o(Ya);m(De.$$.fragment,Nl),Nl.forEach(t),Ol.forEach(t),gr=h(ds),Ga=l(ds,"SPAN",{});var Bl=o(Ga);br=r(Bl,"Loading local files entirely and progressively"),Bl.forEach(t),ds.forEach(t),Et=h(e),P=l(e,"P",{});var Ea=o(P);vr=r(Ea,"It is possible to convert local or remote data files to an Arrow "),Ze=l(Ea,"A",{href:!0});var Ll=o(Ze);wr=r(Ll,"Dataset"),Ll.forEach(t),kr=r(Ea," using "),ea=l(Ea,"A",{href:!0});var Hl=o(ea);$r=r(Hl,"load_dataset()"),Hl.forEach(t),jr=r(Ea,":"),Ea.forEach(t),xt=h(e),m(Ee.$$.fragment,e),It=h(e),aa=l(e,"P",{});var Wl=o(aa);Dr=r(Wl,"However, this requires a conversion step from CSV to Arrow format, which takes time and disk space if your dataset is big."),Wl.forEach(t),At=h(e),te=l(e,"P",{});var ps=o(te);Er=r(ps,"To save disk space and skip the conversion step, you can define an "),Ma=l(ps,"CODE",{});var Fl=o(Ma);xr=r(Fl,"IterableDataset"),Fl.forEach(t),Ir=r(ps,` by streaming from the local files directly.
This way, the data is read progressively from the local files as you iterate over the dataset:`),ps.forEach(t),qt=h(e),m(xe.$$.fragment,e),St=h(e),k=l(e,"P",{});var B=o(k);Ar=r(B,`Many file formats are supported, like CSV, JSONL, and Parquet, as well as image and audio files.
You can find more information in the corresponding guides for loading `),ta=l(B,"A",{href:!0});var Vl=o(ta);qr=r(Vl,"tabular"),Vl.forEach(t),Sr=r(B,", "),sa=l(B,"A",{href:!0});var Yl=o(sa);Pr=r(Yl,"text"),Yl.forEach(t),Tr=r(B,", "),ra=l(B,"A",{href:!0});var Gl=o(ra);Cr=r(Gl,"vision"),Gl.forEach(t),zr=r(B,", and "),na=l(B,"A",{href:!0});var Ml=o(na);Or=r(Ml,"audio"),Ml.forEach(t),Nr=r(B," datasets."),B.forEach(t),Pt=h(e),M=l(e,"H2",{class:!0});var hs=o(M);se=l(hs,"A",{id:!0,class:!0,href:!0});var Rl=o(se);Ra=l(Rl,"SPAN",{});var Jl=o(Ra);m(Ie.$$.fragment,Jl),Jl.forEach(t),Rl.forEach(t),Br=h(hs),Ja=l(hs,"SPAN",{});var Ul=o(Ja);Lr=r(Ul,"Eager data processing and lazy data processing"),Ul.forEach(t),hs.forEach(t),Tt=h(e),I=l(e,"P",{});var pe=o(I);Hr=r(pe,"When you process a "),la=l(pe,"A",{href:!0});var Kl=o(la);Wr=r(Kl,"Dataset"),Kl.forEach(t),Fr=r(pe," object using "),oa=l(pe,"A",{href:!0});var Ql=o(oa);Vr=r(Ql,"Dataset.map()"),Ql.forEach(t),Yr=r(pe,`, the entire dataset is processed immediately and returned.
This is similar to how `),Ua=l(pe,"CODE",{});var Xl=o(Ua);Gr=r(Xl,"pandas"),Xl.forEach(t),Mr=r(pe," works for example."),pe.forEach(t),Ct=h(e),m(Ae.$$.fragment,e),zt=h(e),$=l(e,"P",{});var L=o($);Rr=r(L,"On the other hand, due to the \u201Clazy\u201D nature of an "),Ka=l(L,"CODE",{});var Zl=o(Ka);Jr=r(Zl,"IterableDataset"),Zl.forEach(t),Ur=r(L,", calling "),ia=l(L,"A",{href:!0});var eo=o(ia);Kr=r(eo,"IterableDataset.map()"),eo.forEach(t),Qr=r(L," does not apply your "),Qa=l(L,"CODE",{});var ao=o(Qa);Xr=r(ao,"map"),ao.forEach(t),Zr=r(L,` function over the full dataset.
Instead, your `),Xa=l(L,"CODE",{});var to=o(Xa);en=r(to,"map"),to.forEach(t),an=r(L," function is applied on-the-fly."),L.forEach(t),Ot=h(e),fa=l(e,"P",{});var so=o(fa);tn=r(so,"Because of that, you can chain multiple processing steps and they will all run at once when you start iterating over the dataset:"),so.forEach(t),Nt=h(e),m(qe.$$.fragment,e),Bt=h(e),R=l(e,"H2",{class:!0});var cs=o(R);re=l(cs,"A",{id:!0,class:!0,href:!0});var ro=o(re);Za=l(ro,"SPAN",{});var no=o(Za);m(Se.$$.fragment,no),no.forEach(t),ro.forEach(t),sn=h(cs),et=l(cs,"SPAN",{});var lo=o(et);rn=r(lo,"Exact and fast approximate shuffling"),lo.forEach(t),cs.forEach(t),Lt=h(e),j=l(e,"P",{});var H=o(j);nn=r(H,"When you shuffle a "),da=l(H,"A",{href:!0});var oo=o(da);ln=r(oo,"Dataset"),oo.forEach(t),on=r(H," using "),pa=l(H,"A",{href:!0});var io=o(pa);fn=r(io,"Dataset.shuffle()"),io.forEach(t),dn=r(H,`, you apply an exact shuffling of the dataset.
It works by taking a list of indices `),at=l(H,"CODE",{});var fo=o(at);pn=r(fo,"[0, 1, 2, ... len(my_dataset) - 1]"),fo.forEach(t),hn=r(H,` and shuffling this list.
Then, accessing `),tt=l(H,"CODE",{});var po=o(tt);cn=r(po,"my_dataset[0]"),po.forEach(t),mn=r(H," returns the row and index defined by the first element of the indices mapping that has been shuffled:"),H.forEach(t),Ht=h(e),m(Pe.$$.fragment,e),Wt=h(e),T=l(e,"P",{});var xa=o(T);un=r(xa,"Since we don\u2019t have random access to the rows in the case of an "),st=l(xa,"CODE",{});var ho=o(st);_n=r(ho,"IterableDataset"),ho.forEach(t),yn=r(xa,`, we can\u2019t use a shuffled list of indices and access a row at an arbitrary position.
This prevents the use of exact shuffling.
Instead, a fast approximate shuffling is used in `),ha=l(xa,"A",{href:!0});var co=o(ha);gn=r(co,"IterableDataset.shuffle()"),co.forEach(t),bn=r(xa,`.
It uses a shuffle buffer to sample random examples iteratively from the dataset.
Since the dataset is still read iteratively, it provides excellent speed performance:`),xa.forEach(t),Ft=h(e),m(Te.$$.fragment,e),Vt=h(e),ne=l(e,"P",{});var ms=o(ne);vn=r(ms,"But using a shuffle buffer is not enough to provide a satisfactory shuffling for machine learning model training. So "),ca=l(ms,"A",{href:!0});var mo=o(ca);wn=r(mo,"IterableDataset.shuffle()"),mo.forEach(t),kn=r(ms," also shuffles the dataset shards if your dataset is made of multiple files or sources:"),ms.forEach(t),Yt=h(e),m(Ce.$$.fragment,e),Gt=h(e),J=l(e,"H2",{class:!0});var us=o(J);le=l(us,"A",{id:!0,class:!0,href:!0});var uo=o(le);rt=l(uo,"SPAN",{});var _o=o(rt);m(ze.$$.fragment,_o),_o.forEach(t),uo.forEach(t),$n=h(us),nt=l(us,"SPAN",{});var yo=o(nt);jn=r(yo,"Speed differences"),yo.forEach(t),us.forEach(t),Mt=h(e),C=l(e,"P",{});var Ia=o(C);Dn=r(Ia,"Regular "),ma=l(Ia,"A",{href:!0});var go=o(ma);En=r(go,"Dataset"),go.forEach(t),xn=r(Ia,` objects are based on Arrow which provides fast random access to the rows.
Thanks to memory mapping and the fact that Arrow is an in-memory format, reading data from disk doesn\u2019t do expensive system calls and deserialization.
It provides even faster data loading when iterating using a `),lt=l(Ia,"CODE",{});var bo=o(lt);In=r(bo,"for"),bo.forEach(t),An=r(Ia," loop by iterating on contiguous Arrow record batches."),Ia.forEach(t),Rt=h(e),A=l(e,"P",{});var he=o(A);qn=r(he,"However as soon as your "),ua=l(he,"A",{href:!0});var vo=o(ua);Sn=r(vo,"Dataset"),vo.forEach(t),Pn=r(he," has an indices mapping (via "),_a=l(he,"A",{href:!0});var wo=o(_a);Tn=r(wo,"Dataset.shuffle()"),wo.forEach(t),Cn=r(he,` for example), the speed can become 10x slower.
This is because there is an extra step to get the row index to read using the indices mapping, and most importantly, you aren\u2019t reading contiguous chunks of data anymore.
To restore the speed, you\u2019d need to rewrite the entire dataset on your disk again using `),ya=l(he,"A",{href:!0});var ko=o(ya);zn=r(ko,"Dataset.flatten_indices()"),ko.forEach(t),On=r(he,`, which removes the indices mapping.
This may take a lot of time depending of the size of your dataset though:`),he.forEach(t),Jt=h(e),m(Oe.$$.fragment,e),Ut=h(e),z=l(e,"P",{});var Aa=o(z);Nn=r(Aa,"In this case, we recommend switching to an "),ga=l(Aa,"A",{href:!0});var $o=o(ga);Bn=r($o,"IterableDataset"),$o.forEach(t),Ln=r(Aa," and leveraging its fast approximate shuffling method "),ba=l(Aa,"A",{href:!0});var jo=o(ba);Hn=r(jo,"IterableDataset.shuffle()"),jo.forEach(t),Wn=r(Aa,`.
It only shuffles the shards order and adds a shuffle buffer to your dataset, which keeps the speed of your dataset optimal.
You can also reshuffle the dataset easily:`),Aa.forEach(t),Kt=h(e),m(Ne.$$.fragment,e),Qt=h(e),oe=l(e,"P",{});var _s=o(oe);Fn=r(_s,"If you\u2019re using your dataset on multiple epochs, the effective seed to shuffle the shards order in the shuffle buffer is "),ot=l(_s,"CODE",{});var Do=o(ot);Vn=r(Do,"seed + epoch"),Do.forEach(t),Yn=r(_s,`.
It makes it easy to reshuffle a dataset betwen epochs:`),_s.forEach(t),Xt=h(e),m(Be.$$.fragment,e),Zt=h(e),U=l(e,"H2",{class:!0});var ys=o(U);ie=l(ys,"A",{id:!0,class:!0,href:!0});var Eo=o(ie);it=l(Eo,"SPAN",{});var xo=o(it);m(Le.$$.fragment,xo),xo.forEach(t),Eo.forEach(t),Gn=h(ys),ft=l(ys,"SPAN",{});var Io=o(ft);Mn=r(Io,"Switch from map-style to iterable"),Io.forEach(t),ys.forEach(t),es=h(e),q=l(e,"P",{});var ce=o(q);Rn=r(ce,"If you want to benefit from the \u201Clazy\u201D behavior of an "),va=l(ce,"A",{href:!0});var Ao=o(va);Jn=r(Ao,"IterableDataset"),Ao.forEach(t),Un=r(ce," or their speed advantages, you can switch your map-style "),wa=l(ce,"A",{href:!0});var qo=o(wa);Kn=r(qo,"Dataset"),qo.forEach(t),Qn=r(ce," to an "),ka=l(ce,"A",{href:!0});var So=o(ka);Xn=r(So,"IterableDataset"),So.forEach(t),Zn=r(ce,":"),ce.forEach(t),as=h(e),m(He.$$.fragment,e),ts=h(e),O=l(e,"P",{});var qa=o(O);el=r(qa,"If you want to shuffle your dataset or "),$a=l(qa,"A",{href:!0});var Po=o($a);al=r(Po,"use it with a PyTorch DataLoader"),Po.forEach(t),tl=r(qa,", we recommend generating a shared "),ja=l(qa,"A",{href:!0});var To=o(ja);sl=r(To,"IterableDataset"),To.forEach(t),rl=r(qa,":"),qa.forEach(t),ss=h(e),m(We.$$.fragment,e),this.h()},h(){d(W,"name","hf:doc:metadata"),d(W,"content",JSON.stringify(Wo)),d(K,"id","differences-between-dataset-and-iterabledataset"),d(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(K,"href","#differences-between-dataset-and-iterabledataset"),d(F,"class","relative group"),d(Fe,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(Ve,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset"),d(Ye,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset"),d(Ge,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(Me,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(Re,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset"),d(Q,"id","downloading-and-streaming"),d(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Q,"href","#downloading-and-streaming"),d(V,"class","relative group"),d(Je,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(Ue,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset"),d(be,"href","https://huggingface.co/datasets/c4"),d(be,"rel","nofollow"),d(ve,"href","https://huggingface.co/datasets/oscar"),d(ve,"rel","nofollow"),d(we,"href","https://huggingface.co/datasets/laion/laion2B-en"),d(we,"rel","nofollow"),d(Qe,"href","./stream"),d(Z,"id","creating-mapstyle-datasets-and-iterable-datasets"),d(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Z,"href","#creating-mapstyle-datasets-and-iterable-datasets"),d(Y,"class","relative group"),d(Xe,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(ae,"id","loading-local-files-entirely-and-progressively"),d(ae,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ae,"href","#loading-local-files-entirely-and-progressively"),d(G,"class","relative group"),d(Ze,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(ea,"href","/docs/datasets/v2.11.0/en/package_reference/loading_methods#datasets.load_dataset"),d(ta,"href","./tabular_load"),d(sa,"href","./nlp_load"),d(ra,"href","./image_load"),d(na,"href","./audio_load%5D"),d(se,"id","eager-data-processing-and-lazy-data-processing"),d(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(se,"href","#eager-data-processing-and-lazy-data-processing"),d(M,"class","relative group"),d(la,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(oa,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.map"),d(ia,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset.map"),d(re,"id","exact-and-fast-approximate-shuffling"),d(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(re,"href","#exact-and-fast-approximate-shuffling"),d(R,"class","relative group"),d(da,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(pa,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.shuffle"),d(ha,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset.shuffle"),d(ca,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset.shuffle"),d(le,"id","speed-differences"),d(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(le,"href","#speed-differences"),d(J,"class","relative group"),d(ma,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(ua,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(_a,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.shuffle"),d(ya,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.flatten_indices"),d(ga,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset"),d(ba,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset.shuffle"),d(ie,"id","switch-from-mapstyle-to-iterable"),d(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ie,"href","#switch-from-mapstyle-to-iterable"),d(U,"class","relative group"),d(va,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset"),d(wa,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),d(ka,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset"),d($a,"href","./use_with_pytorch#stream-data"),d(ja,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset")},m(e,i){a(document.head,W),f(e,dt,i),f(e,F,i),a(F,K),a(K,Sa),u(ue,Sa,null),a(F,gs),a(F,Pa),a(Pa,bs),f(e,pt,i),f(e,b,i),a(b,vs),a(b,Fe),a(Fe,ws),a(b,ks),a(b,Ve),a(Ve,$s),a(b,js),a(b,Ye),a(Ye,Ds),a(b,Es),a(b,Ge),a(Ge,xs),a(b,Is),a(b,Me),a(Me,As),a(b,qs),a(b,Re),a(Re,Ss),a(b,Ps),f(e,ht,i),f(e,V,i),a(V,Q),a(Q,Ta),u(_e,Ta,null),a(V,Ts),a(V,Ca),a(Ca,Cs),f(e,ct,i),f(e,S,i),a(S,zs),a(S,Je),a(Je,Os),a(S,Ns),a(S,za),a(za,Bs),a(S,Ls),f(e,mt,i),u(ye,e,i),f(e,ut,i),f(e,E,i),a(E,Hs),a(E,Ue),a(Ue,Ws),a(E,Fs),a(E,Oa),a(Oa,Vs),a(E,Ys),a(E,Na),a(Na,Gs),a(E,Ms),f(e,_t,i),f(e,Ke,i),a(Ke,Rs),f(e,yt,i),u(ge,e,i),f(e,gt,i),f(e,w,i),a(w,Js),a(w,be),a(be,Us),a(w,Ks),a(w,ve),a(ve,Qs),a(w,Xs),a(w,we),a(we,Zs),a(w,er),a(w,Qe),a(Qe,ar),a(w,tr),f(e,bt,i),f(e,X,i),a(X,sr),a(X,Ba),a(Ba,rr),a(X,nr),f(e,vt,i),f(e,Y,i),a(Y,Z),a(Z,La),u(ke,La,null),a(Y,lr),a(Y,Ha),a(Ha,or),f(e,wt,i),f(e,ee,i),a(ee,ir),a(ee,Xe),a(Xe,fr),a(ee,dr),f(e,kt,i),u($e,e,i),f(e,$t,i),f(e,x,i),a(x,pr),a(x,Wa),a(Wa,hr),a(x,cr),a(x,Fa),a(Fa,mr),a(x,ur),a(x,Va),a(Va,_r),a(x,yr),f(e,jt,i),u(je,e,i),f(e,Dt,i),f(e,G,i),a(G,ae),a(ae,Ya),u(De,Ya,null),a(G,gr),a(G,Ga),a(Ga,br),f(e,Et,i),f(e,P,i),a(P,vr),a(P,Ze),a(Ze,wr),a(P,kr),a(P,ea),a(ea,$r),a(P,jr),f(e,xt,i),u(Ee,e,i),f(e,It,i),f(e,aa,i),a(aa,Dr),f(e,At,i),f(e,te,i),a(te,Er),a(te,Ma),a(Ma,xr),a(te,Ir),f(e,qt,i),u(xe,e,i),f(e,St,i),f(e,k,i),a(k,Ar),a(k,ta),a(ta,qr),a(k,Sr),a(k,sa),a(sa,Pr),a(k,Tr),a(k,ra),a(ra,Cr),a(k,zr),a(k,na),a(na,Or),a(k,Nr),f(e,Pt,i),f(e,M,i),a(M,se),a(se,Ra),u(Ie,Ra,null),a(M,Br),a(M,Ja),a(Ja,Lr),f(e,Tt,i),f(e,I,i),a(I,Hr),a(I,la),a(la,Wr),a(I,Fr),a(I,oa),a(oa,Vr),a(I,Yr),a(I,Ua),a(Ua,Gr),a(I,Mr),f(e,Ct,i),u(Ae,e,i),f(e,zt,i),f(e,$,i),a($,Rr),a($,Ka),a(Ka,Jr),a($,Ur),a($,ia),a(ia,Kr),a($,Qr),a($,Qa),a(Qa,Xr),a($,Zr),a($,Xa),a(Xa,en),a($,an),f(e,Ot,i),f(e,fa,i),a(fa,tn),f(e,Nt,i),u(qe,e,i),f(e,Bt,i),f(e,R,i),a(R,re),a(re,Za),u(Se,Za,null),a(R,sn),a(R,et),a(et,rn),f(e,Lt,i),f(e,j,i),a(j,nn),a(j,da),a(da,ln),a(j,on),a(j,pa),a(pa,fn),a(j,dn),a(j,at),a(at,pn),a(j,hn),a(j,tt),a(tt,cn),a(j,mn),f(e,Ht,i),u(Pe,e,i),f(e,Wt,i),f(e,T,i),a(T,un),a(T,st),a(st,_n),a(T,yn),a(T,ha),a(ha,gn),a(T,bn),f(e,Ft,i),u(Te,e,i),f(e,Vt,i),f(e,ne,i),a(ne,vn),a(ne,ca),a(ca,wn),a(ne,kn),f(e,Yt,i),u(Ce,e,i),f(e,Gt,i),f(e,J,i),a(J,le),a(le,rt),u(ze,rt,null),a(J,$n),a(J,nt),a(nt,jn),f(e,Mt,i),f(e,C,i),a(C,Dn),a(C,ma),a(ma,En),a(C,xn),a(C,lt),a(lt,In),a(C,An),f(e,Rt,i),f(e,A,i),a(A,qn),a(A,ua),a(ua,Sn),a(A,Pn),a(A,_a),a(_a,Tn),a(A,Cn),a(A,ya),a(ya,zn),a(A,On),f(e,Jt,i),u(Oe,e,i),f(e,Ut,i),f(e,z,i),a(z,Nn),a(z,ga),a(ga,Bn),a(z,Ln),a(z,ba),a(ba,Hn),a(z,Wn),f(e,Kt,i),u(Ne,e,i),f(e,Qt,i),f(e,oe,i),a(oe,Fn),a(oe,ot),a(ot,Vn),a(oe,Yn),f(e,Xt,i),u(Be,e,i),f(e,Zt,i),f(e,U,i),a(U,ie),a(ie,it),u(Le,it,null),a(U,Gn),a(U,ft),a(ft,Mn),f(e,es,i),f(e,q,i),a(q,Rn),a(q,va),a(va,Jn),a(q,Un),a(q,wa),a(wa,Kn),a(q,Qn),a(q,ka),a(ka,Xn),a(q,Zn),f(e,as,i),u(He,e,i),f(e,ts,i),f(e,O,i),a(O,el),a(O,$a),a($a,al),a(O,tl),a(O,ja),a(ja,sl),a(O,rl),f(e,ss,i),u(We,e,i),rs=!0},p:Bo,i(e){rs||(_(ue.$$.fragment,e),_(_e.$$.fragment,e),_(ye.$$.fragment,e),_(ge.$$.fragment,e),_(ke.$$.fragment,e),_($e.$$.fragment,e),_(je.$$.fragment,e),_(De.$$.fragment,e),_(Ee.$$.fragment,e),_(xe.$$.fragment,e),_(Ie.$$.fragment,e),_(Ae.$$.fragment,e),_(qe.$$.fragment,e),_(Se.$$.fragment,e),_(Pe.$$.fragment,e),_(Te.$$.fragment,e),_(Ce.$$.fragment,e),_(ze.$$.fragment,e),_(Oe.$$.fragment,e),_(Ne.$$.fragment,e),_(Be.$$.fragment,e),_(Le.$$.fragment,e),_(He.$$.fragment,e),_(We.$$.fragment,e),rs=!0)},o(e){y(ue.$$.fragment,e),y(_e.$$.fragment,e),y(ye.$$.fragment,e),y(ge.$$.fragment,e),y(ke.$$.fragment,e),y($e.$$.fragment,e),y(je.$$.fragment,e),y(De.$$.fragment,e),y(Ee.$$.fragment,e),y(xe.$$.fragment,e),y(Ie.$$.fragment,e),y(Ae.$$.fragment,e),y(qe.$$.fragment,e),y(Se.$$.fragment,e),y(Pe.$$.fragment,e),y(Te.$$.fragment,e),y(Ce.$$.fragment,e),y(ze.$$.fragment,e),y(Oe.$$.fragment,e),y(Ne.$$.fragment,e),y(Be.$$.fragment,e),y(Le.$$.fragment,e),y(He.$$.fragment,e),y(We.$$.fragment,e),rs=!1},d(e){t(W),e&&t(dt),e&&t(F),g(ue),e&&t(pt),e&&t(b),e&&t(ht),e&&t(V),g(_e),e&&t(ct),e&&t(S),e&&t(mt),g(ye,e),e&&t(ut),e&&t(E),e&&t(_t),e&&t(Ke),e&&t(yt),g(ge,e),e&&t(gt),e&&t(w),e&&t(bt),e&&t(X),e&&t(vt),e&&t(Y),g(ke),e&&t(wt),e&&t(ee),e&&t(kt),g($e,e),e&&t($t),e&&t(x),e&&t(jt),g(je,e),e&&t(Dt),e&&t(G),g(De),e&&t(Et),e&&t(P),e&&t(xt),g(Ee,e),e&&t(It),e&&t(aa),e&&t(At),e&&t(te),e&&t(qt),g(xe,e),e&&t(St),e&&t(k),e&&t(Pt),e&&t(M),g(Ie),e&&t(Tt),e&&t(I),e&&t(Ct),g(Ae,e),e&&t(zt),e&&t($),e&&t(Ot),e&&t(fa),e&&t(Nt),g(qe,e),e&&t(Bt),e&&t(R),g(Se),e&&t(Lt),e&&t(j),e&&t(Ht),g(Pe,e),e&&t(Wt),e&&t(T),e&&t(Ft),g(Te,e),e&&t(Vt),e&&t(ne),e&&t(Yt),g(Ce,e),e&&t(Gt),e&&t(J),g(ze),e&&t(Mt),e&&t(C),e&&t(Rt),e&&t(A),e&&t(Jt),g(Oe,e),e&&t(Ut),e&&t(z),e&&t(Kt),g(Ne,e),e&&t(Qt),e&&t(oe),e&&t(Xt),g(Be,e),e&&t(Zt),e&&t(U),g(Le),e&&t(es),e&&t(q),e&&t(as),g(He,e),e&&t(ts),e&&t(O),e&&t(ss),g(We,e)}}}const Wo={local:"differences-between-dataset-and-iterabledataset",sections:[{local:"downloading-and-streaming",title:"Downloading and streaming"},{local:"creating-mapstyle-datasets-and-iterable-datasets",title:"Creating map-style datasets and iterable datasets"},{local:"loading-local-files-entirely-and-progressively",title:"Loading local files entirely and progressively"},{local:"eager-data-processing-and-lazy-data-processing",title:"Eager data processing and lazy data processing"},{local:"exact-and-fast-approximate-shuffling",title:"Exact and fast approximate shuffling"},{local:"speed-differences",title:"Speed differences"},{local:"switch-from-mapstyle-to-iterable",title:"Switch from map-style to iterable"}],title:"Differences between Dataset and IterableDataset"};function Fo(nl){return Lo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Mo extends Co{constructor(W){super();zo(this,W,Fo,Ho,Oo,{})}}export{Mo as default,Wo as metadata};
