import{S as Ao,i as To,s as So,e as r,k as h,w as f,t as e,M as Co,c as o,d as t,m as d,a as p,x as g,h as l,b as c,G as a,g as i,y as b,q as j,o as _,B as y,v as Lo}from"../chunks/vendor-hf-doc-builder.js";import{T as lr}from"../chunks/Tip-hf-doc-builder.js";import{I as G}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as T}from"../chunks/CodeBlock-hf-doc-builder.js";function Oo(ns){let m,q,u,E,D;return{c(){m=r("p"),q=e("A "),u=r("a"),E=e("Dataset"),D=e(" object is a wrapper of an Arrow table, which allows fast zero-copy reads from arrays in the dataset to PyTorch tensors."),this.h()},l($){m=o($,"P",{});var k=p(m);q=l(k,"A "),u=o(k,"A",{href:!0});var P=p(u);E=l(P,"Dataset"),P.forEach(t),D=l(k," object is a wrapper of an Arrow table, which allows fast zero-copy reads from arrays in the dataset to PyTorch tensors."),k.forEach(t),this.h()},h(){c(u,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset")},m($,k){i($,m,k),a(m,q),a(m,u),a(u,E),a(m,D)},d($){$&&t(m)}}}function zo(ns){let m,q,u,E,D,$,k,P,x,S,w;return{c(){m=r("p"),q=e("To use the "),u=r("a"),E=e("Image"),D=e(" feature type, you\u2019ll need to install the "),$=r("code"),k=e("vision"),P=e(` extra as
`),x=r("code"),S=e("pip install datasets[vision]"),w=e("."),this.h()},l(A){m=o(A,"P",{});var v=p(m);q=l(v,"To use the "),u=o(v,"A",{href:!0});var K=p(u);E=l(K,"Image"),K.forEach(t),D=l(v," feature type, you\u2019ll need to install the "),$=o(v,"CODE",{});var Z=p($);k=l(Z,"vision"),Z.forEach(t),P=l(v,` extra as
`),x=o(v,"CODE",{});var N=p(x);S=l(N,"pip install datasets[vision]"),N.forEach(t),w=l(v,"."),v.forEach(t),this.h()},h(){c(u,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Image")},m(A,v){i(A,m,v),a(m,q),a(m,u),a(u,E),a(m,D),a(m,$),a($,k),a(m,P),a(m,x),a(x,S),a(m,w)},d(A){A&&t(m)}}}function Io(ns){let m,q,u,E,D,$,k,P,x,S,w;return{c(){m=r("p"),q=e("To use the "),u=r("a"),E=e("Audio"),D=e(" feature type, you\u2019ll need to install the "),$=r("code"),k=e("audio"),P=e(` extra as
`),x=r("code"),S=e("pip install datasets[audio]"),w=e("."),this.h()},l(A){m=o(A,"P",{});var v=p(m);q=l(v,"To use the "),u=o(v,"A",{href:!0});var K=p(u);E=l(K,"Audio"),K.forEach(t),D=l(v," feature type, you\u2019ll need to install the "),$=o(v,"CODE",{});var Z=p($);k=l(Z,"audio"),Z.forEach(t),P=l(v,` extra as
`),x=o(v,"CODE",{});var N=p(x);S=l(N,"pip install datasets[audio]"),N.forEach(t),w=l(v,"."),v.forEach(t),this.h()},h(){c(u,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Audio")},m(A,v){i(A,m,v),a(m,q),a(m,u),a(u,E),a(m,D),a(m,$),a($,k),a(m,P),a(m,x),a(x,S),a(m,w)},d(A){A&&t(m)}}}function Fo(ns){let m,q,u,E,D,$,k,P,x,S,w,A,v,K,Z,N,Ne,Be,Ea,Ue,He,ka,Re,Me,$t,J,rs,Da,As,We,qa,Ye,wt,ta,Ge,Et,B,Ke,Pa,Ze,Je,ea,Qe,Ve,kt,Ts,Dt,os,qt,ps,Xe,xa,sl,al,Pt,Ss,xt,Q,is,Aa,Cs,tl,Ta,el,At,la,ll,Tt,Ls,St,hs,nl,Sa,rl,ol,Ct,Os,Lt,V,ds,Ca,zs,pl,La,il,Ot,Is,na,hl,dl,zt,Fs,It,ra,cl,Ft,U,ml,oa,ul,fl,pa,gl,bl,Nt,cs,Bt,Ns,Ut,ms,Ht,Bs,Rt,X,us,Oa,Us,jl,za,_l,Mt,C,yl,Ia,vl,$l,ia,wl,El,Fa,kl,Dl,Wt,Hs,Yt,ss,fs,Na,Rs,ql,Ba,Pl,Gt,ha,xl,Kt,as,gs,Ua,Ms,Al,Ha,Tl,Zt,H,Sl,Ra,Cl,Ll,Ma,Ol,zl,Jt,L,Il,Wa,Fl,Nl,Ya,Bl,Ul,Ga,Hl,Rl,Qt,Ws,Vt,ts,bs,Ka,Ys,Ml,Za,Wl,Xt,O,Yl,Ja,Gl,Kl,Qa,Zl,Jl,Va,Ql,Vl,se,Gs,ae,da,Xl,te,Ks,ee,R,sn,Xa,an,tn,st,en,ln,le,Zs,ne,z,nn,at,rn,on,tt,pn,hn,et,dn,cn,re,M,mn,lt,un,fn,nt,gn,bn,oe,es,js,rt,Js,jn,ot,_n,pe,_s,yn,ca,vn,$n,ie,I,wn,pt,En,kn,it,Dn,qn,ht,Pn,xn,he,Qs,de,ys,An,dt,Tn,Sn,ce,Vs,me,ma,Cn,ue,ls,vs,ct,Xs,Ln,mt,On,fe,$s,zn,ua,In,Fn,ge,sa,be,W,Nn,ut,Bn,Un,ft,Hn,Rn,je,fa,Mn,_e,ga,Wn,ye,ba,Yn,ve,F,Gn,gt,Kn,Zn,bt,Jn,Qn,jt,Vn,Xn,$e,ws,sr,_t,ar,tr,we;return $=new G({}),As=new G({}),Ts=new T({props:{code:`from datasets import Dataset
data = [[1, 2],[3, 4]]
ds = Dataset.from_dict({"data": data})
ds = ds.with_format("torch")
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>])}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
         [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])}`}}),os=new lr({props:{$$slots:{default:[Oo]},$$scope:{ctx:ns}}}),Ss=new T({props:{code:`import torch
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
ds = ds.with_format("torch", device=device)
ds[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span>device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>, device=device)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], device=<span class="hljs-string">&#x27;cuda:0&#x27;</span>)}`}}),Cs=new G({}),Ls=new T({props:{code:`from datasets import Dataset
data = [[[1, 2],[3, 4]],[[5, 6],[7, 8]]]
ds = Dataset.from_dict({"data": data})
ds = ds.with_format("torch")
ds[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: [tensor([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>]), tensor([<span class="hljs-number">3</span>, <span class="hljs-number">4</span>])]}`}}),Os=new T({props:{code:`from datasets import Dataset, Features, Array2D
data = [[[1, 2],[3, 4]],[[5, 6],[7, 8]]]
features = Features({"data": Array2D(shape=(2, 2), dtype='int32')})
ds = Dataset.from_dict({"data": data}, features=features)
ds = ds.with_format("torch")
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Array2D
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;data&quot;</span>: Array2D(shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=<span class="hljs-string">&#x27;int32&#x27;</span>)})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data}, features=features)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
         [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
          [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],
 
         [[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],
          [<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]])}`}}),zs=new G({}),Fs=new T({props:{code:`from datasets import Dataset, Features, ClassLabel
labels = [0, 0, 1]
features = Features({"label": ClassLabel(names=["negative", "positive"])})
ds = Dataset.from_dict({"label": labels}, features=features) 
ds = ds.with_format("torch")  
ds[:3]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, ClassLabel
<span class="hljs-meta">&gt;&gt;&gt; </span>labels = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;label&quot;</span>: ClassLabel(names=[<span class="hljs-string">&quot;negative&quot;</span>, <span class="hljs-string">&quot;positive&quot;</span>])})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;label&quot;</span>: labels}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">3</span>]
{<span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>])}`}}),cs=new lr({props:{$$slots:{default:[zo]},$$scope:{ctx:ns}}}),Ns=new T({props:{code:`from datasets import Dataset, Features, Audio, Image
images = ["path/to/image.png"] * 10
features = Features({"image": Image()})
ds = Dataset.from_dict({"image": images}, features=features) 
ds = ds.with_format("torch")
ds[0]["image"].shape
ds[0]
ds[:2]["image"].shape
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio, Image
<span class="hljs-meta">&gt;&gt;&gt; </span>images = [<span class="hljs-string">&quot;path/to/image.png&quot;</span>] * <span class="hljs-number">10</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;image&quot;</span>: Image()})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;image&quot;</span>: images}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>].shape
torch.Size([<span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">4</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: tensor([[[<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
         [<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
         ...,
         [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
         [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]], dtype=torch.uint8)}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>][<span class="hljs-string">&quot;image&quot;</span>].shape
torch.Size([<span class="hljs-number">2</span>, <span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">4</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: tensor([[[[<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
          [<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
          ...,
          [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
          [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]]], dtype=torch.uint8)}`}}),ms=new lr({props:{$$slots:{default:[Io]},$$scope:{ctx:ns}}}),Bs=new T({props:{code:`from datasets import Dataset, Features, Audio, Image
audio = ["path/to/audio.wav"] * 10
features = Features({"audio": Audio()})
ds = Dataset.from_dict({"audio": audio}, features=features) 
ds = ds.with_format("torch")  
ds[0]["audio"]["array"]
ds[0]["audio"]["sampling_rate"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio, Image
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = [<span class="hljs-string">&quot;path/to/audio.wav&quot;</span>] * <span class="hljs-number">10</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;audio&quot;</span>: Audio()})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;audio&quot;</span>: audio}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;torch&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]
tensor([ <span class="hljs-number">6.1035e-05</span>,  <span class="hljs-number">1.5259e-05</span>,  <span class="hljs-number">1.6785e-04</span>,  ..., -<span class="hljs-number">1.5259e-05</span>,
        -<span class="hljs-number">1.5259e-05</span>,  <span class="hljs-number">1.5259e-05</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;sampling_rate&quot;</span>]
tensor(<span class="hljs-number">44100</span>)`}}),Us=new G({}),Hs=new T({props:{code:`import numpy as np
from datasets import Dataset 
from torch.utils.data import DataLoader
data = np.random.rand(16)
label = np.random.randint(0, 2, size=16)
ds = Dataset.from_dict({"data": data, "label": label}).with_format("torch")
dataloader = DataLoader(ds, batch_size=4)
for batch in dataloader:
    print(batch)                                                                                            `,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset 
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-meta">&gt;&gt;&gt; </span>data = np.random.rand(<span class="hljs-number">16</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>label = np.random.randint(<span class="hljs-number">0</span>, <span class="hljs-number">2</span>, size=<span class="hljs-number">16</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data, <span class="hljs-string">&quot;label&quot;</span>: label}).with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(ds, batch_size=<span class="hljs-number">4</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> dataloader:
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(batch)                                                                                            
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">0.0047</span>, <span class="hljs-number">0.4979</span>, <span class="hljs-number">0.6726</span>, <span class="hljs-number">0.8105</span>]), <span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>])}
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">0.4832</span>, <span class="hljs-number">0.2723</span>, <span class="hljs-number">0.4259</span>, <span class="hljs-number">0.2224</span>]), <span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>])}
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">0.5837</span>, <span class="hljs-number">0.3444</span>, <span class="hljs-number">0.4658</span>, <span class="hljs-number">0.6417</span>]), <span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>])}
{<span class="hljs-string">&#x27;data&#x27;</span>: tensor([<span class="hljs-number">0.7022</span>, <span class="hljs-number">0.1225</span>, <span class="hljs-number">0.7228</span>, <span class="hljs-number">0.8259</span>]), <span class="hljs-string">&#x27;label&#x27;</span>: tensor([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>])}`}}),Rs=new G({}),Ms=new G({}),Ws=new T({props:{code:`import numpy as np
from datasets import Dataset, load_from_disk
from torch.utils.data import DataLoader
data = np.random.rand(10_000)
Dataset.from_dict({"data": data}).save_to_disk("my_dataset")
ds = load_from_disk("my_dataset").with_format("torch")
dataloader = DataLoader(ds, batch_size=32, num_workers=4)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, load_from_disk
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-meta">&gt;&gt;&gt; </span>data = np.random.rand(<span class="hljs-number">10_000</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data}).save_to_disk(<span class="hljs-string">&quot;my_dataset&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_from_disk(<span class="hljs-string">&quot;my_dataset&quot;</span>).with_format(<span class="hljs-string">&quot;torch&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(ds, batch_size=<span class="hljs-number">32</span>, num_workers=<span class="hljs-number">4</span>)`}}),Ys=new G({}),Gs=new T({props:{code:"batch = [dataset[idx] for idx in range(start, end)]",highlighted:'batch = [dataset[idx] <span class="hljs-keyword">for</span> idx <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(start, end)]'}}),Ks=new T({props:{code:`batch = dataset[start:end]
# or
batch = dataset[list_of_indices]`,highlighted:`batch = dataset[start:end]
<span class="hljs-comment"># or</span>
batch = dataset[list_of_indices]`}}),Zs=new T({props:{code:`from torch.utils.data.sampler import BatchSampler, RandomSampler
batch_sampler = BatchSampler(RandomSampler(ds), batch_size=32, drop_last=False)
dataloader = DataLoader(ds, batch_sampler=batch_sampler)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data.sampler <span class="hljs-keyword">import</span> BatchSampler, RandomSampler
<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sampler = BatchSampler(RandomSampler(ds), batch_size=<span class="hljs-number">32</span>, drop_last=<span class="hljs-literal">False</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(ds, batch_sampler=batch_sampler)`}}),Js=new G({}),Qs=new T({props:{code:`import numpy as np
from datasets import Dataset, load_dataset
from torch.utils.data import DataLoader
data = np.random.rand(10_000)
Dataset.from_dict({"data": data}).push_to_hub("<username>/my_dataset")  # Upload to the Hugging Face Hub
my_iterable_dataset = load_dataset("<username>/my_dataset", streaming=True, split="train")
dataloader = DataLoader(my_iterable_dataset, batch_size=32)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader
<span class="hljs-meta">&gt;&gt;&gt; </span>data = np.random.rand(<span class="hljs-number">10_000</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data}).push_to_hub(<span class="hljs-string">&quot;&lt;username&gt;/my_dataset&quot;</span>)  <span class="hljs-comment"># Upload to the Hugging Face Hub</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>my_iterable_dataset = load_dataset(<span class="hljs-string">&quot;&lt;username&gt;/my_dataset&quot;</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(my_iterable_dataset, batch_size=<span class="hljs-number">32</span>)`}}),Vs=new T({props:{code:`my_iterable_dataset = load_dataset("c4", "en", streaming=True, split="train")
my_iterable_dataset.n_shards
dataloader = DataLoader(my_iterable_dataset, batch_size=32, num_workers=4)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>my_iterable_dataset = load_dataset(<span class="hljs-string">&quot;c4&quot;</span>, <span class="hljs-string">&quot;en&quot;</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>my_iterable_dataset.n_shards
<span class="hljs-number">1024</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(my_iterable_dataset, batch_size=<span class="hljs-number">32</span>, num_workers=<span class="hljs-number">4</span>)`}}),Xs=new G({}),sa=new T({props:{code:`import os
from datasets.distributed import split_dataset_by_node

ds = split_dataset_by_node(ds, rank=int(os.environ["RANK"]), world_size=int(os.environ["WORLD_SIZE"]))`,highlighted:`<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">from</span> datasets.distributed <span class="hljs-keyword">import</span> split_dataset_by_node

ds = split_dataset_by_node(ds, rank=<span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;RANK&quot;</span>]), world_size=<span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;WORLD_SIZE&quot;</span>]))`}}),{c(){m=r("meta"),q=h(),u=r("h1"),E=r("a"),D=r("span"),f($.$$.fragment),k=h(),P=r("span"),x=e("Use with PyTorch"),S=h(),w=r("p"),A=e("This document is a quick introduction to using "),v=r("code"),K=e("datasets"),Z=e(` with PyTorch, with a particular focus on how to get
`),N=r("code"),Ne=e("torch.Tensor"),Be=e(" objects out of our datasets, and how to use a PyTorch "),Ea=r("code"),Ue=e("DataLoader"),He=e(" and a Hugging Face "),ka=r("code"),Re=e("Dataset"),Me=e(`
with the best performance.`),$t=h(),J=r("h2"),rs=r("a"),Da=r("span"),f(As.$$.fragment),We=h(),qa=r("span"),Ye=e("Dataset format"),wt=h(),ta=r("p"),Ge=e("By default, datasets return regular python objects: integers, floats, strings, lists, etc."),Et=h(),B=r("p"),Ke=e("To get PyTorch tensors instead, you can set the format of the dataset to "),Pa=r("code"),Ze=e("pytorch"),Je=e(" using "),ea=r("a"),Qe=e("Dataset.with_format()"),Ve=e(":"),kt=h(),f(Ts.$$.fragment),Dt=h(),f(os.$$.fragment),qt=h(),ps=r("p"),Xe=e("To load the data as tensors on a GPU, specify the "),xa=r("code"),sl=e("device"),al=e(" argument:"),Pt=h(),f(Ss.$$.fragment),xt=h(),Q=r("h2"),is=r("a"),Aa=r("span"),f(Cs.$$.fragment),tl=h(),Ta=r("span"),el=e("N-dimensional arrays"),At=h(),la=r("p"),ll=e(`If your dataset consists of N-dimensional arrays, you will see that by default they are considered as nested lists.
In particular, a PyTorch formatted dataset outputs nested lists instead of a single tensor:`),Tt=h(),f(Ls.$$.fragment),St=h(),hs=r("p"),nl=e("To get a single tensor, you must explicitly use the "),Sa=r("code"),rl=e("Array"),ol=e(" feature type and specify the shape of your tensors:"),Ct=h(),f(Os.$$.fragment),Lt=h(),V=r("h2"),ds=r("a"),Ca=r("span"),f(zs.$$.fragment),pl=h(),La=r("span"),il=e("Other feature types"),Ot=h(),Is=r("p"),na=r("a"),hl=e("ClassLabel"),dl=e(" data are properly converted to tensors:"),zt=h(),f(Fs.$$.fragment),It=h(),ra=r("p"),cl=e("String and binary objects are unchanged, since PyTorch only supports numbers."),Ft=h(),U=r("p"),ml=e("The "),oa=r("a"),ul=e("Image"),fl=e(" and "),pa=r("a"),gl=e("Audio"),bl=e(" feature types are also supported."),Nt=h(),f(cs.$$.fragment),Bt=h(),f(Ns.$$.fragment),Ut=h(),f(ms.$$.fragment),Ht=h(),f(Bs.$$.fragment),Rt=h(),X=r("h2"),us=r("a"),Oa=r("span"),f(Us.$$.fragment),jl=h(),za=r("span"),_l=e("Data loading"),Mt=h(),C=r("p"),yl=e("Like "),Ia=r("code"),vl=e("torch.utils.data.Dataset"),$l=e(" objects, a "),ia=r("a"),wl=e("Dataset"),El=e(" can be passed directly to a PyTorch "),Fa=r("code"),kl=e("DataLoader"),Dl=e(":"),Wt=h(),f(Hs.$$.fragment),Yt=h(),ss=r("h3"),fs=r("a"),Na=r("span"),f(Rs.$$.fragment),ql=h(),Ba=r("span"),Pl=e("Optimize data loading"),Gt=h(),ha=r("p"),xl=e(`There are several ways you can increase the speed your data is loaded which can save you time, especially if you are working with large datasets.
PyTorch offers parallelized data loading, retrieving batches of indices instead of individually, and streaming to iterate over the dataset without downloading it on disk.`),Kt=h(),as=r("h4"),gs=r("a"),Ua=r("span"),f(Ms.$$.fragment),Al=h(),Ha=r("span"),Tl=e("Use multiple Workers"),Zt=h(),H=r("p"),Sl=e("You can parallelize data loading with the "),Ra=r("code"),Cl=e("num_workers"),Ll=e(" argument of a PyTorch "),Ma=r("code"),Ol=e("DataLoader"),zl=e(" and get a higher throughput."),Jt=h(),L=r("p"),Il=e("Under the hood, the "),Wa=r("code"),Fl=e("DataLoader"),Nl=e(" starts "),Ya=r("code"),Bl=e("num_workers"),Ul=e(` processes.
Each process reloads the dataset passed to the `),Ga=r("code"),Hl=e("DataLoader"),Rl=e(` and is used to query examples.
Reloading the dataset inside a worker doesn\u2019t fill up your RAM, since it simply memory-maps the dataset again from your disk.`),Qt=h(),f(Ws.$$.fragment),Vt=h(),ts=r("h4"),bs=r("a"),Ka=r("span"),f(Ys.$$.fragment),Ml=h(),Za=r("span"),Wl=e("Use a BatchSampler (torch&lt;=1.12.1)"),Xt=h(),O=r("p"),Yl=e("For old versions of PyTorch, using a "),Ja=r("code"),Gl=e("BatchSampler"),Kl=e(` can speed up data loading.
Indeed if you are using `),Qa=r("code"),Zl=e("torch<=1.12.1"),Jl=e(", the PyTorch "),Va=r("code"),Ql=e("DataLoader"),Vl=e(" load batches of data from a dataset one by one like this:"),se=h(),f(Gs.$$.fragment),ae=h(),da=r("p"),Xl=e(`Unfortunately, this does numerous read operations on the dataset.
It is more efficient to query batches of examples using a list:`),te=h(),f(Ks.$$.fragment),ee=h(),R=r("p"),sn=e("For the PyTorch "),Xa=r("code"),an=e("DataLoader"),tn=e(" to query batches using a list, you can use a "),st=r("code"),en=e("BatchSampler"),ln=e(":"),le=h(),f(Zs.$$.fragment),ne=h(),z=r("p"),nn=e("Moreover, this is particularly useful if you used "),at=r("code"),rn=e("set_transform"),on=e(` to apply a transform on-the-fly when examples are accessed.
You must use a `),tt=r("code"),pn=e("BatchSampler"),hn=e(" if you want the transform to be given full batches instead of receiving "),et=r("code"),dn=e("batch_size"),cn=e(" times one single element."),re=h(),M=r("p"),mn=e("Recent versions of PyTorch use a list of indices, so a "),lt=r("code"),un=e("BatchSampler"),fn=e(" is not needed to get the best speed even if you used "),nt=r("code"),gn=e("set_transform"),bn=e("."),oe=h(),es=r("h3"),js=r("a"),rt=r("span"),f(Js.$$.fragment),jn=h(),ot=r("span"),_n=e("Stream data"),pe=h(),_s=r("p"),yn=e("Stream a dataset by loading it as an "),ca=r("a"),vn=e("IterableDataset"),$n=e(`. This allows you to progressively iterate over a remote dataset without downloading it on disk and or over local data files.
Learn more about which type of dataset is best for your use case in the (choosing between a regular dataset or an iterable dataset)[./dataset_vs_iterable_dataset] guide.`),ie=h(),I=r("p"),wn=e("An iterable dataset from "),pt=r("code"),En=e("datasets"),kn=e(" inherits from "),it=r("code"),Dn=e("torch.utils.data.IterableDataset"),qn=e(" so you can pass it to a "),ht=r("code"),Pn=e("torch.utils.data.DataLoader"),xn=e(":"),he=h(),f(Qs.$$.fragment),de=h(),ys=r("p"),An=e("If the dataset is split in several shards (i.e. if the dataset consists of multiple data files), then you can stream in parallel using "),dt=r("code"),Tn=e("num_workers"),Sn=e(":"),ce=h(),f(Vs.$$.fragment),me=h(),ma=r("p"),Cn=e("In this case each worker is given a subset of the list of shards to stream from."),ue=h(),ls=r("h3"),vs=r("a"),ct=r("span"),f(Xs.$$.fragment),Ln=h(),mt=r("span"),On=e("Distributed"),fe=h(),$s=r("p"),zn=e("To split your dataset across your training nodes, you can use "),ua=r("a"),In=e("datasets.distributed.split_dataset_by_node()"),Fn=e(":"),ge=h(),f(sa.$$.fragment),be=h(),W=r("p"),Nn=e(`This works for both map-style datasets and iterable datasets.
The dataset is split for the node at rank `),ut=r("code"),Bn=e("rank"),Un=e(" in a pool of nodes of size "),ft=r("code"),Hn=e("world_size"),Rn=e("."),je=h(),fa=r("p"),Mn=e("For map-style datasets:"),_e=h(),ga=r("p"),Wn=e("Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset."),ye=h(),ba=r("p"),Yn=e("For iterable datasets:"),ve=h(),F=r("p"),Gn=e("If the dataset has a number of shards that is a factor of "),gt=r("code"),Kn=e("world_size"),Zn=e(" (i.e. if "),bt=r("code"),Jn=e("dataset.n_shards % world_size == 0"),Qn=e(`),
then the shards are evenly assigned across the nodes, which is the most optimized.
Otherwise, each node keeps 1 example out of `),jt=r("code"),Vn=e("world_size"),Xn=e(", skipping the other examples."),$e=h(),ws=r("p"),sr=e("This can also be combined with a "),_t=r("code"),ar=e("torch.utils.data.DataLoader"),tr=e(" if you want each node to use multiple workers to load the data."),this.h()},l(s){const n=Co('[data-svelte="svelte-1phssyn"]',document.head);m=o(n,"META",{name:!0,content:!0}),n.forEach(t),q=d(s),u=o(s,"H1",{class:!0});var aa=p(u);E=o(aa,"A",{id:!0,class:!0,href:!0});var yt=p(E);D=o(yt,"SPAN",{});var vt=p(D);g($.$$.fragment,vt),vt.forEach(t),yt.forEach(t),k=d(aa),P=o(aa,"SPAN",{});var nr=p(P);x=l(nr,"Use with PyTorch"),nr.forEach(t),aa.forEach(t),S=d(s),w=o(s,"P",{});var Y=p(w);A=l(Y,"This document is a quick introduction to using "),v=o(Y,"CODE",{});var rr=p(v);K=l(rr,"datasets"),rr.forEach(t),Z=l(Y,` with PyTorch, with a particular focus on how to get
`),N=o(Y,"CODE",{});var or=p(N);Ne=l(or,"torch.Tensor"),or.forEach(t),Be=l(Y," objects out of our datasets, and how to use a PyTorch "),Ea=o(Y,"CODE",{});var pr=p(Ea);Ue=l(pr,"DataLoader"),pr.forEach(t),He=l(Y," and a Hugging Face "),ka=o(Y,"CODE",{});var ir=p(ka);Re=l(ir,"Dataset"),ir.forEach(t),Me=l(Y,`
with the best performance.`),Y.forEach(t),$t=d(s),J=o(s,"H2",{class:!0});var Ee=p(J);rs=o(Ee,"A",{id:!0,class:!0,href:!0});var hr=p(rs);Da=o(hr,"SPAN",{});var dr=p(Da);g(As.$$.fragment,dr),dr.forEach(t),hr.forEach(t),We=d(Ee),qa=o(Ee,"SPAN",{});var cr=p(qa);Ye=l(cr,"Dataset format"),cr.forEach(t),Ee.forEach(t),wt=d(s),ta=o(s,"P",{});var mr=p(ta);Ge=l(mr,"By default, datasets return regular python objects: integers, floats, strings, lists, etc."),mr.forEach(t),Et=d(s),B=o(s,"P",{});var ja=p(B);Ke=l(ja,"To get PyTorch tensors instead, you can set the format of the dataset to "),Pa=o(ja,"CODE",{});var ur=p(Pa);Ze=l(ur,"pytorch"),ur.forEach(t),Je=l(ja," using "),ea=o(ja,"A",{href:!0});var fr=p(ea);Qe=l(fr,"Dataset.with_format()"),fr.forEach(t),Ve=l(ja,":"),ja.forEach(t),kt=d(s),g(Ts.$$.fragment,s),Dt=d(s),g(os.$$.fragment,s),qt=d(s),ps=o(s,"P",{});var ke=p(ps);Xe=l(ke,"To load the data as tensors on a GPU, specify the "),xa=o(ke,"CODE",{});var gr=p(xa);sl=l(gr,"device"),gr.forEach(t),al=l(ke," argument:"),ke.forEach(t),Pt=d(s),g(Ss.$$.fragment,s),xt=d(s),Q=o(s,"H2",{class:!0});var De=p(Q);is=o(De,"A",{id:!0,class:!0,href:!0});var br=p(is);Aa=o(br,"SPAN",{});var jr=p(Aa);g(Cs.$$.fragment,jr),jr.forEach(t),br.forEach(t),tl=d(De),Ta=o(De,"SPAN",{});var _r=p(Ta);el=l(_r,"N-dimensional arrays"),_r.forEach(t),De.forEach(t),At=d(s),la=o(s,"P",{});var yr=p(la);ll=l(yr,`If your dataset consists of N-dimensional arrays, you will see that by default they are considered as nested lists.
In particular, a PyTorch formatted dataset outputs nested lists instead of a single tensor:`),yr.forEach(t),Tt=d(s),g(Ls.$$.fragment,s),St=d(s),hs=o(s,"P",{});var qe=p(hs);nl=l(qe,"To get a single tensor, you must explicitly use the "),Sa=o(qe,"CODE",{});var vr=p(Sa);rl=l(vr,"Array"),vr.forEach(t),ol=l(qe," feature type and specify the shape of your tensors:"),qe.forEach(t),Ct=d(s),g(Os.$$.fragment,s),Lt=d(s),V=o(s,"H2",{class:!0});var Pe=p(V);ds=o(Pe,"A",{id:!0,class:!0,href:!0});var $r=p(ds);Ca=o($r,"SPAN",{});var wr=p(Ca);g(zs.$$.fragment,wr),wr.forEach(t),$r.forEach(t),pl=d(Pe),La=o(Pe,"SPAN",{});var Er=p(La);il=l(Er,"Other feature types"),Er.forEach(t),Pe.forEach(t),Ot=d(s),Is=o(s,"P",{});var er=p(Is);na=o(er,"A",{href:!0});var kr=p(na);hl=l(kr,"ClassLabel"),kr.forEach(t),dl=l(er," data are properly converted to tensors:"),er.forEach(t),zt=d(s),g(Fs.$$.fragment,s),It=d(s),ra=o(s,"P",{});var Dr=p(ra);cl=l(Dr,"String and binary objects are unchanged, since PyTorch only supports numbers."),Dr.forEach(t),Ft=d(s),U=o(s,"P",{});var _a=p(U);ml=l(_a,"The "),oa=o(_a,"A",{href:!0});var qr=p(oa);ul=l(qr,"Image"),qr.forEach(t),fl=l(_a," and "),pa=o(_a,"A",{href:!0});var Pr=p(pa);gl=l(Pr,"Audio"),Pr.forEach(t),bl=l(_a," feature types are also supported."),_a.forEach(t),Nt=d(s),g(cs.$$.fragment,s),Bt=d(s),g(Ns.$$.fragment,s),Ut=d(s),g(ms.$$.fragment,s),Ht=d(s),g(Bs.$$.fragment,s),Rt=d(s),X=o(s,"H2",{class:!0});var xe=p(X);us=o(xe,"A",{id:!0,class:!0,href:!0});var xr=p(us);Oa=o(xr,"SPAN",{});var Ar=p(Oa);g(Us.$$.fragment,Ar),Ar.forEach(t),xr.forEach(t),jl=d(xe),za=o(xe,"SPAN",{});var Tr=p(za);_l=l(Tr,"Data loading"),Tr.forEach(t),xe.forEach(t),Mt=d(s),C=o(s,"P",{});var Es=p(C);yl=l(Es,"Like "),Ia=o(Es,"CODE",{});var Sr=p(Ia);vl=l(Sr,"torch.utils.data.Dataset"),Sr.forEach(t),$l=l(Es," objects, a "),ia=o(Es,"A",{href:!0});var Cr=p(ia);wl=l(Cr,"Dataset"),Cr.forEach(t),El=l(Es," can be passed directly to a PyTorch "),Fa=o(Es,"CODE",{});var Lr=p(Fa);kl=l(Lr,"DataLoader"),Lr.forEach(t),Dl=l(Es,":"),Es.forEach(t),Wt=d(s),g(Hs.$$.fragment,s),Yt=d(s),ss=o(s,"H3",{class:!0});var Ae=p(ss);fs=o(Ae,"A",{id:!0,class:!0,href:!0});var Or=p(fs);Na=o(Or,"SPAN",{});var zr=p(Na);g(Rs.$$.fragment,zr),zr.forEach(t),Or.forEach(t),ql=d(Ae),Ba=o(Ae,"SPAN",{});var Ir=p(Ba);Pl=l(Ir,"Optimize data loading"),Ir.forEach(t),Ae.forEach(t),Gt=d(s),ha=o(s,"P",{});var Fr=p(ha);xl=l(Fr,`There are several ways you can increase the speed your data is loaded which can save you time, especially if you are working with large datasets.
PyTorch offers parallelized data loading, retrieving batches of indices instead of individually, and streaming to iterate over the dataset without downloading it on disk.`),Fr.forEach(t),Kt=d(s),as=o(s,"H4",{class:!0});var Te=p(as);gs=o(Te,"A",{id:!0,class:!0,href:!0});var Nr=p(gs);Ua=o(Nr,"SPAN",{});var Br=p(Ua);g(Ms.$$.fragment,Br),Br.forEach(t),Nr.forEach(t),Al=d(Te),Ha=o(Te,"SPAN",{});var Ur=p(Ha);Tl=l(Ur,"Use multiple Workers"),Ur.forEach(t),Te.forEach(t),Zt=d(s),H=o(s,"P",{});var ya=p(H);Sl=l(ya,"You can parallelize data loading with the "),Ra=o(ya,"CODE",{});var Hr=p(Ra);Cl=l(Hr,"num_workers"),Hr.forEach(t),Ll=l(ya," argument of a PyTorch "),Ma=o(ya,"CODE",{});var Rr=p(Ma);Ol=l(Rr,"DataLoader"),Rr.forEach(t),zl=l(ya," and get a higher throughput."),ya.forEach(t),Jt=d(s),L=o(s,"P",{});var ks=p(L);Il=l(ks,"Under the hood, the "),Wa=o(ks,"CODE",{});var Mr=p(Wa);Fl=l(Mr,"DataLoader"),Mr.forEach(t),Nl=l(ks," starts "),Ya=o(ks,"CODE",{});var Wr=p(Ya);Bl=l(Wr,"num_workers"),Wr.forEach(t),Ul=l(ks,` processes.
Each process reloads the dataset passed to the `),Ga=o(ks,"CODE",{});var Yr=p(Ga);Hl=l(Yr,"DataLoader"),Yr.forEach(t),Rl=l(ks,` and is used to query examples.
Reloading the dataset inside a worker doesn\u2019t fill up your RAM, since it simply memory-maps the dataset again from your disk.`),ks.forEach(t),Qt=d(s),g(Ws.$$.fragment,s),Vt=d(s),ts=o(s,"H4",{class:!0});var Se=p(ts);bs=o(Se,"A",{id:!0,class:!0,href:!0});var Gr=p(bs);Ka=o(Gr,"SPAN",{});var Kr=p(Ka);g(Ys.$$.fragment,Kr),Kr.forEach(t),Gr.forEach(t),Ml=d(Se),Za=o(Se,"SPAN",{});var Zr=p(Za);Wl=l(Zr,"Use a BatchSampler (torch&lt;=1.12.1)"),Zr.forEach(t),Se.forEach(t),Xt=d(s),O=o(s,"P",{});var Ds=p(O);Yl=l(Ds,"For old versions of PyTorch, using a "),Ja=o(Ds,"CODE",{});var Jr=p(Ja);Gl=l(Jr,"BatchSampler"),Jr.forEach(t),Kl=l(Ds,` can speed up data loading.
Indeed if you are using `),Qa=o(Ds,"CODE",{});var Qr=p(Qa);Zl=l(Qr,"torch<=1.12.1"),Qr.forEach(t),Jl=l(Ds,", the PyTorch "),Va=o(Ds,"CODE",{});var Vr=p(Va);Ql=l(Vr,"DataLoader"),Vr.forEach(t),Vl=l(Ds," load batches of data from a dataset one by one like this:"),Ds.forEach(t),se=d(s),g(Gs.$$.fragment,s),ae=d(s),da=o(s,"P",{});var Xr=p(da);Xl=l(Xr,`Unfortunately, this does numerous read operations on the dataset.
It is more efficient to query batches of examples using a list:`),Xr.forEach(t),te=d(s),g(Ks.$$.fragment,s),ee=d(s),R=o(s,"P",{});var va=p(R);sn=l(va,"For the PyTorch "),Xa=o(va,"CODE",{});var so=p(Xa);an=l(so,"DataLoader"),so.forEach(t),tn=l(va," to query batches using a list, you can use a "),st=o(va,"CODE",{});var ao=p(st);en=l(ao,"BatchSampler"),ao.forEach(t),ln=l(va,":"),va.forEach(t),le=d(s),g(Zs.$$.fragment,s),ne=d(s),z=o(s,"P",{});var qs=p(z);nn=l(qs,"Moreover, this is particularly useful if you used "),at=o(qs,"CODE",{});var to=p(at);rn=l(to,"set_transform"),to.forEach(t),on=l(qs,` to apply a transform on-the-fly when examples are accessed.
You must use a `),tt=o(qs,"CODE",{});var eo=p(tt);pn=l(eo,"BatchSampler"),eo.forEach(t),hn=l(qs," if you want the transform to be given full batches instead of receiving "),et=o(qs,"CODE",{});var lo=p(et);dn=l(lo,"batch_size"),lo.forEach(t),cn=l(qs," times one single element."),qs.forEach(t),re=d(s),M=o(s,"P",{});var $a=p(M);mn=l($a,"Recent versions of PyTorch use a list of indices, so a "),lt=o($a,"CODE",{});var no=p(lt);un=l(no,"BatchSampler"),no.forEach(t),fn=l($a," is not needed to get the best speed even if you used "),nt=o($a,"CODE",{});var ro=p(nt);gn=l(ro,"set_transform"),ro.forEach(t),bn=l($a,"."),$a.forEach(t),oe=d(s),es=o(s,"H3",{class:!0});var Ce=p(es);js=o(Ce,"A",{id:!0,class:!0,href:!0});var oo=p(js);rt=o(oo,"SPAN",{});var po=p(rt);g(Js.$$.fragment,po),po.forEach(t),oo.forEach(t),jn=d(Ce),ot=o(Ce,"SPAN",{});var io=p(ot);_n=l(io,"Stream data"),io.forEach(t),Ce.forEach(t),pe=d(s),_s=o(s,"P",{});var Le=p(_s);yn=l(Le,"Stream a dataset by loading it as an "),ca=o(Le,"A",{href:!0});var ho=p(ca);vn=l(ho,"IterableDataset"),ho.forEach(t),$n=l(Le,`. This allows you to progressively iterate over a remote dataset without downloading it on disk and or over local data files.
Learn more about which type of dataset is best for your use case in the (choosing between a regular dataset or an iterable dataset)[./dataset_vs_iterable_dataset] guide.`),Le.forEach(t),ie=d(s),I=o(s,"P",{});var Ps=p(I);wn=l(Ps,"An iterable dataset from "),pt=o(Ps,"CODE",{});var co=p(pt);En=l(co,"datasets"),co.forEach(t),kn=l(Ps," inherits from "),it=o(Ps,"CODE",{});var mo=p(it);Dn=l(mo,"torch.utils.data.IterableDataset"),mo.forEach(t),qn=l(Ps," so you can pass it to a "),ht=o(Ps,"CODE",{});var uo=p(ht);Pn=l(uo,"torch.utils.data.DataLoader"),uo.forEach(t),xn=l(Ps,":"),Ps.forEach(t),he=d(s),g(Qs.$$.fragment,s),de=d(s),ys=o(s,"P",{});var Oe=p(ys);An=l(Oe,"If the dataset is split in several shards (i.e. if the dataset consists of multiple data files), then you can stream in parallel using "),dt=o(Oe,"CODE",{});var fo=p(dt);Tn=l(fo,"num_workers"),fo.forEach(t),Sn=l(Oe,":"),Oe.forEach(t),ce=d(s),g(Vs.$$.fragment,s),me=d(s),ma=o(s,"P",{});var go=p(ma);Cn=l(go,"In this case each worker is given a subset of the list of shards to stream from."),go.forEach(t),ue=d(s),ls=o(s,"H3",{class:!0});var ze=p(ls);vs=o(ze,"A",{id:!0,class:!0,href:!0});var bo=p(vs);ct=o(bo,"SPAN",{});var jo=p(ct);g(Xs.$$.fragment,jo),jo.forEach(t),bo.forEach(t),Ln=d(ze),mt=o(ze,"SPAN",{});var _o=p(mt);On=l(_o,"Distributed"),_o.forEach(t),ze.forEach(t),fe=d(s),$s=o(s,"P",{});var Ie=p($s);zn=l(Ie,"To split your dataset across your training nodes, you can use "),ua=o(Ie,"A",{href:!0});var yo=p(ua);In=l(yo,"datasets.distributed.split_dataset_by_node()"),yo.forEach(t),Fn=l(Ie,":"),Ie.forEach(t),ge=d(s),g(sa.$$.fragment,s),be=d(s),W=o(s,"P",{});var wa=p(W);Nn=l(wa,`This works for both map-style datasets and iterable datasets.
The dataset is split for the node at rank `),ut=o(wa,"CODE",{});var vo=p(ut);Bn=l(vo,"rank"),vo.forEach(t),Un=l(wa," in a pool of nodes of size "),ft=o(wa,"CODE",{});var $o=p(ft);Hn=l($o,"world_size"),$o.forEach(t),Rn=l(wa,"."),wa.forEach(t),je=d(s),fa=o(s,"P",{});var wo=p(fa);Mn=l(wo,"For map-style datasets:"),wo.forEach(t),_e=d(s),ga=o(s,"P",{});var Eo=p(ga);Wn=l(Eo,"Each node is assigned a chunk of data, e.g. rank 0 is given the first chunk of the dataset."),Eo.forEach(t),ye=d(s),ba=o(s,"P",{});var ko=p(ba);Yn=l(ko,"For iterable datasets:"),ko.forEach(t),ve=d(s),F=o(s,"P",{});var xs=p(F);Gn=l(xs,"If the dataset has a number of shards that is a factor of "),gt=o(xs,"CODE",{});var Do=p(gt);Kn=l(Do,"world_size"),Do.forEach(t),Zn=l(xs," (i.e. if "),bt=o(xs,"CODE",{});var qo=p(bt);Jn=l(qo,"dataset.n_shards % world_size == 0"),qo.forEach(t),Qn=l(xs,`),
then the shards are evenly assigned across the nodes, which is the most optimized.
Otherwise, each node keeps 1 example out of `),jt=o(xs,"CODE",{});var Po=p(jt);Vn=l(Po,"world_size"),Po.forEach(t),Xn=l(xs,", skipping the other examples."),xs.forEach(t),$e=d(s),ws=o(s,"P",{});var Fe=p(ws);sr=l(Fe,"This can also be combined with a "),_t=o(Fe,"CODE",{});var xo=p(_t);ar=l(xo,"torch.utils.data.DataLoader"),xo.forEach(t),tr=l(Fe," if you want each node to use multiple workers to load the data."),Fe.forEach(t),this.h()},h(){c(m,"name","hf:doc:metadata"),c(m,"content",JSON.stringify(No)),c(E,"id","use-with-pytorch"),c(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(E,"href","#use-with-pytorch"),c(u,"class","relative group"),c(rs,"id","dataset-format"),c(rs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(rs,"href","#dataset-format"),c(J,"class","relative group"),c(ea,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.with_format"),c(is,"id","ndimensional-arrays"),c(is,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(is,"href","#ndimensional-arrays"),c(Q,"class","relative group"),c(ds,"id","other-feature-types"),c(ds,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ds,"href","#other-feature-types"),c(V,"class","relative group"),c(na,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.ClassLabel"),c(oa,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Image"),c(pa,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Audio"),c(us,"id","data-loading"),c(us,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(us,"href","#data-loading"),c(X,"class","relative group"),c(ia,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset"),c(fs,"id","optimize-data-loading"),c(fs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(fs,"href","#optimize-data-loading"),c(ss,"class","relative group"),c(gs,"id","use-multiple-workers"),c(gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(gs,"href","#use-multiple-workers"),c(as,"class","relative group"),c(bs,"id","use-a-batchsampler-torchamplt1121"),c(bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(bs,"href","#use-a-batchsampler-torchamplt1121"),c(ts,"class","relative group"),c(js,"id","stream-data"),c(js,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(js,"href","#stream-data"),c(es,"class","relative group"),c(ca,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset"),c(vs,"id","distributed"),c(vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(vs,"href","#distributed"),c(ls,"class","relative group"),c(ua,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.distributed.split_dataset_by_node")},m(s,n){a(document.head,m),i(s,q,n),i(s,u,n),a(u,E),a(E,D),b($,D,null),a(u,k),a(u,P),a(P,x),i(s,S,n),i(s,w,n),a(w,A),a(w,v),a(v,K),a(w,Z),a(w,N),a(N,Ne),a(w,Be),a(w,Ea),a(Ea,Ue),a(w,He),a(w,ka),a(ka,Re),a(w,Me),i(s,$t,n),i(s,J,n),a(J,rs),a(rs,Da),b(As,Da,null),a(J,We),a(J,qa),a(qa,Ye),i(s,wt,n),i(s,ta,n),a(ta,Ge),i(s,Et,n),i(s,B,n),a(B,Ke),a(B,Pa),a(Pa,Ze),a(B,Je),a(B,ea),a(ea,Qe),a(B,Ve),i(s,kt,n),b(Ts,s,n),i(s,Dt,n),b(os,s,n),i(s,qt,n),i(s,ps,n),a(ps,Xe),a(ps,xa),a(xa,sl),a(ps,al),i(s,Pt,n),b(Ss,s,n),i(s,xt,n),i(s,Q,n),a(Q,is),a(is,Aa),b(Cs,Aa,null),a(Q,tl),a(Q,Ta),a(Ta,el),i(s,At,n),i(s,la,n),a(la,ll),i(s,Tt,n),b(Ls,s,n),i(s,St,n),i(s,hs,n),a(hs,nl),a(hs,Sa),a(Sa,rl),a(hs,ol),i(s,Ct,n),b(Os,s,n),i(s,Lt,n),i(s,V,n),a(V,ds),a(ds,Ca),b(zs,Ca,null),a(V,pl),a(V,La),a(La,il),i(s,Ot,n),i(s,Is,n),a(Is,na),a(na,hl),a(Is,dl),i(s,zt,n),b(Fs,s,n),i(s,It,n),i(s,ra,n),a(ra,cl),i(s,Ft,n),i(s,U,n),a(U,ml),a(U,oa),a(oa,ul),a(U,fl),a(U,pa),a(pa,gl),a(U,bl),i(s,Nt,n),b(cs,s,n),i(s,Bt,n),b(Ns,s,n),i(s,Ut,n),b(ms,s,n),i(s,Ht,n),b(Bs,s,n),i(s,Rt,n),i(s,X,n),a(X,us),a(us,Oa),b(Us,Oa,null),a(X,jl),a(X,za),a(za,_l),i(s,Mt,n),i(s,C,n),a(C,yl),a(C,Ia),a(Ia,vl),a(C,$l),a(C,ia),a(ia,wl),a(C,El),a(C,Fa),a(Fa,kl),a(C,Dl),i(s,Wt,n),b(Hs,s,n),i(s,Yt,n),i(s,ss,n),a(ss,fs),a(fs,Na),b(Rs,Na,null),a(ss,ql),a(ss,Ba),a(Ba,Pl),i(s,Gt,n),i(s,ha,n),a(ha,xl),i(s,Kt,n),i(s,as,n),a(as,gs),a(gs,Ua),b(Ms,Ua,null),a(as,Al),a(as,Ha),a(Ha,Tl),i(s,Zt,n),i(s,H,n),a(H,Sl),a(H,Ra),a(Ra,Cl),a(H,Ll),a(H,Ma),a(Ma,Ol),a(H,zl),i(s,Jt,n),i(s,L,n),a(L,Il),a(L,Wa),a(Wa,Fl),a(L,Nl),a(L,Ya),a(Ya,Bl),a(L,Ul),a(L,Ga),a(Ga,Hl),a(L,Rl),i(s,Qt,n),b(Ws,s,n),i(s,Vt,n),i(s,ts,n),a(ts,bs),a(bs,Ka),b(Ys,Ka,null),a(ts,Ml),a(ts,Za),a(Za,Wl),i(s,Xt,n),i(s,O,n),a(O,Yl),a(O,Ja),a(Ja,Gl),a(O,Kl),a(O,Qa),a(Qa,Zl),a(O,Jl),a(O,Va),a(Va,Ql),a(O,Vl),i(s,se,n),b(Gs,s,n),i(s,ae,n),i(s,da,n),a(da,Xl),i(s,te,n),b(Ks,s,n),i(s,ee,n),i(s,R,n),a(R,sn),a(R,Xa),a(Xa,an),a(R,tn),a(R,st),a(st,en),a(R,ln),i(s,le,n),b(Zs,s,n),i(s,ne,n),i(s,z,n),a(z,nn),a(z,at),a(at,rn),a(z,on),a(z,tt),a(tt,pn),a(z,hn),a(z,et),a(et,dn),a(z,cn),i(s,re,n),i(s,M,n),a(M,mn),a(M,lt),a(lt,un),a(M,fn),a(M,nt),a(nt,gn),a(M,bn),i(s,oe,n),i(s,es,n),a(es,js),a(js,rt),b(Js,rt,null),a(es,jn),a(es,ot),a(ot,_n),i(s,pe,n),i(s,_s,n),a(_s,yn),a(_s,ca),a(ca,vn),a(_s,$n),i(s,ie,n),i(s,I,n),a(I,wn),a(I,pt),a(pt,En),a(I,kn),a(I,it),a(it,Dn),a(I,qn),a(I,ht),a(ht,Pn),a(I,xn),i(s,he,n),b(Qs,s,n),i(s,de,n),i(s,ys,n),a(ys,An),a(ys,dt),a(dt,Tn),a(ys,Sn),i(s,ce,n),b(Vs,s,n),i(s,me,n),i(s,ma,n),a(ma,Cn),i(s,ue,n),i(s,ls,n),a(ls,vs),a(vs,ct),b(Xs,ct,null),a(ls,Ln),a(ls,mt),a(mt,On),i(s,fe,n),i(s,$s,n),a($s,zn),a($s,ua),a(ua,In),a($s,Fn),i(s,ge,n),b(sa,s,n),i(s,be,n),i(s,W,n),a(W,Nn),a(W,ut),a(ut,Bn),a(W,Un),a(W,ft),a(ft,Hn),a(W,Rn),i(s,je,n),i(s,fa,n),a(fa,Mn),i(s,_e,n),i(s,ga,n),a(ga,Wn),i(s,ye,n),i(s,ba,n),a(ba,Yn),i(s,ve,n),i(s,F,n),a(F,Gn),a(F,gt),a(gt,Kn),a(F,Zn),a(F,bt),a(bt,Jn),a(F,Qn),a(F,jt),a(jt,Vn),a(F,Xn),i(s,$e,n),i(s,ws,n),a(ws,sr),a(ws,_t),a(_t,ar),a(ws,tr),we=!0},p(s,[n]){const aa={};n&2&&(aa.$$scope={dirty:n,ctx:s}),os.$set(aa);const yt={};n&2&&(yt.$$scope={dirty:n,ctx:s}),cs.$set(yt);const vt={};n&2&&(vt.$$scope={dirty:n,ctx:s}),ms.$set(vt)},i(s){we||(j($.$$.fragment,s),j(As.$$.fragment,s),j(Ts.$$.fragment,s),j(os.$$.fragment,s),j(Ss.$$.fragment,s),j(Cs.$$.fragment,s),j(Ls.$$.fragment,s),j(Os.$$.fragment,s),j(zs.$$.fragment,s),j(Fs.$$.fragment,s),j(cs.$$.fragment,s),j(Ns.$$.fragment,s),j(ms.$$.fragment,s),j(Bs.$$.fragment,s),j(Us.$$.fragment,s),j(Hs.$$.fragment,s),j(Rs.$$.fragment,s),j(Ms.$$.fragment,s),j(Ws.$$.fragment,s),j(Ys.$$.fragment,s),j(Gs.$$.fragment,s),j(Ks.$$.fragment,s),j(Zs.$$.fragment,s),j(Js.$$.fragment,s),j(Qs.$$.fragment,s),j(Vs.$$.fragment,s),j(Xs.$$.fragment,s),j(sa.$$.fragment,s),we=!0)},o(s){_($.$$.fragment,s),_(As.$$.fragment,s),_(Ts.$$.fragment,s),_(os.$$.fragment,s),_(Ss.$$.fragment,s),_(Cs.$$.fragment,s),_(Ls.$$.fragment,s),_(Os.$$.fragment,s),_(zs.$$.fragment,s),_(Fs.$$.fragment,s),_(cs.$$.fragment,s),_(Ns.$$.fragment,s),_(ms.$$.fragment,s),_(Bs.$$.fragment,s),_(Us.$$.fragment,s),_(Hs.$$.fragment,s),_(Rs.$$.fragment,s),_(Ms.$$.fragment,s),_(Ws.$$.fragment,s),_(Ys.$$.fragment,s),_(Gs.$$.fragment,s),_(Ks.$$.fragment,s),_(Zs.$$.fragment,s),_(Js.$$.fragment,s),_(Qs.$$.fragment,s),_(Vs.$$.fragment,s),_(Xs.$$.fragment,s),_(sa.$$.fragment,s),we=!1},d(s){t(m),s&&t(q),s&&t(u),y($),s&&t(S),s&&t(w),s&&t($t),s&&t(J),y(As),s&&t(wt),s&&t(ta),s&&t(Et),s&&t(B),s&&t(kt),y(Ts,s),s&&t(Dt),y(os,s),s&&t(qt),s&&t(ps),s&&t(Pt),y(Ss,s),s&&t(xt),s&&t(Q),y(Cs),s&&t(At),s&&t(la),s&&t(Tt),y(Ls,s),s&&t(St),s&&t(hs),s&&t(Ct),y(Os,s),s&&t(Lt),s&&t(V),y(zs),s&&t(Ot),s&&t(Is),s&&t(zt),y(Fs,s),s&&t(It),s&&t(ra),s&&t(Ft),s&&t(U),s&&t(Nt),y(cs,s),s&&t(Bt),y(Ns,s),s&&t(Ut),y(ms,s),s&&t(Ht),y(Bs,s),s&&t(Rt),s&&t(X),y(Us),s&&t(Mt),s&&t(C),s&&t(Wt),y(Hs,s),s&&t(Yt),s&&t(ss),y(Rs),s&&t(Gt),s&&t(ha),s&&t(Kt),s&&t(as),y(Ms),s&&t(Zt),s&&t(H),s&&t(Jt),s&&t(L),s&&t(Qt),y(Ws,s),s&&t(Vt),s&&t(ts),y(Ys),s&&t(Xt),s&&t(O),s&&t(se),y(Gs,s),s&&t(ae),s&&t(da),s&&t(te),y(Ks,s),s&&t(ee),s&&t(R),s&&t(le),y(Zs,s),s&&t(ne),s&&t(z),s&&t(re),s&&t(M),s&&t(oe),s&&t(es),y(Js),s&&t(pe),s&&t(_s),s&&t(ie),s&&t(I),s&&t(he),y(Qs,s),s&&t(de),s&&t(ys),s&&t(ce),y(Vs,s),s&&t(me),s&&t(ma),s&&t(ue),s&&t(ls),y(Xs),s&&t(fe),s&&t($s),s&&t(ge),y(sa,s),s&&t(be),s&&t(W),s&&t(je),s&&t(fa),s&&t(_e),s&&t(ga),s&&t(ye),s&&t(ba),s&&t(ve),s&&t(F),s&&t($e),s&&t(ws)}}}const No={local:"use-with-pytorch",sections:[{local:"dataset-format",title:"Dataset format"},{local:"ndimensional-arrays",title:"N-dimensional arrays"},{local:"other-feature-types",title:"Other feature types"},{local:"data-loading",sections:[{local:"optimize-data-loading",sections:[{local:"use-multiple-workers",title:"Use multiple Workers"},{local:"use-a-batchsampler-torchamplt1121",title:"Use a BatchSampler (torch&amp;lt;=1.12.1)"}],title:"Optimize data loading"},{local:"stream-data",title:"Stream data"},{local:"distributed",title:"Distributed"}],title:"Data loading"}],title:"Use with PyTorch"};function Bo(ns){return Lo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wo extends Ao{constructor(m){super();To(this,m,Bo,Fo,So,{})}}export{Wo as default,No as metadata};
