import{S as Zo,i as er,s as ar,e as r,k as c,w as m,t as s,M as tr,c as l,d as t,m as h,a as n,x as g,h as o,b as d,N as sr,G as a,g as p,y,L as or,q as _,o as v,B as w,v as rr}from"../chunks/vendor-hf-doc-builder.js";import{I as tt}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as K}from"../chunks/CodeBlock-hf-doc-builder.js";function lr(Vs){let I,wa,P,O,Ze,Q,st,ea,ot,ba,f,rt,ye,lt,nt,X,it,dt,_e,pt,ct,ka,ve,ht,ja,H,aa,ft,ut,we,ta,mt,gt,$a,F,M,sa,Z,yt,oa,_t,qa,j,vt,ra,wt,bt,la,kt,jt,Ea,U,$,na,$t,qt,be,Et,At,ee,xt,Dt,It,q,ia,Pt,Ft,ke,Tt,Ct,ae,Nt,St,Aa,je,Lt,xa,$e,Ot,Da,te,Ia,qe,Ht,Pa,se,da,zs,Fa,E,Mt,pa,Ut,Yt,Ee,Gt,Wt,Ta,oe,Ca,A,Bt,ca,Jt,Rt,Ae,Vt,zt,Na,re,Sa,x,Kt,ha,Qt,Xt,fa,Zt,es,La,le,Oa,D,as,xe,De,ts,ss,Ie,Pe,os,rs,Ha,T,Y,ua,ne,ls,ma,ns,Ma,G,is,ga,ds,ps,Ua,W,b,C,cs,Fe,hs,fs,ie,us,ms,gs,de,ys,N,_s,Te,vs,ws,ya,bs,ks,js,pe,$s,k,ce,qs,Ce,Es,As,xs,he,Ds,S,Is,Ne,Ps,Fs,Se,Ts,Cs,Ns,fe,Ya,L,B,_a,ue,Ss,va,Ls,Ga,Le,Os,Wa,u,Hs,Oe,He,Ms,Us,Me,Ue,Ys,Gs,Ye,Ge,Ws,Bs,Ba,We,Js,Ja;return Q=new tt({}),Z=new tt({}),te=new K({props:{code:`pokemon/train/grass/bulbasaur.png
pokemon/train/fire/charmander.png
pokemon/train/water/squirtle.png

pokemon/test/grass/ivysaur.png
pokemon/test/fire/charmeleon.png
pokemon/test/water/wartortle.png`,highlighted:`pokemon<span class="hljs-regexp">/train/g</span>rass/bulbasaur.png
pokemon<span class="hljs-regexp">/train/</span>fire/charmander.png
pokemon<span class="hljs-regexp">/train/</span>water/squirtle.png

pokemon<span class="hljs-regexp">/test/g</span>rass/ivysaur.png
pokemon<span class="hljs-regexp">/test/</span>fire/charmeleon.png
pokemon<span class="hljs-regexp">/test/</span>water/wartortle.png`}}),oe=new K({props:{code:`from datasets import ImageFolder

dataset = load_dataset("imagefolder", data_dir="/path/to/pokemon")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> ImageFolder

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;imagefolder&quot;</span>, data_dir=<span class="hljs-string">&quot;/path/to/pokemon&quot;</span>)`}}),re=new K({props:{code:`from datasets import AudioFolder

dataset = load_dataset("audiofolder", data_dir="/path/to/folder")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> AudioFolder

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;audiofolder&quot;</span>, data_dir=<span class="hljs-string">&quot;/path/to/folder&quot;</span>)`}}),le=new K({props:{code:`file_name, text
bulbasaur.png, There is a plant seed on its back right from the day this Pok\xE9mon is born.
charmander.png, It has a preference for hot things.
squirtle.png, When it retracts its long neck into its shell, it squirts out water with vigorous force.`,highlighted:`file_name, <span class="hljs-built_in">text</span>
bulbasaur.png, There <span class="hljs-keyword">is</span> a plant seed <span class="hljs-keyword">on</span> <span class="hljs-keyword">its</span> <span class="hljs-keyword">back</span> right <span class="hljs-keyword">from</span> <span class="hljs-keyword">the</span> <span class="hljs-built_in">day</span> this Pok\xE9mon <span class="hljs-keyword">is</span> born.
charmander.png, It has a preference <span class="hljs-keyword">for</span> hot things.
squirtle.png, When <span class="hljs-keyword">it</span> retracts <span class="hljs-keyword">its</span> long neck <span class="hljs-keyword">into</span> <span class="hljs-keyword">its</span> shell, <span class="hljs-keyword">it</span> squirts out water <span class="hljs-keyword">with</span> vigorous force.`}}),ne=new tt({}),de=new K({props:{code:`from datasets import Dataset
def gen():
    yield {"pokemon": "bulbasaur", "type": "grass"}
    yield {"pokemon": "squirtle", "type": "water"}
ds = Dataset.from_generator(gen)
ds[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">gen</span>():
<span class="hljs-meta">... </span>    <span class="hljs-keyword">yield</span> {<span class="hljs-string">&quot;pokemon&quot;</span>: <span class="hljs-string">&quot;bulbasaur&quot;</span>, <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;grass&quot;</span>}
<span class="hljs-meta">... </span>    <span class="hljs-keyword">yield</span> {<span class="hljs-string">&quot;pokemon&quot;</span>: <span class="hljs-string">&quot;squirtle&quot;</span>, <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;water&quot;</span>}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_generator(gen)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&quot;pokemon&quot;</span>: <span class="hljs-string">&quot;bulbasaur&quot;</span>, <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;grass&quot;</span>}`}}),pe=new K({props:{code:`from datasets import IterableDataset
ds = IterableDataset.from_generator(gen)
for example in ds:
    print(example)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> IterableDataset
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = IterableDataset.from_generator(gen)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> ds:
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(example)
{<span class="hljs-string">&quot;pokemon&quot;</span>: <span class="hljs-string">&quot;bulbasaur&quot;</span>, <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;grass&quot;</span>}
{<span class="hljs-string">&quot;pokemon&quot;</span>: <span class="hljs-string">&quot;squirtle&quot;</span>, <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;water&quot;</span>}`}}),he=new K({props:{code:`from datasets import Dataset
ds = Dataset.from_dict({"pokemon": ["bulbasaur", "squirtle"], "type": ["grass", "water"]})
ds[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;pokemon&quot;</span>: [<span class="hljs-string">&quot;bulbasaur&quot;</span>, <span class="hljs-string">&quot;squirtle&quot;</span>], <span class="hljs-string">&quot;type&quot;</span>: [<span class="hljs-string">&quot;grass&quot;</span>, <span class="hljs-string">&quot;water&quot;</span>]})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&quot;pokemon&quot;</span>: <span class="hljs-string">&quot;bulbasaur&quot;</span>, <span class="hljs-string">&quot;type&quot;</span>: <span class="hljs-string">&quot;grass&quot;</span>}`}}),fe=new K({props:{code:'audio_dataset = Dataset.from_dict({"audio": ["path/to/audio_1", ..., "path/to/audio_n"]}).cast_column("audio", Audio())',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>audio_dataset = Dataset.from_dict({<span class="hljs-string">&quot;audio&quot;</span>: [<span class="hljs-string">&quot;path/to/audio_1&quot;</span>, ..., <span class="hljs-string">&quot;path/to/audio_n&quot;</span>]}).cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio())'}}),ue=new tt({}),{c(){I=r("meta"),wa=c(),P=r("h1"),O=r("a"),Ze=r("span"),m(Q.$$.fragment),st=c(),ea=r("span"),ot=s("Create a dataset"),ba=c(),f=r("p"),rt=s("Sometimes, you may need to create a dataset if you\u2019re working with your own data. Creating a dataset with \u{1F917} Datasets confers all the advantages of the library to your dataset: fast loading and processing, "),ye=r("a"),lt=s("stream enormous datasets"),nt=s(", "),X=r("a"),it=s("memory-mapping"),dt=s(", and more. You can easily and rapidly create a dataset with \u{1F917} Datasets low-code approaches, reducing the time it takes to start training a model. In many cases, it is as easy as "),_e=r("a"),pt=s("dragging and dropping"),ct=s(" your data files into a dataset repository on the Hub."),ka=c(),ve=r("p"),ht=s("In this tutorial, you\u2019ll learn how to use \u{1F917} Datasets low-code methods for creating all types of datasets:"),ja=c(),H=r("ul"),aa=r("li"),ft=s("Folder-based builders for quickly creating an image or audio dataset"),ut=c(),we=r("li"),ta=r("code"),mt=s("from_"),gt=s(" methods for creating datasets from local files"),$a=c(),F=r("h2"),M=r("a"),sa=r("span"),m(Z.$$.fragment),yt=c(),oa=r("span"),_t=s("Folder-based builders"),qa=c(),j=r("p"),vt=s("There are two folder-based builders, "),ra=r("code"),wt=s("ImageFolder"),bt=s(" and "),la=r("code"),kt=s("AudioFolder"),jt=s(". These are low-code methods for quickly creating an image or speech and audio dataset with several thousand examples. They are great for rapidly prototyping computer vision and speech models before scaling to a larger dataset. Folder-based builders takes your data and automatically generates the dataset\u2019s features, splits, and labels. Under the hood:"),Ea=c(),U=r("ul"),$=r("li"),na=r("code"),$t=s("ImageFolder"),qt=s(" uses the "),be=r("a"),Et=s("Image"),At=s(" feature to decode an image file. Many image extension formats are supported, such as jpg and png, but other formats are also supported. You can check the complete "),ee=r("a"),xt=s("list"),Dt=s(" of supported image extensions."),It=c(),q=r("li"),ia=r("code"),Pt=s("AudioFolder"),Ft=s(" uses the "),ke=r("a"),Tt=s("Audio"),Ct=s(" feature to decode an audio file. Audio extensions such as wav and mp3 are supported, and you can check the complete "),ae=r("a"),Nt=s("list"),St=s(" of supported audio extensions."),Aa=c(),je=r("p"),Lt=s("The dataset splits are generated from the repository structure, and the label names are automatically inferred from the directory name."),xa=c(),$e=r("p"),Ot=s("For example, if your image dataset (it is the same for an audio dataset) is stored like this:"),Da=c(),m(te.$$.fragment),Ia=c(),qe=r("p"),Ht=s("Then this is how the folder-based builder generates an example:"),Pa=c(),se=r("div"),da=r("img"),Fa=c(),E=r("p"),Mt=s("Create the image dataset by specifying "),pa=r("code"),Ut=s("imagefolder"),Yt=s(" in "),Ee=r("a"),Gt=s("load_dataset()"),Wt=s(":"),Ta=c(),m(oe.$$.fragment),Ca=c(),A=r("p"),Bt=s("An audio dataset is created in the same way, except you specify "),ca=r("code"),Jt=s("audiofolder"),Rt=s(" in "),Ae=r("a"),Vt=s("load_dataset()"),zt=s(" instead:"),Na=c(),m(re.$$.fragment),Sa=c(),x=r("p"),Kt=s("Any additional information about your dataset, such as text captions or transcriptions, can be included with a "),ha=r("code"),Qt=s("metadata.csv"),Xt=s(" file in the folder containing your dataset. The metadata file needs to have a "),fa=r("code"),Zt=s("file_name"),es=s(" column that links the image or audio file to its corresponding metadata:"),La=c(),m(le.$$.fragment),Oa=c(),D=r("p"),as=s("To learn more about each of these folder-based builders, check out the and "),xe=r("a"),De=r("span"),ts=s("ImageFolder"),ss=s(" or "),Ie=r("a"),Pe=r("span"),os=s("AudioFolder"),rs=s(" guides."),Ha=c(),T=r("h2"),Y=r("a"),ua=r("span"),m(ne.$$.fragment),ls=c(),ma=r("span"),ns=s("From local files"),Ma=c(),G=r("p"),is=s("You can also create a dataset from local files by specifying the path to the data files. There are two ways you can create a dataset using the "),ga=r("code"),ds=s("from_"),ps=s(" methods:"),Ua=c(),W=r("ul"),b=r("li"),C=r("p"),cs=s("The "),Fe=r("a"),hs=s("from_generator()"),fs=s(" method is the most memory-efficient way to create a dataset from a "),ie=r("a"),us=s("generator"),ms=s(" due to a generators iterative behavior. This is especially useful when you\u2019re working with a really large dataset that may not fit in memory, since the dataset is generated on disk progressively and then memory-mapped."),gs=c(),m(de.$$.fragment),ys=c(),N=r("p"),_s=s("A generator-based "),Te=r("a"),vs=s("IterableDataset"),ws=s(" needs to be iterated over with a "),ya=r("code"),bs=s("for"),ks=s(" loop for example:"),js=c(),m(pe.$$.fragment),$s=c(),k=r("li"),ce=r("p"),qs=s("The "),Ce=r("a"),Es=s("from_dict()"),As=s(" method is a straightforward way to create a dataset from a dictionary:"),xs=c(),m(he.$$.fragment),Ds=c(),S=r("p"),Is=s("To create an image or audio dataset, chain the "),Ne=r("a"),Ps=s("cast_column()"),Fs=s(" method with "),Se=r("a"),Ts=s("from_dict()"),Cs=s(" and specify the column and feature type. For example, to create an audio dataset:"),Ns=c(),m(fe.$$.fragment),Ya=c(),L=r("h2"),B=r("a"),_a=r("span"),m(ue.$$.fragment),Ss=c(),va=r("span"),Ls=s("Next steps"),Ga=c(),Le=r("p"),Os=s("We didn\u2019t mention this in the tutorial, but you can also create a dataset with a loading script. A loading script is a more manual and code-intensive method for creating a dataset, but it also gives you the most flexibility and control over how a dataset is generated. It lets you configure additional options such as creating multiple configurations within a dataset, or enabling your dataset to be streamed."),Wa=c(),u=r("p"),Hs=s("To learn more about how to write loading scripts, take a look at the "),Oe=r("a"),He=r("span"),Ms=s("image loading script"),Us=s(", "),Me=r("a"),Ue=r("span"),Ys=s("audio loading script"),Gs=s(", and "),Ye=r("a"),Ge=r("span"),Ws=s("text loading script"),Bs=s(" guides."),Ba=c(),We=r("p"),Js=s("Now that you know how to create a dataset, consider sharing it on the Hub so the community can also benefit from your work! Go on to the next section to learn how to share your dataset."),this.h()},l(e){const i=tr('[data-svelte="svelte-1phssyn"]',document.head);I=l(i,"META",{name:!0,content:!0}),i.forEach(t),wa=h(e),P=l(e,"H1",{class:!0});var Ra=n(P);O=l(Ra,"A",{id:!0,class:!0,href:!0});var Ks=n(O);Ze=l(Ks,"SPAN",{});var Qs=n(Ze);g(Q.$$.fragment,Qs),Qs.forEach(t),Ks.forEach(t),st=h(Ra),ea=l(Ra,"SPAN",{});var Xs=n(ea);ot=o(Xs,"Create a dataset"),Xs.forEach(t),Ra.forEach(t),ba=h(e),f=l(e,"P",{});var J=n(f);rt=o(J,"Sometimes, you may need to create a dataset if you\u2019re working with your own data. Creating a dataset with \u{1F917} Datasets confers all the advantages of the library to your dataset: fast loading and processing, "),ye=l(J,"A",{href:!0});var Zs=n(ye);lt=o(Zs,"stream enormous datasets"),Zs.forEach(t),nt=o(J,", "),X=l(J,"A",{href:!0,rel:!0});var eo=n(X);it=o(eo,"memory-mapping"),eo.forEach(t),dt=o(J,", and more. You can easily and rapidly create a dataset with \u{1F917} Datasets low-code approaches, reducing the time it takes to start training a model. In many cases, it is as easy as "),_e=l(J,"A",{href:!0});var ao=n(_e);pt=o(ao,"dragging and dropping"),ao.forEach(t),ct=o(J," your data files into a dataset repository on the Hub."),J.forEach(t),ka=h(e),ve=l(e,"P",{});var to=n(ve);ht=o(to,"In this tutorial, you\u2019ll learn how to use \u{1F917} Datasets low-code methods for creating all types of datasets:"),to.forEach(t),ja=h(e),H=l(e,"UL",{});var Va=n(H);aa=l(Va,"LI",{});var so=n(aa);ft=o(so,"Folder-based builders for quickly creating an image or audio dataset"),so.forEach(t),ut=h(Va),we=l(Va,"LI",{});var Rs=n(we);ta=l(Rs,"CODE",{});var oo=n(ta);mt=o(oo,"from_"),oo.forEach(t),gt=o(Rs," methods for creating datasets from local files"),Rs.forEach(t),Va.forEach(t),$a=h(e),F=l(e,"H2",{class:!0});var za=n(F);M=l(za,"A",{id:!0,class:!0,href:!0});var ro=n(M);sa=l(ro,"SPAN",{});var lo=n(sa);g(Z.$$.fragment,lo),lo.forEach(t),ro.forEach(t),yt=h(za),oa=l(za,"SPAN",{});var no=n(oa);_t=o(no,"Folder-based builders"),no.forEach(t),za.forEach(t),qa=h(e),j=l(e,"P",{});var Be=n(j);vt=o(Be,"There are two folder-based builders, "),ra=l(Be,"CODE",{});var io=n(ra);wt=o(io,"ImageFolder"),io.forEach(t),bt=o(Be," and "),la=l(Be,"CODE",{});var po=n(la);kt=o(po,"AudioFolder"),po.forEach(t),jt=o(Be,". These are low-code methods for quickly creating an image or speech and audio dataset with several thousand examples. They are great for rapidly prototyping computer vision and speech models before scaling to a larger dataset. Folder-based builders takes your data and automatically generates the dataset\u2019s features, splits, and labels. Under the hood:"),Be.forEach(t),Ea=h(e),U=l(e,"UL",{});var Ka=n(U);$=l(Ka,"LI",{});var me=n($);na=l(me,"CODE",{});var co=n(na);$t=o(co,"ImageFolder"),co.forEach(t),qt=o(me," uses the "),be=l(me,"A",{href:!0});var ho=n(be);Et=o(ho,"Image"),ho.forEach(t),At=o(me," feature to decode an image file. Many image extension formats are supported, such as jpg and png, but other formats are also supported. You can check the complete "),ee=l(me,"A",{href:!0,rel:!0});var fo=n(ee);xt=o(fo,"list"),fo.forEach(t),Dt=o(me," of supported image extensions."),me.forEach(t),It=h(Ka),q=l(Ka,"LI",{});var ge=n(q);ia=l(ge,"CODE",{});var uo=n(ia);Pt=o(uo,"AudioFolder"),uo.forEach(t),Ft=o(ge," uses the "),ke=l(ge,"A",{href:!0});var mo=n(ke);Tt=o(mo,"Audio"),mo.forEach(t),Ct=o(ge," feature to decode an audio file. Audio extensions such as wav and mp3 are supported, and you can check the complete "),ae=l(ge,"A",{href:!0,rel:!0});var go=n(ae);Nt=o(go,"list"),go.forEach(t),St=o(ge," of supported audio extensions."),ge.forEach(t),Ka.forEach(t),Aa=h(e),je=l(e,"P",{});var yo=n(je);Lt=o(yo,"The dataset splits are generated from the repository structure, and the label names are automatically inferred from the directory name."),yo.forEach(t),xa=h(e),$e=l(e,"P",{});var _o=n($e);Ot=o(_o,"For example, if your image dataset (it is the same for an audio dataset) is stored like this:"),_o.forEach(t),Da=h(e),g(te.$$.fragment,e),Ia=h(e),qe=l(e,"P",{});var vo=n(qe);Ht=o(vo,"Then this is how the folder-based builder generates an example:"),vo.forEach(t),Pa=h(e),se=l(e,"DIV",{class:!0});var wo=n(se);da=l(wo,"IMG",{src:!0}),wo.forEach(t),Fa=h(e),E=l(e,"P",{});var Je=n(E);Mt=o(Je,"Create the image dataset by specifying "),pa=l(Je,"CODE",{});var bo=n(pa);Ut=o(bo,"imagefolder"),bo.forEach(t),Yt=o(Je," in "),Ee=l(Je,"A",{href:!0});var ko=n(Ee);Gt=o(ko,"load_dataset()"),ko.forEach(t),Wt=o(Je,":"),Je.forEach(t),Ta=h(e),g(oe.$$.fragment,e),Ca=h(e),A=l(e,"P",{});var Re=n(A);Bt=o(Re,"An audio dataset is created in the same way, except you specify "),ca=l(Re,"CODE",{});var jo=n(ca);Jt=o(jo,"audiofolder"),jo.forEach(t),Rt=o(Re," in "),Ae=l(Re,"A",{href:!0});var $o=n(Ae);Vt=o($o,"load_dataset()"),$o.forEach(t),zt=o(Re," instead:"),Re.forEach(t),Na=h(e),g(re.$$.fragment,e),Sa=h(e),x=l(e,"P",{});var Ve=n(x);Kt=o(Ve,"Any additional information about your dataset, such as text captions or transcriptions, can be included with a "),ha=l(Ve,"CODE",{});var qo=n(ha);Qt=o(qo,"metadata.csv"),qo.forEach(t),Xt=o(Ve," file in the folder containing your dataset. The metadata file needs to have a "),fa=l(Ve,"CODE",{});var Eo=n(fa);Zt=o(Eo,"file_name"),Eo.forEach(t),es=o(Ve," column that links the image or audio file to its corresponding metadata:"),Ve.forEach(t),La=h(e),g(le.$$.fragment,e),Oa=h(e),D=l(e,"P",{});var ze=n(D);as=o(ze,"To learn more about each of these folder-based builders, check out the and "),xe=l(ze,"A",{href:!0});var Ao=n(xe);De=l(Ao,"SPAN",{class:!0});var xo=n(De);ts=o(xo,"ImageFolder"),xo.forEach(t),Ao.forEach(t),ss=o(ze," or "),Ie=l(ze,"A",{href:!0});var Do=n(Ie);Pe=l(Do,"SPAN",{class:!0});var Io=n(Pe);os=o(Io,"AudioFolder"),Io.forEach(t),Do.forEach(t),rs=o(ze," guides."),ze.forEach(t),Ha=h(e),T=l(e,"H2",{class:!0});var Qa=n(T);Y=l(Qa,"A",{id:!0,class:!0,href:!0});var Po=n(Y);ua=l(Po,"SPAN",{});var Fo=n(ua);g(ne.$$.fragment,Fo),Fo.forEach(t),Po.forEach(t),ls=h(Qa),ma=l(Qa,"SPAN",{});var To=n(ma);ns=o(To,"From local files"),To.forEach(t),Qa.forEach(t),Ma=h(e),G=l(e,"P",{});var Xa=n(G);is=o(Xa,"You can also create a dataset from local files by specifying the path to the data files. There are two ways you can create a dataset using the "),ga=l(Xa,"CODE",{});var Co=n(ga);ds=o(Co,"from_"),Co.forEach(t),ps=o(Xa," methods:"),Xa.forEach(t),Ua=h(e),W=l(e,"UL",{});var Za=n(W);b=l(Za,"LI",{});var R=n(b);C=l(R,"P",{});var Ke=n(C);cs=o(Ke,"The "),Fe=l(Ke,"A",{href:!0});var No=n(Fe);hs=o(No,"from_generator()"),No.forEach(t),fs=o(Ke," method is the most memory-efficient way to create a dataset from a "),ie=l(Ke,"A",{href:!0,rel:!0});var So=n(ie);us=o(So,"generator"),So.forEach(t),ms=o(Ke," due to a generators iterative behavior. This is especially useful when you\u2019re working with a really large dataset that may not fit in memory, since the dataset is generated on disk progressively and then memory-mapped."),Ke.forEach(t),gs=h(R),g(de.$$.fragment,R),ys=h(R),N=l(R,"P",{});var Qe=n(N);_s=o(Qe,"A generator-based "),Te=l(Qe,"A",{href:!0});var Lo=n(Te);vs=o(Lo,"IterableDataset"),Lo.forEach(t),ws=o(Qe," needs to be iterated over with a "),ya=l(Qe,"CODE",{});var Oo=n(ya);bs=o(Oo,"for"),Oo.forEach(t),ks=o(Qe," loop for example:"),Qe.forEach(t),js=h(R),g(pe.$$.fragment,R),R.forEach(t),$s=h(Za),k=l(Za,"LI",{});var V=n(k);ce=l(V,"P",{});var et=n(ce);qs=o(et,"The "),Ce=l(et,"A",{href:!0});var Ho=n(Ce);Es=o(Ho,"from_dict()"),Ho.forEach(t),As=o(et," method is a straightforward way to create a dataset from a dictionary:"),et.forEach(t),xs=h(V),g(he.$$.fragment,V),Ds=h(V),S=l(V,"P",{});var Xe=n(S);Is=o(Xe,"To create an image or audio dataset, chain the "),Ne=l(Xe,"A",{href:!0});var Mo=n(Ne);Ps=o(Mo,"cast_column()"),Mo.forEach(t),Fs=o(Xe," method with "),Se=l(Xe,"A",{href:!0});var Uo=n(Se);Ts=o(Uo,"from_dict()"),Uo.forEach(t),Cs=o(Xe," and specify the column and feature type. For example, to create an audio dataset:"),Xe.forEach(t),Ns=h(V),g(fe.$$.fragment,V),V.forEach(t),Za.forEach(t),Ya=h(e),L=l(e,"H2",{class:!0});var at=n(L);B=l(at,"A",{id:!0,class:!0,href:!0});var Yo=n(B);_a=l(Yo,"SPAN",{});var Go=n(_a);g(ue.$$.fragment,Go),Go.forEach(t),Yo.forEach(t),Ss=h(at),va=l(at,"SPAN",{});var Wo=n(va);Ls=o(Wo,"Next steps"),Wo.forEach(t),at.forEach(t),Ga=h(e),Le=l(e,"P",{});var Bo=n(Le);Os=o(Bo,"We didn\u2019t mention this in the tutorial, but you can also create a dataset with a loading script. A loading script is a more manual and code-intensive method for creating a dataset, but it also gives you the most flexibility and control over how a dataset is generated. It lets you configure additional options such as creating multiple configurations within a dataset, or enabling your dataset to be streamed."),Bo.forEach(t),Wa=h(e),u=l(e,"P",{});var z=n(u);Hs=o(z,"To learn more about how to write loading scripts, take a look at the "),Oe=l(z,"A",{href:!0});var Jo=n(Oe);He=l(Jo,"SPAN",{class:!0});var Ro=n(He);Ms=o(Ro,"image loading script"),Ro.forEach(t),Jo.forEach(t),Us=o(z,", "),Me=l(z,"A",{href:!0});var Vo=n(Me);Ue=l(Vo,"SPAN",{class:!0});var zo=n(Ue);Ys=o(zo,"audio loading script"),zo.forEach(t),Vo.forEach(t),Gs=o(z,", and "),Ye=l(z,"A",{href:!0});var Ko=n(Ye);Ge=l(Ko,"SPAN",{class:!0});var Qo=n(Ge);Ws=o(Qo,"text loading script"),Qo.forEach(t),Ko.forEach(t),Bs=o(z," guides."),z.forEach(t),Ba=h(e),We=l(e,"P",{});var Xo=n(We);Js=o(Xo,"Now that you know how to create a dataset, consider sharing it on the Hub so the community can also benefit from your work! Go on to the next section to learn how to share your dataset."),Xo.forEach(t),this.h()},h(){d(I,"name","hf:doc:metadata"),d(I,"content",JSON.stringify(nr)),d(O,"id","create-a-dataset"),d(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(O,"href","#create-a-dataset"),d(P,"class","relative group"),d(ye,"href","stream"),d(X,"href","https://huggingface.co/course/chapter5/4?fw=pt#the-magic-of-memory-mapping"),d(X,"rel","nofollow"),d(_e,"href","upload_dataset#upload-with-the-hub-ui"),d(M,"id","folderbased-builders"),d(M,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(M,"href","#folderbased-builders"),d(F,"class","relative group"),d(be,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Image"),d(ee,"href","https://github.com/huggingface/datasets/blob/b5672a956d5de864e6f5550e493527d962d6ae55/src/datasets/packaged_modules/imagefolder/imagefolder.py#L39"),d(ee,"rel","nofollow"),d(ke,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Audio"),d(ae,"href","https://github.com/huggingface/datasets/blob/b5672a956d5de864e6f5550e493527d962d6ae55/src/datasets/packaged_modules/audiofolder/audiofolder.py#L39"),d(ae,"rel","nofollow"),sr(da.src,zs="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/folder-based-builder.png")||d(da,"src",zs),d(se,"class","flex justify-center"),d(Ee,"href","/docs/datasets/v2.11.0/en/package_reference/loading_methods#datasets.load_dataset"),d(Ae,"href","/docs/datasets/v2.11.0/en/package_reference/loading_methods#datasets.load_dataset"),d(De,"class","underline decoration-yellow-400 decoration-2 font-semibold"),d(xe,"href","https://huggingface.co/docs/datasets/image_dataset#imagefolder"),d(Pe,"class","underline decoration-pink-400 decoration-2 font-semibold"),d(Ie,"href","https://huggingface.co/docs/datasets/audio_dataset#audiofolder"),d(Y,"id","from-local-files"),d(Y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Y,"href","#from-local-files"),d(T,"class","relative group"),d(Fe,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.from_generator"),d(ie,"href","https://wiki.python.org/moin/Generators"),d(ie,"rel","nofollow"),d(Te,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.IterableDataset"),d(Ce,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.from_dict"),d(Ne,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.cast_column"),d(Se,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.from_dict"),d(B,"id","next-steps"),d(B,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(B,"href","#next-steps"),d(L,"class","relative group"),d(He,"class","underline decoration-yellow-400 decoration-2 font-semibold"),d(Oe,"href","https://huggingface.co/docs/datasets/main/en/image_dataset#loading-script"),d(Ue,"class","underline decoration-pink-400 decoration-2 font-semibold"),d(Me,"href","https://huggingface.co/docs/datasets/main/en/audio_dataset"),d(Ge,"class","underline decoration-green-400 decoration-2 font-semibold"),d(Ye,"href","https://huggingface.co/docs/datasets/main/en/dataset_script")},m(e,i){a(document.head,I),p(e,wa,i),p(e,P,i),a(P,O),a(O,Ze),y(Q,Ze,null),a(P,st),a(P,ea),a(ea,ot),p(e,ba,i),p(e,f,i),a(f,rt),a(f,ye),a(ye,lt),a(f,nt),a(f,X),a(X,it),a(f,dt),a(f,_e),a(_e,pt),a(f,ct),p(e,ka,i),p(e,ve,i),a(ve,ht),p(e,ja,i),p(e,H,i),a(H,aa),a(aa,ft),a(H,ut),a(H,we),a(we,ta),a(ta,mt),a(we,gt),p(e,$a,i),p(e,F,i),a(F,M),a(M,sa),y(Z,sa,null),a(F,yt),a(F,oa),a(oa,_t),p(e,qa,i),p(e,j,i),a(j,vt),a(j,ra),a(ra,wt),a(j,bt),a(j,la),a(la,kt),a(j,jt),p(e,Ea,i),p(e,U,i),a(U,$),a($,na),a(na,$t),a($,qt),a($,be),a(be,Et),a($,At),a($,ee),a(ee,xt),a($,Dt),a(U,It),a(U,q),a(q,ia),a(ia,Pt),a(q,Ft),a(q,ke),a(ke,Tt),a(q,Ct),a(q,ae),a(ae,Nt),a(q,St),p(e,Aa,i),p(e,je,i),a(je,Lt),p(e,xa,i),p(e,$e,i),a($e,Ot),p(e,Da,i),y(te,e,i),p(e,Ia,i),p(e,qe,i),a(qe,Ht),p(e,Pa,i),p(e,se,i),a(se,da),p(e,Fa,i),p(e,E,i),a(E,Mt),a(E,pa),a(pa,Ut),a(E,Yt),a(E,Ee),a(Ee,Gt),a(E,Wt),p(e,Ta,i),y(oe,e,i),p(e,Ca,i),p(e,A,i),a(A,Bt),a(A,ca),a(ca,Jt),a(A,Rt),a(A,Ae),a(Ae,Vt),a(A,zt),p(e,Na,i),y(re,e,i),p(e,Sa,i),p(e,x,i),a(x,Kt),a(x,ha),a(ha,Qt),a(x,Xt),a(x,fa),a(fa,Zt),a(x,es),p(e,La,i),y(le,e,i),p(e,Oa,i),p(e,D,i),a(D,as),a(D,xe),a(xe,De),a(De,ts),a(D,ss),a(D,Ie),a(Ie,Pe),a(Pe,os),a(D,rs),p(e,Ha,i),p(e,T,i),a(T,Y),a(Y,ua),y(ne,ua,null),a(T,ls),a(T,ma),a(ma,ns),p(e,Ma,i),p(e,G,i),a(G,is),a(G,ga),a(ga,ds),a(G,ps),p(e,Ua,i),p(e,W,i),a(W,b),a(b,C),a(C,cs),a(C,Fe),a(Fe,hs),a(C,fs),a(C,ie),a(ie,us),a(C,ms),a(b,gs),y(de,b,null),a(b,ys),a(b,N),a(N,_s),a(N,Te),a(Te,vs),a(N,ws),a(N,ya),a(ya,bs),a(N,ks),a(b,js),y(pe,b,null),a(W,$s),a(W,k),a(k,ce),a(ce,qs),a(ce,Ce),a(Ce,Es),a(ce,As),a(k,xs),y(he,k,null),a(k,Ds),a(k,S),a(S,Is),a(S,Ne),a(Ne,Ps),a(S,Fs),a(S,Se),a(Se,Ts),a(S,Cs),a(k,Ns),y(fe,k,null),p(e,Ya,i),p(e,L,i),a(L,B),a(B,_a),y(ue,_a,null),a(L,Ss),a(L,va),a(va,Ls),p(e,Ga,i),p(e,Le,i),a(Le,Os),p(e,Wa,i),p(e,u,i),a(u,Hs),a(u,Oe),a(Oe,He),a(He,Ms),a(u,Us),a(u,Me),a(Me,Ue),a(Ue,Ys),a(u,Gs),a(u,Ye),a(Ye,Ge),a(Ge,Ws),a(u,Bs),p(e,Ba,i),p(e,We,i),a(We,Js),Ja=!0},p:or,i(e){Ja||(_(Q.$$.fragment,e),_(Z.$$.fragment,e),_(te.$$.fragment,e),_(oe.$$.fragment,e),_(re.$$.fragment,e),_(le.$$.fragment,e),_(ne.$$.fragment,e),_(de.$$.fragment,e),_(pe.$$.fragment,e),_(he.$$.fragment,e),_(fe.$$.fragment,e),_(ue.$$.fragment,e),Ja=!0)},o(e){v(Q.$$.fragment,e),v(Z.$$.fragment,e),v(te.$$.fragment,e),v(oe.$$.fragment,e),v(re.$$.fragment,e),v(le.$$.fragment,e),v(ne.$$.fragment,e),v(de.$$.fragment,e),v(pe.$$.fragment,e),v(he.$$.fragment,e),v(fe.$$.fragment,e),v(ue.$$.fragment,e),Ja=!1},d(e){t(I),e&&t(wa),e&&t(P),w(Q),e&&t(ba),e&&t(f),e&&t(ka),e&&t(ve),e&&t(ja),e&&t(H),e&&t($a),e&&t(F),w(Z),e&&t(qa),e&&t(j),e&&t(Ea),e&&t(U),e&&t(Aa),e&&t(je),e&&t(xa),e&&t($e),e&&t(Da),w(te,e),e&&t(Ia),e&&t(qe),e&&t(Pa),e&&t(se),e&&t(Fa),e&&t(E),e&&t(Ta),w(oe,e),e&&t(Ca),e&&t(A),e&&t(Na),w(re,e),e&&t(Sa),e&&t(x),e&&t(La),w(le,e),e&&t(Oa),e&&t(D),e&&t(Ha),e&&t(T),w(ne),e&&t(Ma),e&&t(G),e&&t(Ua),e&&t(W),w(de),w(pe),w(he),w(fe),e&&t(Ya),e&&t(L),w(ue),e&&t(Ga),e&&t(Le),e&&t(Wa),e&&t(u),e&&t(Ba),e&&t(We)}}}const nr={local:"create-a-dataset",sections:[{local:"folderbased-builders",title:"Folder-based builders"},{local:"from-local-files",title:"From local files"},{local:"next-steps",title:"Next steps"}],title:"Create a dataset"};function ir(Vs){return rr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class hr extends Zo{constructor(I){super();er(this,I,ir,lr,ar,{})}}export{hr as default,nr as metadata};
