import{S as xn,i as Hn,s as Bn,e as a,k as d,w as c,t as n,M as Gn,c as o,d as e,m as f,a as l,x as h,h as p,b as u,G as s,g as i,y as m,q as g,o as _,B as v,v as Yn}from"../chunks/vendor-hf-doc-builder.js";import{T as Rn}from"../chunks/Tip-hf-doc-builder.js";import{I as D}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as y}from"../chunks/CodeBlock-hf-doc-builder.js";function Mn(Ms){let $,U,w,j,O,E,$t,P;return{c(){$=a("p"),U=n("Remember to define your credentials in your "),w=a("a"),j=n("FileSystem instance"),O=d(),E=a("code"),$t=n("fs"),P=n(" whenever you are interacting with a private cloud storage."),this.h()},l(L){$=o(L,"P",{});var k=l($);U=p(k,"Remember to define your credentials in your "),w=o(k,"A",{href:!0});var C=l(w);j=p(C,"FileSystem instance"),C.forEach(e),O=f(k),E=o(k,"CODE",{});var ye=l(E);$t=p(ye,"fs"),ye.forEach(e),P=p(k," whenever you are interacting with a private cloud storage."),k.forEach(e),this.h()},h(){u(w,"href","#set-up-your-cloud-storage-filesystem")},m(L,k){i(L,$,k),s($,U),s($,w),s(w,j),s($,O),s($,E),s(E,$t),s($,P)},d(L){L&&e($)}}}function Un(Ms){let $,U,w,j,O,E,$t,P,L,k,C,ye,ze,To,Oo,Us,K,Ie,yt,Fe,Po,Lo,Ne,zo,Io,b,wt,xe,Fo,No,He,bt,xo,Ho,Et,Be,Bo,Go,Ge,kt,Yo,Ro,jt,Ye,Mo,Uo,Re,qt,Ko,Wo,St,Me,Jo,Qo,Ue,At,Vo,Xo,Dt,Ke,Zo,tl,We,Ct,el,sl,Tt,Je,al,ol,Qe,Ot,ll,Ks,we,rl,Ws,z,W,Ve,Pt,il,Xe,nl,Js,I,J,Ze,Lt,pl,ts,dl,Qs,be,es,fl,Vs,zt,Xs,It,ss,ul,Zs,q,cl,as,hl,ml,os,gl,_l,ls,vl,$l,ta,Ft,ea,Nt,rs,yl,sa,xt,aa,F,Q,is,Ht,wl,ns,bl,oa,Ee,ps,El,la,Bt,ra,Gt,ds,kl,ia,Yt,na,Rt,fs,jl,pa,Mt,da,N,V,us,Ut,ql,cs,Sl,fa,ke,hs,Al,ua,Kt,ca,Wt,ms,Dl,ha,Jt,ma,Qt,gs,Cl,ga,Vt,_a,x,X,_s,Xt,Tl,vs,Ol,va,je,$s,Pl,$a,Zt,ya,te,ys,Ll,wa,ee,ba,se,ws,zl,Ea,ae,ka,H,Z,bs,oe,Il,Es,Fl,ja,B,tt,ks,le,Nl,js,xl,qa,S,Hl,qs,Bl,Gl,Ss,Yl,Rl,As,Ml,Ul,Sa,et,Kl,Ds,Wl,Jl,Aa,st,at,Ql,Cs,Vl,Xl,qe,Zl,tr,Ts,er,Da,ot,sr,Se,ar,or,Ca,re,Ta,lt,lr,Ae,rr,ir,Oa,ie,Pa,rt,nr,De,pr,dr,La,ne,za,it,fr,Os,ur,cr,Ia,nt,hr,Ps,mr,gr,Fa,pe,Na,G,pt,Ls,de,_r,zs,vr,xa,Ce,$r,Ha,Te,yr,Ba,fe,Ga,dt,wr,ue,br,Er,Ya,Y,ft,Is,ce,kr,Fs,jr,Ra,ut,qr,Oe,Sr,Ar,Ma,he,Ua,ct,Ka,R,ht,Ns,me,Dr,xs,Cr,Wa,T,Tr,Hs,Or,Pr,Bs,Lr,zr,Ja,ge,Qa,M,mt,Gs,_e,Ir,Ys,Fr,Va,gt,Nr,Pe,xr,Hr,Xa,ve,Za;return E=new D({}),Pt=new D({}),Lt=new D({}),zt=new y({props:{code:"pip install s3fs",highlighted:'<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">pip install s3fs</span>'}}),Ft=new y({props:{code:`storage_options = {"anon": True}  # for anonymous connection
storage_options = {"key": aws_access_key_id, "secret": aws_secret_access_key}  # for private buckets
import aiobotocore.session
s3_session = aiobotocore.session.AioSession(profile="my_profile_name")
storage_options = {"session": s3_session}`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;anon&quot;</span>: <span class="hljs-literal">True</span>}  <span class="hljs-comment"># for anonymous connection</span>
<span class="hljs-comment"># or use your credentials</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;key&quot;</span>: aws_access_key_id, <span class="hljs-string">&quot;secret&quot;</span>: aws_secret_access_key}  <span class="hljs-comment"># for private buckets</span>
<span class="hljs-comment"># or use a botocore session</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> aiobotocore.session
<span class="hljs-meta">&gt;&gt;&gt; </span>s3_session = aiobotocore.session.AioSession(profile=<span class="hljs-string">&quot;my_profile_name&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;session&quot;</span>: s3_session}`}}),xt=new y({props:{code:`import s3fs
fs = s3fs.S3FileSystem(**storage_options)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> s3fs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = s3fs.S3FileSystem(**storage_options)`}}),Ht=new D({}),Bt=new y({props:{code:`conda install -c conda-forge gcsfs
pip install gcsfs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">conda install -c conda-forge gcsfs</span>
# or install with pip
<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">pip install gcsfs</span>`}}),Yt=new y({props:{code:`storage_options={"token": "anon"}  # for anonymous connection
storage_options={"project": "my-google-project"}
storage_options={"project": "my-google-project", "token": TOKEN}`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;token&quot;</span>: <span class="hljs-string">&quot;anon&quot;</span>}  <span class="hljs-comment"># for anonymous connection</span>
<span class="hljs-comment"># or use your credentials of your default gcloud credentials or from the google metadata service</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;project&quot;</span>: <span class="hljs-string">&quot;my-google-project&quot;</span>}
<span class="hljs-comment"># or use your credentials from elsewhere, see the documentation at https://gcsfs.readthedocs.io/</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;project&quot;</span>: <span class="hljs-string">&quot;my-google-project&quot;</span>, <span class="hljs-string">&quot;token&quot;</span>: TOKEN}`}}),Mt=new y({props:{code:`import gcsfs
fs = gcsfs.GCSFileSystem(**storage_options)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> gcsfs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = gcsfs.GCSFileSystem(**storage_options)`}}),Ut=new D({}),Kt=new y({props:{code:`conda install -c conda-forge adlfs
pip install adlfs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">conda install -c conda-forge adlfs</span>
# or install with pip
<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">pip install adlfs</span>`}}),Jt=new y({props:{code:`storage_options = {"anon": True}  # for anonymous connection
storage_options = {"account_name": ACCOUNT_NAME, "account_key": ACCOUNT_KEY}  # gen 2 filesystem
storage_options={"tenant_id": TENANT_ID, "client_id": CLIENT_ID, "client_secret": CLIENT_SECRET}`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;anon&quot;</span>: <span class="hljs-literal">True</span>}  <span class="hljs-comment"># for anonymous connection</span>
<span class="hljs-comment"># or use your credentials</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;account_name&quot;</span>: ACCOUNT_NAME, <span class="hljs-string">&quot;account_key&quot;</span>: ACCOUNT_KEY}  <span class="hljs-comment"># gen 2 filesystem</span>
<span class="hljs-comment"># or use your credentials with the gen 1 filesystem</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;tenant_id&quot;</span>: TENANT_ID, <span class="hljs-string">&quot;client_id&quot;</span>: CLIENT_ID, <span class="hljs-string">&quot;client_secret&quot;</span>: CLIENT_SECRET}`}}),Vt=new y({props:{code:`import adlfs
fs = adlfs.AzureBlobFileSystem(**storage_options)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> adlfs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = adlfs.AzureBlobFileSystem(**storage_options)`}}),Xt=new D({}),Zt=new y({props:{code:"pip install ocifs",highlighted:'<span class="hljs-meta">&gt;&gt;&gt;</span> <span class="language-python">pip install ocifs</span>'}}),ee=new y({props:{code:'storage_options = {"config": "~/.oci/config", "region": "us-ashburn-1"} ',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;config&quot;</span>: <span class="hljs-string">&quot;~/.oci/config&quot;</span>, <span class="hljs-string">&quot;region&quot;</span>: <span class="hljs-string">&quot;us-ashburn-1&quot;</span>} '}}),ae=new y({props:{code:`import ocifs
fs = ocifs.OCIFileSystem(**storage_options)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> ocifs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = ocifs.OCIFileSystem(**storage_options)`}}),oe=new D({}),le=new D({}),re=new y({props:{code:`output_dir = "s3://my-bucket/imdb"
builder = load_dataset_builder("imdb")
builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>output_dir = <span class="hljs-string">&quot;s3://my-bucket/imdb&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>builder = load_dataset_builder(<span class="hljs-string">&quot;imdb&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>)`}}),ie=new y({props:{code:`output_dir = "s3://my-bucket/imdb"
builder = load_dataset_builder("path/to/local/loading_script/loading_script.py")
builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>output_dir = <span class="hljs-string">&quot;s3://my-bucket/imdb&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>builder = load_dataset_builder(<span class="hljs-string">&quot;path/to/local/loading_script/loading_script.py&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>)`}}),ne=new y({props:{code:`data_files = {"train": ["path/to/train.csv"]}
output_dir = "s3://my-bucket/imdb"
builder = load_dataset_builder("csv", data_files=data_files)
builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>data_files = {<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;path/to/train.csv&quot;</span>]}
<span class="hljs-meta">&gt;&gt;&gt; </span>output_dir = <span class="hljs-string">&quot;s3://my-bucket/imdb&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>builder = load_dataset_builder(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files)
<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>)`}}),pe=new y({props:{code:'builder.download_and_prepare(output_dir, storage_options=storage_options, file_format="parquet", max_shard_size="1GB")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>, max_shard_size=<span class="hljs-string">&quot;1GB&quot;</span>)'}}),de=new D({}),fe=new y({props:{code:`import dask.dataframe as dd

df = dd.read_parquet(output_dir, storage_options=storage_options)

# or if your dataset is split into train/valid/test
df_train = dd.read_parquet(output_dir + f"/{builder.name}-train-*.parquet", storage_options=storage_options)
df_valid = dd.read_parquet(output_dir + f"/{builder.name}-validation-*.parquet", storage_options=storage_options)
df_test = dd.read_parquet(output_dir + f"/{builder.name}-test-*.parquet", storage_options=storage_options)`,highlighted:`<span class="hljs-keyword">import</span> dask.dataframe <span class="hljs-keyword">as</span> dd

df = dd.read_parquet(output_dir, storage_options=storage_options)

<span class="hljs-comment"># or if your dataset is split into train/valid/test</span>
df_train = dd.read_parquet(output_dir + <span class="hljs-string">f&quot;/<span class="hljs-subst">{builder.name}</span>-train-*.parquet&quot;</span>, storage_options=storage_options)
df_valid = dd.read_parquet(output_dir + <span class="hljs-string">f&quot;/<span class="hljs-subst">{builder.name}</span>-validation-*.parquet&quot;</span>, storage_options=storage_options)
df_test = dd.read_parquet(output_dir + <span class="hljs-string">f&quot;/<span class="hljs-subst">{builder.name}</span>-test-*.parquet&quot;</span>, storage_options=storage_options)`}}),ce=new D({}),he=new y({props:{code:`encoded_dataset.save_to_disk("s3://my-private-datasets/imdb/train", storage_options=storage_options)
encoded_dataset.save_to_disk("gcs://my-private-datasets/imdb/train", storage_options=storage_options)
encoded_dataset.save_to_disk("adl://my-private-datasets/imdb/train", storage_options=storage_options)`,highlighted:`<span class="hljs-comment"># saves encoded_dataset to amazon s3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_dataset.save_to_disk(<span class="hljs-string">&quot;s3://my-private-datasets/imdb/train&quot;</span>, storage_options=storage_options)
<span class="hljs-comment"># saves encoded_dataset to google cloud storage</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_dataset.save_to_disk(<span class="hljs-string">&quot;gcs://my-private-datasets/imdb/train&quot;</span>, storage_options=storage_options)
<span class="hljs-comment"># saves encoded_dataset to microsoft azure blob/datalake</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_dataset.save_to_disk(<span class="hljs-string">&quot;adl://my-private-datasets/imdb/train&quot;</span>, storage_options=storage_options)`}}),ct=new Rn({props:{$$slots:{default:[Mn]},$$scope:{ctx:Ms}}}),me=new D({}),ge=new y({props:{code:'fs.ls("my-private-datasets/imdb/train", detail=False)',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>fs.ls(<span class="hljs-string">&quot;my-private-datasets/imdb/train&quot;</span>, detail=<span class="hljs-literal">False</span>)
[<span class="hljs-string">&quot;dataset_info.json.json&quot;</span>,<span class="hljs-string">&quot;dataset.arrow&quot;</span>,<span class="hljs-string">&quot;state.json&quot;</span>]`}}),_e=new D({}),ve=new y({props:{code:`from datasets import load_from_disk
dataset = load_from_disk("s3://a-public-datasets/imdb/train", storage_options=storage_options)  
print(len(dataset))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_from_disk
<span class="hljs-comment"># load encoded_dataset from cloud storage</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_from_disk(<span class="hljs-string">&quot;s3://a-public-datasets/imdb/train&quot;</span>, storage_options=storage_options)  
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(dataset))
<span class="hljs-number">25000</span>`}}),{c(){$=a("meta"),U=d(),w=a("h1"),j=a("a"),O=a("span"),c(E.$$.fragment),$t=d(),P=a("span"),L=n("Cloud storage"),k=d(),C=a("p"),ye=n("\u{1F917} Datasets supports access to cloud storage providers through a "),ze=a("code"),To=n("fsspec"),Oo=n(` FileSystem implementations.
You can save and load datasets from any cloud storage in a Pythonic way.
Take a look at the following table for some example of supported cloud storage providers:`),Us=d(),K=a("table"),Ie=a("thead"),yt=a("tr"),Fe=a("th"),Po=n("Storage provider"),Lo=d(),Ne=a("th"),zo=n("Filesystem implementation"),Io=d(),b=a("tbody"),wt=a("tr"),xe=a("td"),Fo=n("Amazon S3"),No=d(),He=a("td"),bt=a("a"),xo=n("s3fs"),Ho=d(),Et=a("tr"),Be=a("td"),Bo=n("Google Cloud Storage"),Go=d(),Ge=a("td"),kt=a("a"),Yo=n("gcsfs"),Ro=d(),jt=a("tr"),Ye=a("td"),Mo=n("Azure Blob/DataLake"),Uo=d(),Re=a("td"),qt=a("a"),Ko=n("adlfs"),Wo=d(),St=a("tr"),Me=a("td"),Jo=n("Dropbox"),Qo=d(),Ue=a("td"),At=a("a"),Vo=n("dropboxdrivefs"),Xo=d(),Dt=a("tr"),Ke=a("td"),Zo=n("Google Drive"),tl=d(),We=a("td"),Ct=a("a"),el=n("gdrivefs"),sl=d(),Tt=a("tr"),Je=a("td"),al=n("Oracle Cloud Storage"),ol=d(),Qe=a("td"),Ot=a("a"),ll=n("ocifs"),Ks=d(),we=a("p"),rl=n(`This guide will show you how to save and load datasets with any cloud storage.
Here are examples for S3, Google Cloud Storage, Azure Blob Storage, and Oracle Cloud Object Storage.`),Ws=d(),z=a("h2"),W=a("a"),Ve=a("span"),c(Pt.$$.fragment),il=d(),Xe=a("span"),nl=n("Set up your cloud storage FileSystem"),Js=d(),I=a("h3"),J=a("a"),Ze=a("span"),c(Lt.$$.fragment),pl=d(),ts=a("span"),dl=n("Amazon S3"),Qs=d(),be=a("ol"),es=a("li"),fl=n("Install the S3 FileSystem implementation:"),Vs=d(),c(zt.$$.fragment),Xs=d(),It=a("ol"),ss=a("li"),ul=n("Define your credentials"),Zs=d(),q=a("p"),cl=n("To use an anonymous connection, use "),as=a("code"),hl=n("anon=True"),ml=n(`.
Otherwise, include your `),os=a("code"),gl=n("aws_access_key_id"),_l=n(" and "),ls=a("code"),vl=n("aws_secret_access_key"),$l=n(" whenever you are interacting with a private S3 bucket."),ta=d(),c(Ft.$$.fragment),ea=d(),Nt=a("ol"),rs=a("li"),yl=n("Create your FileSystem instance"),sa=d(),c(xt.$$.fragment),aa=d(),F=a("h3"),Q=a("a"),is=a("span"),c(Ht.$$.fragment),wl=d(),ns=a("span"),bl=n("Google Cloud Storage"),oa=d(),Ee=a("ol"),ps=a("li"),El=n("Install the Google Cloud Storage implementation:"),la=d(),c(Bt.$$.fragment),ra=d(),Gt=a("ol"),ds=a("li"),kl=n("Define your credentials"),ia=d(),c(Yt.$$.fragment),na=d(),Rt=a("ol"),fs=a("li"),jl=n("Create your FileSystem instance"),pa=d(),c(Mt.$$.fragment),da=d(),N=a("h3"),V=a("a"),us=a("span"),c(Ut.$$.fragment),ql=d(),cs=a("span"),Sl=n("Azure Blob Storage"),fa=d(),ke=a("ol"),hs=a("li"),Al=n("Install the Azure Blob Storage implementation:"),ua=d(),c(Kt.$$.fragment),ca=d(),Wt=a("ol"),ms=a("li"),Dl=n("Define your credentials"),ha=d(),c(Jt.$$.fragment),ma=d(),Qt=a("ol"),gs=a("li"),Cl=n("Create your FileSystem instance"),ga=d(),c(Vt.$$.fragment),_a=d(),x=a("h3"),X=a("a"),_s=a("span"),c(Xt.$$.fragment),Tl=d(),vs=a("span"),Ol=n("Oracle Cloud Object Storage"),va=d(),je=a("ol"),$s=a("li"),Pl=n("Install the OCI FileSystem implementation:"),$a=d(),c(Zt.$$.fragment),ya=d(),te=a("ol"),ys=a("li"),Ll=n("Define your credentials"),wa=d(),c(ee.$$.fragment),ba=d(),se=a("ol"),ws=a("li"),zl=n("Create your FileSystem instance"),Ea=d(),c(ae.$$.fragment),ka=d(),H=a("h2"),Z=a("a"),bs=a("span"),c(oe.$$.fragment),Il=d(),Es=a("span"),Fl=n("Load and Save your datasets using your cloud storage FileSystem"),ja=d(),B=a("h3"),tt=a("a"),ks=a("span"),c(le.$$.fragment),Nl=d(),js=a("span"),xl=n("Download and prepare a dataset into a cloud storage"),qa=d(),S=a("p"),Hl=n("You can download and prepare a dataset into your cloud storage by specifying a remote "),qs=a("code"),Bl=n("output_dir"),Gl=n(" in "),Ss=a("code"),Yl=n("download_and_prepare"),Rl=n(`.
Don\u2019t forget to use the previously defined `),As=a("code"),Ml=n("storage_options"),Ul=n(" containing your credentials to write into a private cloud storage."),Sa=d(),et=a("p"),Kl=n("The "),Ds=a("code"),Wl=n("download_and_prepare"),Jl=n(" method works in two steps:"),Aa=d(),st=a("ol"),at=a("li"),Ql=n("it first downloads the raw data files (if any) in your local cache. You can set your cache directory by passing "),Cs=a("code"),Vl=n("cache_dir"),Xl=n(" to "),qe=a("a"),Zl=n("load_dataset_builder()"),tr=d(),Ts=a("li"),er=n("then it generates the dataset in Arrow or Parquet format in your cloud storage by iterating over the raw data files."),Da=d(),ot=a("p"),sr=n("Load a dataset builder from the Hugging Face Hub (see "),Se=a("a"),ar=n("how to load from the Hugging Face Hub"),or=n("):"),Ca=d(),c(re.$$.fragment),Ta=d(),lt=a("p"),lr=n("Load a dataset builder using a loading script (see "),Ae=a("a"),rr=n("how to load a local loading script"),ir=n("):"),Oa=d(),c(ie.$$.fragment),Pa=d(),rt=a("p"),nr=n("Use your own data files (see "),De=a("a"),pr=n("how to load local and remote files"),dr=n("):"),La=d(),c(ne.$$.fragment),za=d(),it=a("p"),fr=n("It is highly recommended to save the files as compressed Parquet files to optimize I/O by specifying "),Os=a("code"),ur=n('file_format="parquet"'),cr=n(`.
Otherwise the dataset is saved as an uncompressed Arrow file.`),Ia=d(),nt=a("p"),hr=n("You can also specify the size of the shards using "),Ps=a("code"),mr=n("max_shard_size"),gr=n(" (default is 500MB):"),Fa=d(),c(pe.$$.fragment),Na=d(),G=a("h4"),pt=a("a"),Ls=a("span"),c(de.$$.fragment),_r=d(),zs=a("span"),vr=n("Dask"),xa=d(),Ce=a("p"),$r=n(`Dask is a parallel computing library and it has a pandas-like API for working with larger than memory Parquet datasets in parallel.
Dask can use multiple threads or processes on a single machine, or a cluster of machines to process data in parallel.
Dask supports local data but also data from a cloud storage.`),Ha=d(),Te=a("p"),yr=n("Therefore you can load a dataset saved as sharded Parquet files in Dask with"),Ba=d(),c(fe.$$.fragment),Ga=d(),dt=a("p"),wr=n("You can find more about dask dataframes in their "),ue=a("a"),br=n("documentation"),Er=n("."),Ya=d(),Y=a("h2"),ft=a("a"),Is=a("span"),c(ce.$$.fragment),kr=d(),Fs=a("span"),jr=n("Saving serialized datasets"),Ra=d(),ut=a("p"),qr=n("After you have processed your dataset, you can save it to your cloud storage with "),Oe=a("a"),Sr=n("Dataset.save_to_disk()"),Ar=n(":"),Ma=d(),c(he.$$.fragment),Ua=d(),c(ct.$$.fragment),Ka=d(),R=a("h2"),ht=a("a"),Ns=a("span"),c(me.$$.fragment),Dr=d(),xs=a("span"),Cr=n("Listing serialized datasets"),Wa=d(),T=a("p"),Tr=n("List files from a cloud storage with your FileSystem instance "),Hs=a("code"),Or=n("fs"),Pr=n(", using "),Bs=a("code"),Lr=n("fs.ls"),zr=n(":"),Ja=d(),c(ge.$$.fragment),Qa=d(),M=a("h3"),mt=a("a"),Gs=a("span"),c(_e.$$.fragment),Ir=d(),Ys=a("span"),Fr=n("Load serialized datasets"),Va=d(),gt=a("p"),Nr=n("When you are ready to use your dataset again, reload it with "),Pe=a("a"),xr=n("Dataset.load_from_disk()"),Hr=n(":"),Xa=d(),c(ve.$$.fragment),this.h()},l(t){const r=Gn('[data-svelte="svelte-1phssyn"]',document.head);$=o(r,"META",{name:!0,content:!0}),r.forEach(e),U=f(t),w=o(t,"H1",{class:!0});var $e=l(w);j=o($e,"A",{id:!0,class:!0,href:!0});var Br=l(j);O=o(Br,"SPAN",{});var Gr=l(O);h(E.$$.fragment,Gr),Gr.forEach(e),Br.forEach(e),$t=f($e),P=o($e,"SPAN",{});var Yr=l(P);L=p(Yr,"Cloud storage"),Yr.forEach(e),$e.forEach(e),k=f(t),C=o(t,"P",{});var to=l(C);ye=p(to,"\u{1F917} Datasets supports access to cloud storage providers through a "),ze=o(to,"CODE",{});var Rr=l(ze);To=p(Rr,"fsspec"),Rr.forEach(e),Oo=p(to,` FileSystem implementations.
You can save and load datasets from any cloud storage in a Pythonic way.
Take a look at the following table for some example of supported cloud storage providers:`),to.forEach(e),Us=f(t),K=o(t,"TABLE",{});var eo=l(K);Ie=o(eo,"THEAD",{});var Mr=l(Ie);yt=o(Mr,"TR",{});var so=l(yt);Fe=o(so,"TH",{});var Ur=l(Fe);Po=p(Ur,"Storage provider"),Ur.forEach(e),Lo=f(so),Ne=o(so,"TH",{});var Kr=l(Ne);zo=p(Kr,"Filesystem implementation"),Kr.forEach(e),so.forEach(e),Mr.forEach(e),Io=f(eo),b=o(eo,"TBODY",{});var A=l(b);wt=o(A,"TR",{});var ao=l(wt);xe=o(ao,"TD",{});var Wr=l(xe);Fo=p(Wr,"Amazon S3"),Wr.forEach(e),No=f(ao),He=o(ao,"TD",{});var Jr=l(He);bt=o(Jr,"A",{href:!0,rel:!0});var Qr=l(bt);xo=p(Qr,"s3fs"),Qr.forEach(e),Jr.forEach(e),ao.forEach(e),Ho=f(A),Et=o(A,"TR",{});var oo=l(Et);Be=o(oo,"TD",{});var Vr=l(Be);Bo=p(Vr,"Google Cloud Storage"),Vr.forEach(e),Go=f(oo),Ge=o(oo,"TD",{});var Xr=l(Ge);kt=o(Xr,"A",{href:!0,rel:!0});var Zr=l(kt);Yo=p(Zr,"gcsfs"),Zr.forEach(e),Xr.forEach(e),oo.forEach(e),Ro=f(A),jt=o(A,"TR",{});var lo=l(jt);Ye=o(lo,"TD",{});var ti=l(Ye);Mo=p(ti,"Azure Blob/DataLake"),ti.forEach(e),Uo=f(lo),Re=o(lo,"TD",{});var ei=l(Re);qt=o(ei,"A",{href:!0,rel:!0});var si=l(qt);Ko=p(si,"adlfs"),si.forEach(e),ei.forEach(e),lo.forEach(e),Wo=f(A),St=o(A,"TR",{});var ro=l(St);Me=o(ro,"TD",{});var ai=l(Me);Jo=p(ai,"Dropbox"),ai.forEach(e),Qo=f(ro),Ue=o(ro,"TD",{});var oi=l(Ue);At=o(oi,"A",{href:!0,rel:!0});var li=l(At);Vo=p(li,"dropboxdrivefs"),li.forEach(e),oi.forEach(e),ro.forEach(e),Xo=f(A),Dt=o(A,"TR",{});var io=l(Dt);Ke=o(io,"TD",{});var ri=l(Ke);Zo=p(ri,"Google Drive"),ri.forEach(e),tl=f(io),We=o(io,"TD",{});var ii=l(We);Ct=o(ii,"A",{href:!0,rel:!0});var ni=l(Ct);el=p(ni,"gdrivefs"),ni.forEach(e),ii.forEach(e),io.forEach(e),sl=f(A),Tt=o(A,"TR",{});var no=l(Tt);Je=o(no,"TD",{});var pi=l(Je);al=p(pi,"Oracle Cloud Storage"),pi.forEach(e),ol=f(no),Qe=o(no,"TD",{});var di=l(Qe);Ot=o(di,"A",{href:!0,rel:!0});var fi=l(Ot);ll=p(fi,"ocifs"),fi.forEach(e),di.forEach(e),no.forEach(e),A.forEach(e),eo.forEach(e),Ks=f(t),we=o(t,"P",{});var ui=l(we);rl=p(ui,`This guide will show you how to save and load datasets with any cloud storage.
Here are examples for S3, Google Cloud Storage, Azure Blob Storage, and Oracle Cloud Object Storage.`),ui.forEach(e),Ws=f(t),z=o(t,"H2",{class:!0});var po=l(z);W=o(po,"A",{id:!0,class:!0,href:!0});var ci=l(W);Ve=o(ci,"SPAN",{});var hi=l(Ve);h(Pt.$$.fragment,hi),hi.forEach(e),ci.forEach(e),il=f(po),Xe=o(po,"SPAN",{});var mi=l(Xe);nl=p(mi,"Set up your cloud storage FileSystem"),mi.forEach(e),po.forEach(e),Js=f(t),I=o(t,"H3",{class:!0});var fo=l(I);J=o(fo,"A",{id:!0,class:!0,href:!0});var gi=l(J);Ze=o(gi,"SPAN",{});var _i=l(Ze);h(Lt.$$.fragment,_i),_i.forEach(e),gi.forEach(e),pl=f(fo),ts=o(fo,"SPAN",{});var vi=l(ts);dl=p(vi,"Amazon S3"),vi.forEach(e),fo.forEach(e),Qs=f(t),be=o(t,"OL",{});var $i=l(be);es=o($i,"LI",{});var yi=l(es);fl=p(yi,"Install the S3 FileSystem implementation:"),yi.forEach(e),$i.forEach(e),Vs=f(t),h(zt.$$.fragment,t),Xs=f(t),It=o(t,"OL",{start:!0});var wi=l(It);ss=o(wi,"LI",{});var bi=l(ss);ul=p(bi,"Define your credentials"),bi.forEach(e),wi.forEach(e),Zs=f(t),q=o(t,"P",{});var _t=l(q);cl=p(_t,"To use an anonymous connection, use "),as=o(_t,"CODE",{});var Ei=l(as);hl=p(Ei,"anon=True"),Ei.forEach(e),ml=p(_t,`.
Otherwise, include your `),os=o(_t,"CODE",{});var ki=l(os);gl=p(ki,"aws_access_key_id"),ki.forEach(e),_l=p(_t," and "),ls=o(_t,"CODE",{});var ji=l(ls);vl=p(ji,"aws_secret_access_key"),ji.forEach(e),$l=p(_t," whenever you are interacting with a private S3 bucket."),_t.forEach(e),ta=f(t),h(Ft.$$.fragment,t),ea=f(t),Nt=o(t,"OL",{start:!0});var qi=l(Nt);rs=o(qi,"LI",{});var Si=l(rs);yl=p(Si,"Create your FileSystem instance"),Si.forEach(e),qi.forEach(e),sa=f(t),h(xt.$$.fragment,t),aa=f(t),F=o(t,"H3",{class:!0});var uo=l(F);Q=o(uo,"A",{id:!0,class:!0,href:!0});var Ai=l(Q);is=o(Ai,"SPAN",{});var Di=l(is);h(Ht.$$.fragment,Di),Di.forEach(e),Ai.forEach(e),wl=f(uo),ns=o(uo,"SPAN",{});var Ci=l(ns);bl=p(Ci,"Google Cloud Storage"),Ci.forEach(e),uo.forEach(e),oa=f(t),Ee=o(t,"OL",{});var Ti=l(Ee);ps=o(Ti,"LI",{});var Oi=l(ps);El=p(Oi,"Install the Google Cloud Storage implementation:"),Oi.forEach(e),Ti.forEach(e),la=f(t),h(Bt.$$.fragment,t),ra=f(t),Gt=o(t,"OL",{start:!0});var Pi=l(Gt);ds=o(Pi,"LI",{});var Li=l(ds);kl=p(Li,"Define your credentials"),Li.forEach(e),Pi.forEach(e),ia=f(t),h(Yt.$$.fragment,t),na=f(t),Rt=o(t,"OL",{start:!0});var zi=l(Rt);fs=o(zi,"LI",{});var Ii=l(fs);jl=p(Ii,"Create your FileSystem instance"),Ii.forEach(e),zi.forEach(e),pa=f(t),h(Mt.$$.fragment,t),da=f(t),N=o(t,"H3",{class:!0});var co=l(N);V=o(co,"A",{id:!0,class:!0,href:!0});var Fi=l(V);us=o(Fi,"SPAN",{});var Ni=l(us);h(Ut.$$.fragment,Ni),Ni.forEach(e),Fi.forEach(e),ql=f(co),cs=o(co,"SPAN",{});var xi=l(cs);Sl=p(xi,"Azure Blob Storage"),xi.forEach(e),co.forEach(e),fa=f(t),ke=o(t,"OL",{});var Hi=l(ke);hs=o(Hi,"LI",{});var Bi=l(hs);Al=p(Bi,"Install the Azure Blob Storage implementation:"),Bi.forEach(e),Hi.forEach(e),ua=f(t),h(Kt.$$.fragment,t),ca=f(t),Wt=o(t,"OL",{start:!0});var Gi=l(Wt);ms=o(Gi,"LI",{});var Yi=l(ms);Dl=p(Yi,"Define your credentials"),Yi.forEach(e),Gi.forEach(e),ha=f(t),h(Jt.$$.fragment,t),ma=f(t),Qt=o(t,"OL",{start:!0});var Ri=l(Qt);gs=o(Ri,"LI",{});var Mi=l(gs);Cl=p(Mi,"Create your FileSystem instance"),Mi.forEach(e),Ri.forEach(e),ga=f(t),h(Vt.$$.fragment,t),_a=f(t),x=o(t,"H3",{class:!0});var ho=l(x);X=o(ho,"A",{id:!0,class:!0,href:!0});var Ui=l(X);_s=o(Ui,"SPAN",{});var Ki=l(_s);h(Xt.$$.fragment,Ki),Ki.forEach(e),Ui.forEach(e),Tl=f(ho),vs=o(ho,"SPAN",{});var Wi=l(vs);Ol=p(Wi,"Oracle Cloud Object Storage"),Wi.forEach(e),ho.forEach(e),va=f(t),je=o(t,"OL",{});var Ji=l(je);$s=o(Ji,"LI",{});var Qi=l($s);Pl=p(Qi,"Install the OCI FileSystem implementation:"),Qi.forEach(e),Ji.forEach(e),$a=f(t),h(Zt.$$.fragment,t),ya=f(t),te=o(t,"OL",{start:!0});var Vi=l(te);ys=o(Vi,"LI",{});var Xi=l(ys);Ll=p(Xi,"Define your credentials"),Xi.forEach(e),Vi.forEach(e),wa=f(t),h(ee.$$.fragment,t),ba=f(t),se=o(t,"OL",{start:!0});var Zi=l(se);ws=o(Zi,"LI",{});var tn=l(ws);zl=p(tn,"Create your FileSystem instance"),tn.forEach(e),Zi.forEach(e),Ea=f(t),h(ae.$$.fragment,t),ka=f(t),H=o(t,"H2",{class:!0});var mo=l(H);Z=o(mo,"A",{id:!0,class:!0,href:!0});var en=l(Z);bs=o(en,"SPAN",{});var sn=l(bs);h(oe.$$.fragment,sn),sn.forEach(e),en.forEach(e),Il=f(mo),Es=o(mo,"SPAN",{});var an=l(Es);Fl=p(an,"Load and Save your datasets using your cloud storage FileSystem"),an.forEach(e),mo.forEach(e),ja=f(t),B=o(t,"H3",{class:!0});var go=l(B);tt=o(go,"A",{id:!0,class:!0,href:!0});var on=l(tt);ks=o(on,"SPAN",{});var ln=l(ks);h(le.$$.fragment,ln),ln.forEach(e),on.forEach(e),Nl=f(go),js=o(go,"SPAN",{});var rn=l(js);xl=p(rn,"Download and prepare a dataset into a cloud storage"),rn.forEach(e),go.forEach(e),qa=f(t),S=o(t,"P",{});var vt=l(S);Hl=p(vt,"You can download and prepare a dataset into your cloud storage by specifying a remote "),qs=o(vt,"CODE",{});var nn=l(qs);Bl=p(nn,"output_dir"),nn.forEach(e),Gl=p(vt," in "),Ss=o(vt,"CODE",{});var pn=l(Ss);Yl=p(pn,"download_and_prepare"),pn.forEach(e),Rl=p(vt,`.
Don\u2019t forget to use the previously defined `),As=o(vt,"CODE",{});var dn=l(As);Ml=p(dn,"storage_options"),dn.forEach(e),Ul=p(vt," containing your credentials to write into a private cloud storage."),vt.forEach(e),Sa=f(t),et=o(t,"P",{});var _o=l(et);Kl=p(_o,"The "),Ds=o(_o,"CODE",{});var fn=l(Ds);Wl=p(fn,"download_and_prepare"),fn.forEach(e),Jl=p(_o," method works in two steps:"),_o.forEach(e),Aa=f(t),st=o(t,"OL",{});var vo=l(st);at=o(vo,"LI",{});var Rs=l(at);Ql=p(Rs,"it first downloads the raw data files (if any) in your local cache. You can set your cache directory by passing "),Cs=o(Rs,"CODE",{});var un=l(Cs);Vl=p(un,"cache_dir"),un.forEach(e),Xl=p(Rs," to "),qe=o(Rs,"A",{href:!0});var cn=l(qe);Zl=p(cn,"load_dataset_builder()"),cn.forEach(e),Rs.forEach(e),tr=f(vo),Ts=o(vo,"LI",{});var hn=l(Ts);er=p(hn,"then it generates the dataset in Arrow or Parquet format in your cloud storage by iterating over the raw data files."),hn.forEach(e),vo.forEach(e),Da=f(t),ot=o(t,"P",{});var $o=l(ot);sr=p($o,"Load a dataset builder from the Hugging Face Hub (see "),Se=o($o,"A",{href:!0});var mn=l(Se);ar=p(mn,"how to load from the Hugging Face Hub"),mn.forEach(e),or=p($o,"):"),$o.forEach(e),Ca=f(t),h(re.$$.fragment,t),Ta=f(t),lt=o(t,"P",{});var yo=l(lt);lr=p(yo,"Load a dataset builder using a loading script (see "),Ae=o(yo,"A",{href:!0});var gn=l(Ae);rr=p(gn,"how to load a local loading script"),gn.forEach(e),ir=p(yo,"):"),yo.forEach(e),Oa=f(t),h(ie.$$.fragment,t),Pa=f(t),rt=o(t,"P",{});var wo=l(rt);nr=p(wo,"Use your own data files (see "),De=o(wo,"A",{href:!0});var _n=l(De);pr=p(_n,"how to load local and remote files"),_n.forEach(e),dr=p(wo,"):"),wo.forEach(e),La=f(t),h(ne.$$.fragment,t),za=f(t),it=o(t,"P",{});var bo=l(it);fr=p(bo,"It is highly recommended to save the files as compressed Parquet files to optimize I/O by specifying "),Os=o(bo,"CODE",{});var vn=l(Os);ur=p(vn,'file_format="parquet"'),vn.forEach(e),cr=p(bo,`.
Otherwise the dataset is saved as an uncompressed Arrow file.`),bo.forEach(e),Ia=f(t),nt=o(t,"P",{});var Eo=l(nt);hr=p(Eo,"You can also specify the size of the shards using "),Ps=o(Eo,"CODE",{});var $n=l(Ps);mr=p($n,"max_shard_size"),$n.forEach(e),gr=p(Eo," (default is 500MB):"),Eo.forEach(e),Fa=f(t),h(pe.$$.fragment,t),Na=f(t),G=o(t,"H4",{class:!0});var ko=l(G);pt=o(ko,"A",{id:!0,class:!0,href:!0});var yn=l(pt);Ls=o(yn,"SPAN",{});var wn=l(Ls);h(de.$$.fragment,wn),wn.forEach(e),yn.forEach(e),_r=f(ko),zs=o(ko,"SPAN",{});var bn=l(zs);vr=p(bn,"Dask"),bn.forEach(e),ko.forEach(e),xa=f(t),Ce=o(t,"P",{});var En=l(Ce);$r=p(En,`Dask is a parallel computing library and it has a pandas-like API for working with larger than memory Parquet datasets in parallel.
Dask can use multiple threads or processes on a single machine, or a cluster of machines to process data in parallel.
Dask supports local data but also data from a cloud storage.`),En.forEach(e),Ha=f(t),Te=o(t,"P",{});var kn=l(Te);yr=p(kn,"Therefore you can load a dataset saved as sharded Parquet files in Dask with"),kn.forEach(e),Ba=f(t),h(fe.$$.fragment,t),Ga=f(t),dt=o(t,"P",{});var jo=l(dt);wr=p(jo,"You can find more about dask dataframes in their "),ue=o(jo,"A",{href:!0,rel:!0});var jn=l(ue);br=p(jn,"documentation"),jn.forEach(e),Er=p(jo,"."),jo.forEach(e),Ya=f(t),Y=o(t,"H2",{class:!0});var qo=l(Y);ft=o(qo,"A",{id:!0,class:!0,href:!0});var qn=l(ft);Is=o(qn,"SPAN",{});var Sn=l(Is);h(ce.$$.fragment,Sn),Sn.forEach(e),qn.forEach(e),kr=f(qo),Fs=o(qo,"SPAN",{});var An=l(Fs);jr=p(An,"Saving serialized datasets"),An.forEach(e),qo.forEach(e),Ra=f(t),ut=o(t,"P",{});var So=l(ut);qr=p(So,"After you have processed your dataset, you can save it to your cloud storage with "),Oe=o(So,"A",{href:!0});var Dn=l(Oe);Sr=p(Dn,"Dataset.save_to_disk()"),Dn.forEach(e),Ar=p(So,":"),So.forEach(e),Ma=f(t),h(he.$$.fragment,t),Ua=f(t),h(ct.$$.fragment,t),Ka=f(t),R=o(t,"H2",{class:!0});var Ao=l(R);ht=o(Ao,"A",{id:!0,class:!0,href:!0});var Cn=l(ht);Ns=o(Cn,"SPAN",{});var Tn=l(Ns);h(me.$$.fragment,Tn),Tn.forEach(e),Cn.forEach(e),Dr=f(Ao),xs=o(Ao,"SPAN",{});var On=l(xs);Cr=p(On,"Listing serialized datasets"),On.forEach(e),Ao.forEach(e),Wa=f(t),T=o(t,"P",{});var Le=l(T);Tr=p(Le,"List files from a cloud storage with your FileSystem instance "),Hs=o(Le,"CODE",{});var Pn=l(Hs);Or=p(Pn,"fs"),Pn.forEach(e),Pr=p(Le,", using "),Bs=o(Le,"CODE",{});var Ln=l(Bs);Lr=p(Ln,"fs.ls"),Ln.forEach(e),zr=p(Le,":"),Le.forEach(e),Ja=f(t),h(ge.$$.fragment,t),Qa=f(t),M=o(t,"H3",{class:!0});var Do=l(M);mt=o(Do,"A",{id:!0,class:!0,href:!0});var zn=l(mt);Gs=o(zn,"SPAN",{});var In=l(Gs);h(_e.$$.fragment,In),In.forEach(e),zn.forEach(e),Ir=f(Do),Ys=o(Do,"SPAN",{});var Fn=l(Ys);Fr=p(Fn,"Load serialized datasets"),Fn.forEach(e),Do.forEach(e),Va=f(t),gt=o(t,"P",{});var Co=l(gt);Nr=p(Co,"When you are ready to use your dataset again, reload it with "),Pe=o(Co,"A",{href:!0});var Nn=l(Pe);xr=p(Nn,"Dataset.load_from_disk()"),Nn.forEach(e),Hr=p(Co,":"),Co.forEach(e),Xa=f(t),h(ve.$$.fragment,t),this.h()},h(){u($,"name","hf:doc:metadata"),u($,"content",JSON.stringify(Kn)),u(j,"id","cloud-storage"),u(j,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(j,"href","#cloud-storage"),u(w,"class","relative group"),u(bt,"href","https://s3fs.readthedocs.io/en/latest/"),u(bt,"rel","nofollow"),u(kt,"href","https://gcsfs.readthedocs.io/en/latest/"),u(kt,"rel","nofollow"),u(qt,"href","https://github.com/fsspec/adlfs"),u(qt,"rel","nofollow"),u(At,"href","https://github.com/MarineChap/dropboxdrivefs"),u(At,"rel","nofollow"),u(Ct,"href","https://github.com/intake/gdrivefs"),u(Ct,"rel","nofollow"),u(Ot,"href","https://ocifs.readthedocs.io/en/latest/"),u(Ot,"rel","nofollow"),u(W,"id","set-up-your-cloud-storage-filesystem"),u(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(W,"href","#set-up-your-cloud-storage-filesystem"),u(z,"class","relative group"),u(J,"id","amazon-s3"),u(J,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(J,"href","#amazon-s3"),u(I,"class","relative group"),u(It,"start","2"),u(Nt,"start","3"),u(Q,"id","google-cloud-storage"),u(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Q,"href","#google-cloud-storage"),u(F,"class","relative group"),u(Gt,"start","2"),u(Rt,"start","3"),u(V,"id","azure-blob-storage"),u(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(V,"href","#azure-blob-storage"),u(N,"class","relative group"),u(Wt,"start","2"),u(Qt,"start","3"),u(X,"id","oracle-cloud-object-storage"),u(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(X,"href","#oracle-cloud-object-storage"),u(x,"class","relative group"),u(te,"start","2"),u(se,"start","3"),u(Z,"id","load-and-save-your-datasets-using-your-cloud-storage-filesystem"),u(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(Z,"href","#load-and-save-your-datasets-using-your-cloud-storage-filesystem"),u(H,"class","relative group"),u(tt,"id","download-and-prepare-a-dataset-into-a-cloud-storage"),u(tt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(tt,"href","#download-and-prepare-a-dataset-into-a-cloud-storage"),u(B,"class","relative group"),u(qe,"href","/docs/datasets/v2.11.0/en/package_reference/loading_methods#datasets.load_dataset_builder"),u(Se,"href","./loading#hugging-face-hub"),u(Ae,"href","./loading#local-loading-script"),u(De,"href","./loading#local-and-remote-files"),u(pt,"id","dask"),u(pt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(pt,"href","#dask"),u(G,"class","relative group"),u(ue,"href","https://docs.dask.org/en/stable/dataframe.html"),u(ue,"rel","nofollow"),u(ft,"id","saving-serialized-datasets"),u(ft,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ft,"href","#saving-serialized-datasets"),u(Y,"class","relative group"),u(Oe,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.save_to_disk"),u(ht,"id","listing-serialized-datasets"),u(ht,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(ht,"href","#listing-serialized-datasets"),u(R,"class","relative group"),u(mt,"id","load-serialized-datasets"),u(mt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(mt,"href","#load-serialized-datasets"),u(M,"class","relative group"),u(Pe,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.load_from_disk")},m(t,r){s(document.head,$),i(t,U,r),i(t,w,r),s(w,j),s(j,O),m(E,O,null),s(w,$t),s(w,P),s(P,L),i(t,k,r),i(t,C,r),s(C,ye),s(C,ze),s(ze,To),s(C,Oo),i(t,Us,r),i(t,K,r),s(K,Ie),s(Ie,yt),s(yt,Fe),s(Fe,Po),s(yt,Lo),s(yt,Ne),s(Ne,zo),s(K,Io),s(K,b),s(b,wt),s(wt,xe),s(xe,Fo),s(wt,No),s(wt,He),s(He,bt),s(bt,xo),s(b,Ho),s(b,Et),s(Et,Be),s(Be,Bo),s(Et,Go),s(Et,Ge),s(Ge,kt),s(kt,Yo),s(b,Ro),s(b,jt),s(jt,Ye),s(Ye,Mo),s(jt,Uo),s(jt,Re),s(Re,qt),s(qt,Ko),s(b,Wo),s(b,St),s(St,Me),s(Me,Jo),s(St,Qo),s(St,Ue),s(Ue,At),s(At,Vo),s(b,Xo),s(b,Dt),s(Dt,Ke),s(Ke,Zo),s(Dt,tl),s(Dt,We),s(We,Ct),s(Ct,el),s(b,sl),s(b,Tt),s(Tt,Je),s(Je,al),s(Tt,ol),s(Tt,Qe),s(Qe,Ot),s(Ot,ll),i(t,Ks,r),i(t,we,r),s(we,rl),i(t,Ws,r),i(t,z,r),s(z,W),s(W,Ve),m(Pt,Ve,null),s(z,il),s(z,Xe),s(Xe,nl),i(t,Js,r),i(t,I,r),s(I,J),s(J,Ze),m(Lt,Ze,null),s(I,pl),s(I,ts),s(ts,dl),i(t,Qs,r),i(t,be,r),s(be,es),s(es,fl),i(t,Vs,r),m(zt,t,r),i(t,Xs,r),i(t,It,r),s(It,ss),s(ss,ul),i(t,Zs,r),i(t,q,r),s(q,cl),s(q,as),s(as,hl),s(q,ml),s(q,os),s(os,gl),s(q,_l),s(q,ls),s(ls,vl),s(q,$l),i(t,ta,r),m(Ft,t,r),i(t,ea,r),i(t,Nt,r),s(Nt,rs),s(rs,yl),i(t,sa,r),m(xt,t,r),i(t,aa,r),i(t,F,r),s(F,Q),s(Q,is),m(Ht,is,null),s(F,wl),s(F,ns),s(ns,bl),i(t,oa,r),i(t,Ee,r),s(Ee,ps),s(ps,El),i(t,la,r),m(Bt,t,r),i(t,ra,r),i(t,Gt,r),s(Gt,ds),s(ds,kl),i(t,ia,r),m(Yt,t,r),i(t,na,r),i(t,Rt,r),s(Rt,fs),s(fs,jl),i(t,pa,r),m(Mt,t,r),i(t,da,r),i(t,N,r),s(N,V),s(V,us),m(Ut,us,null),s(N,ql),s(N,cs),s(cs,Sl),i(t,fa,r),i(t,ke,r),s(ke,hs),s(hs,Al),i(t,ua,r),m(Kt,t,r),i(t,ca,r),i(t,Wt,r),s(Wt,ms),s(ms,Dl),i(t,ha,r),m(Jt,t,r),i(t,ma,r),i(t,Qt,r),s(Qt,gs),s(gs,Cl),i(t,ga,r),m(Vt,t,r),i(t,_a,r),i(t,x,r),s(x,X),s(X,_s),m(Xt,_s,null),s(x,Tl),s(x,vs),s(vs,Ol),i(t,va,r),i(t,je,r),s(je,$s),s($s,Pl),i(t,$a,r),m(Zt,t,r),i(t,ya,r),i(t,te,r),s(te,ys),s(ys,Ll),i(t,wa,r),m(ee,t,r),i(t,ba,r),i(t,se,r),s(se,ws),s(ws,zl),i(t,Ea,r),m(ae,t,r),i(t,ka,r),i(t,H,r),s(H,Z),s(Z,bs),m(oe,bs,null),s(H,Il),s(H,Es),s(Es,Fl),i(t,ja,r),i(t,B,r),s(B,tt),s(tt,ks),m(le,ks,null),s(B,Nl),s(B,js),s(js,xl),i(t,qa,r),i(t,S,r),s(S,Hl),s(S,qs),s(qs,Bl),s(S,Gl),s(S,Ss),s(Ss,Yl),s(S,Rl),s(S,As),s(As,Ml),s(S,Ul),i(t,Sa,r),i(t,et,r),s(et,Kl),s(et,Ds),s(Ds,Wl),s(et,Jl),i(t,Aa,r),i(t,st,r),s(st,at),s(at,Ql),s(at,Cs),s(Cs,Vl),s(at,Xl),s(at,qe),s(qe,Zl),s(st,tr),s(st,Ts),s(Ts,er),i(t,Da,r),i(t,ot,r),s(ot,sr),s(ot,Se),s(Se,ar),s(ot,or),i(t,Ca,r),m(re,t,r),i(t,Ta,r),i(t,lt,r),s(lt,lr),s(lt,Ae),s(Ae,rr),s(lt,ir),i(t,Oa,r),m(ie,t,r),i(t,Pa,r),i(t,rt,r),s(rt,nr),s(rt,De),s(De,pr),s(rt,dr),i(t,La,r),m(ne,t,r),i(t,za,r),i(t,it,r),s(it,fr),s(it,Os),s(Os,ur),s(it,cr),i(t,Ia,r),i(t,nt,r),s(nt,hr),s(nt,Ps),s(Ps,mr),s(nt,gr),i(t,Fa,r),m(pe,t,r),i(t,Na,r),i(t,G,r),s(G,pt),s(pt,Ls),m(de,Ls,null),s(G,_r),s(G,zs),s(zs,vr),i(t,xa,r),i(t,Ce,r),s(Ce,$r),i(t,Ha,r),i(t,Te,r),s(Te,yr),i(t,Ba,r),m(fe,t,r),i(t,Ga,r),i(t,dt,r),s(dt,wr),s(dt,ue),s(ue,br),s(dt,Er),i(t,Ya,r),i(t,Y,r),s(Y,ft),s(ft,Is),m(ce,Is,null),s(Y,kr),s(Y,Fs),s(Fs,jr),i(t,Ra,r),i(t,ut,r),s(ut,qr),s(ut,Oe),s(Oe,Sr),s(ut,Ar),i(t,Ma,r),m(he,t,r),i(t,Ua,r),m(ct,t,r),i(t,Ka,r),i(t,R,r),s(R,ht),s(ht,Ns),m(me,Ns,null),s(R,Dr),s(R,xs),s(xs,Cr),i(t,Wa,r),i(t,T,r),s(T,Tr),s(T,Hs),s(Hs,Or),s(T,Pr),s(T,Bs),s(Bs,Lr),s(T,zr),i(t,Ja,r),m(ge,t,r),i(t,Qa,r),i(t,M,r),s(M,mt),s(mt,Gs),m(_e,Gs,null),s(M,Ir),s(M,Ys),s(Ys,Fr),i(t,Va,r),i(t,gt,r),s(gt,Nr),s(gt,Pe),s(Pe,xr),s(gt,Hr),i(t,Xa,r),m(ve,t,r),Za=!0},p(t,[r]){const $e={};r&2&&($e.$$scope={dirty:r,ctx:t}),ct.$set($e)},i(t){Za||(g(E.$$.fragment,t),g(Pt.$$.fragment,t),g(Lt.$$.fragment,t),g(zt.$$.fragment,t),g(Ft.$$.fragment,t),g(xt.$$.fragment,t),g(Ht.$$.fragment,t),g(Bt.$$.fragment,t),g(Yt.$$.fragment,t),g(Mt.$$.fragment,t),g(Ut.$$.fragment,t),g(Kt.$$.fragment,t),g(Jt.$$.fragment,t),g(Vt.$$.fragment,t),g(Xt.$$.fragment,t),g(Zt.$$.fragment,t),g(ee.$$.fragment,t),g(ae.$$.fragment,t),g(oe.$$.fragment,t),g(le.$$.fragment,t),g(re.$$.fragment,t),g(ie.$$.fragment,t),g(ne.$$.fragment,t),g(pe.$$.fragment,t),g(de.$$.fragment,t),g(fe.$$.fragment,t),g(ce.$$.fragment,t),g(he.$$.fragment,t),g(ct.$$.fragment,t),g(me.$$.fragment,t),g(ge.$$.fragment,t),g(_e.$$.fragment,t),g(ve.$$.fragment,t),Za=!0)},o(t){_(E.$$.fragment,t),_(Pt.$$.fragment,t),_(Lt.$$.fragment,t),_(zt.$$.fragment,t),_(Ft.$$.fragment,t),_(xt.$$.fragment,t),_(Ht.$$.fragment,t),_(Bt.$$.fragment,t),_(Yt.$$.fragment,t),_(Mt.$$.fragment,t),_(Ut.$$.fragment,t),_(Kt.$$.fragment,t),_(Jt.$$.fragment,t),_(Vt.$$.fragment,t),_(Xt.$$.fragment,t),_(Zt.$$.fragment,t),_(ee.$$.fragment,t),_(ae.$$.fragment,t),_(oe.$$.fragment,t),_(le.$$.fragment,t),_(re.$$.fragment,t),_(ie.$$.fragment,t),_(ne.$$.fragment,t),_(pe.$$.fragment,t),_(de.$$.fragment,t),_(fe.$$.fragment,t),_(ce.$$.fragment,t),_(he.$$.fragment,t),_(ct.$$.fragment,t),_(me.$$.fragment,t),_(ge.$$.fragment,t),_(_e.$$.fragment,t),_(ve.$$.fragment,t),Za=!1},d(t){e($),t&&e(U),t&&e(w),v(E),t&&e(k),t&&e(C),t&&e(Us),t&&e(K),t&&e(Ks),t&&e(we),t&&e(Ws),t&&e(z),v(Pt),t&&e(Js),t&&e(I),v(Lt),t&&e(Qs),t&&e(be),t&&e(Vs),v(zt,t),t&&e(Xs),t&&e(It),t&&e(Zs),t&&e(q),t&&e(ta),v(Ft,t),t&&e(ea),t&&e(Nt),t&&e(sa),v(xt,t),t&&e(aa),t&&e(F),v(Ht),t&&e(oa),t&&e(Ee),t&&e(la),v(Bt,t),t&&e(ra),t&&e(Gt),t&&e(ia),v(Yt,t),t&&e(na),t&&e(Rt),t&&e(pa),v(Mt,t),t&&e(da),t&&e(N),v(Ut),t&&e(fa),t&&e(ke),t&&e(ua),v(Kt,t),t&&e(ca),t&&e(Wt),t&&e(ha),v(Jt,t),t&&e(ma),t&&e(Qt),t&&e(ga),v(Vt,t),t&&e(_a),t&&e(x),v(Xt),t&&e(va),t&&e(je),t&&e($a),v(Zt,t),t&&e(ya),t&&e(te),t&&e(wa),v(ee,t),t&&e(ba),t&&e(se),t&&e(Ea),v(ae,t),t&&e(ka),t&&e(H),v(oe),t&&e(ja),t&&e(B),v(le),t&&e(qa),t&&e(S),t&&e(Sa),t&&e(et),t&&e(Aa),t&&e(st),t&&e(Da),t&&e(ot),t&&e(Ca),v(re,t),t&&e(Ta),t&&e(lt),t&&e(Oa),v(ie,t),t&&e(Pa),t&&e(rt),t&&e(La),v(ne,t),t&&e(za),t&&e(it),t&&e(Ia),t&&e(nt),t&&e(Fa),v(pe,t),t&&e(Na),t&&e(G),v(de),t&&e(xa),t&&e(Ce),t&&e(Ha),t&&e(Te),t&&e(Ba),v(fe,t),t&&e(Ga),t&&e(dt),t&&e(Ya),t&&e(Y),v(ce),t&&e(Ra),t&&e(ut),t&&e(Ma),v(he,t),t&&e(Ua),v(ct,t),t&&e(Ka),t&&e(R),v(me),t&&e(Wa),t&&e(T),t&&e(Ja),v(ge,t),t&&e(Qa),t&&e(M),v(_e),t&&e(Va),t&&e(gt),t&&e(Xa),v(ve,t)}}}const Kn={local:"cloud-storage",sections:[{local:"set-up-your-cloud-storage-filesystem",sections:[{local:"amazon-s3",title:"Amazon S3"},{local:"google-cloud-storage",title:"Google Cloud Storage"},{local:"azure-blob-storage",title:"Azure Blob Storage"},{local:"oracle-cloud-object-storage",title:"Oracle Cloud Object Storage"}],title:"Set up your cloud storage FileSystem"},{local:"load-and-save-your-datasets-using-your-cloud-storage-filesystem",sections:[{local:"download-and-prepare-a-dataset-into-a-cloud-storage",sections:[{local:"dask",title:"Dask"}],title:"Download and prepare a dataset into a cloud storage"}],title:"Load and Save your datasets using your cloud storage FileSystem"},{local:"saving-serialized-datasets",title:"Saving serialized datasets"},{local:"listing-serialized-datasets",sections:[{local:"load-serialized-datasets",title:"Load serialized datasets"}],title:"Listing serialized datasets"}],title:"Cloud storage"};function Wn(Ms){return Yn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Zn extends xn{constructor($){super();Hn(this,$,Wn,Un,Bn,{})}}export{Zn as default,Kn as metadata};
