import{s as hs,o as $s,n as Ts}from"../chunks/scheduler.bdbef820.js";import{S as bs,i as Js,g as c,s as l,r as o,A as js,h as u,f as s,c as n,j as Ms,u as i,x as g,k as $,y as ws,a,v as p,d as r,t as d,w as m}from"../chunks/index.c0aea24a.js";import{T as _s}from"../chunks/Tip.31005f7d.js";import{C as f}from"../chunks/CodeBlock.6ccca92e.js";import{H as M,E as vs}from"../chunks/EditOnGithub.725ee0c1.js";function ks(It){let y,Z='Remember to define your credentials in your <a href="#set-up-your-cloud-storage-filesystem">FileSystem instance</a> <code>fs</code> whenever you are interacting with a private cloud storage.';return{c(){y=c("p"),y.innerHTML=Z},l(h){y=u(h,"P",{"data-svelte-h":!0}),g(y)!=="svelte-f0wftp"&&(y.innerHTML=Z)},m(h,Zt){a(h,y,Zt)},p:Ts,d(h){h&&s(y)}}}function Us(It){let y,Z,h,Zt,z,Gt,I,Le=`🤗 Datasets supports access to cloud storage providers through a <code>fsspec</code> FileSystem implementations.
You can save and load datasets from any cloud storage in a Pythonic way.
Take a look at the following table for some example of supported cloud storage providers:`,Rt,G,Ye='<thead><tr><th>Storage provider</th> <th>Filesystem implementation</th></tr></thead> <tbody><tr><td>Amazon S3</td> <td><a href="https://s3fs.readthedocs.io/en/latest/" rel="nofollow">s3fs</a></td></tr> <tr><td>Google Cloud Storage</td> <td><a href="https://gcsfs.readthedocs.io/en/latest/" rel="nofollow">gcsfs</a></td></tr> <tr><td>Azure Blob/DataLake</td> <td><a href="https://github.com/fsspec/adlfs" rel="nofollow">adlfs</a></td></tr> <tr><td>Dropbox</td> <td><a href="https://github.com/MarineChap/dropboxdrivefs" rel="nofollow">dropboxdrivefs</a></td></tr> <tr><td>Google Drive</td> <td><a href="https://github.com/intake/gdrivefs" rel="nofollow">gdrivefs</a></td></tr> <tr><td>Oracle Cloud Storage</td> <td><a href="https://ocifs.readthedocs.io/en/latest/" rel="nofollow">ocifs</a></td></tr></tbody>',Wt,R,Be=`This guide will show you how to save and load datasets with any cloud storage.
Here are examples for S3, Google Cloud Storage, Azure Blob Storage, and Oracle Cloud Object Storage.`,Ct,W,Ht,C,Xt,H,Ne="<li>Install the S3 FileSystem implementation:</li>",qt,X,Ft,T,Se="<li>Define your credentials</li>",Vt,q,xe=`To use an anonymous connection, use <code>anon=True</code>.
Otherwise, include your <code>aws_access_key_id</code> and <code>aws_secret_access_key</code> whenever you are interacting with a private S3 bucket.`,Lt,F,Yt,b,Qe="<li>Create your FileSystem instance</li>",Bt,V,Nt,L,St,Y,Ee="<li>Install the Google Cloud Storage implementation:</li>",xt,B,Qt,J,Ae="<li>Define your credentials</li>",Et,N,At,j,De="<li>Create your FileSystem instance</li>",Dt,S,Ot,x,Pt,Q,Oe="<li>Install the Azure Blob Storage implementation:</li>",Kt,E,te,w,Pe="<li>Define your credentials</li>",ee,A,se,_,Ke="<li>Create your FileSystem instance</li>",ae,D,le,O,ne,P,ts="<li>Install the OCI FileSystem implementation:</li>",oe,K,ie,v,es="<li>Define your credentials</li>",pe,tt,re,k,ss="<li>Create your FileSystem instance</li>",de,et,me,st,ce,at,ue,lt,as=`You can download and prepare a dataset into your cloud storage by specifying a remote <code>output_dir</code> in <code>download_and_prepare</code>.
Don’t forget to use the previously defined <code>storage_options</code> containing your credentials to write into a private cloud storage.`,ge,nt,ls="The <code>download_and_prepare</code> method works in two steps:",fe,ot,ns='<li>it first downloads the raw data files (if any) in your local cache. You can set your cache directory by passing <code>cache_dir</code> to <a href="/docs/datasets/v3.0.0/en/package_reference/loading_methods#datasets.load_dataset_builder">load_dataset_builder()</a></li> <li>then it generates the dataset in Arrow or Parquet format in your cloud storage by iterating over the raw data files.</li>',ye,it,os='Load a dataset builder from the Hugging Face Hub (see <a href="./loading#hugging-face-hub">how to load from the Hugging Face Hub</a>):',Me,pt,he,rt,is='Use your own data files (see <a href="./loading#local-and-remote-files">how to load local and remote files</a>):',$e,dt,Te,mt,ps=`It is highly recommended to save the files as compressed Parquet files to optimize I/O by specifying <code>file_format=&quot;parquet&quot;</code>.
Otherwise the dataset is saved as an uncompressed Arrow file.`,be,ct,rs="You can also specify the size of the shards using <code>max_shard_size</code> (default is 500MB):",Je,ut,je,gt,we,ft,ds=`Dask is a parallel computing library and it has a pandas-like API for working with larger than memory Parquet datasets in parallel.
Dask can use multiple threads or processes on a single machine, or a cluster of machines to process data in parallel.
Dask supports local data but also data from a cloud storage.`,_e,yt,ms="Therefore you can load a dataset saved as sharded Parquet files in Dask with",ve,Mt,ke,ht,cs='You can find more about dask dataframes in their <a href="https://docs.dask.org/en/stable/dataframe.html" rel="nofollow">documentation</a>.',Ue,$t,Ze,Tt,us='After you have processed your dataset, you can save it to your cloud storage with <a href="/docs/datasets/v3.0.0/en/package_reference/main_classes#datasets.Dataset.save_to_disk">Dataset.save_to_disk()</a>:',ze,bt,Ie,U,Ge,Jt,Re,jt,gs="List files from a cloud storage with your FileSystem instance <code>fs</code>, using <code>fs.ls</code>:",We,wt,Ce,_t,He,vt,fs='When you are ready to use your dataset again, reload it with <a href="/docs/datasets/v3.0.0/en/package_reference/main_classes#datasets.Dataset.load_from_disk">Dataset.load_from_disk()</a>:',Xe,kt,qe,Ut,Fe,zt,Ve;return z=new M({props:{title:"Cloud storage",local:"cloud-storage",headingTag:"h1"}}),W=new M({props:{title:"Set up your cloud storage FileSystem",local:"set-up-your-cloud-storage-filesystem",headingTag:"h2"}}),C=new M({props:{title:"Amazon S3",local:"amazon-s3",headingTag:"h3"}}),X=new f({props:{code:"cGlwJTIwaW5zdGFsbCUyMHMzZnM=",highlighted:'<span class="hljs-meta prompt_">&gt;&gt;&gt;</span> <span class="language-python">pip install s3fs</span>',wrap:!1}}),F=new f({props:{code:"c3RvcmFnZV9vcHRpb25zJTIwJTNEJTIwJTdCJTIyYW5vbiUyMiUzQSUyMFRydWUlN0QlMjAlMjAlMjMlMjBmb3IlMjBhbm9ueW1vdXMlMjBjb25uZWN0aW9uJTBBc3RvcmFnZV9vcHRpb25zJTIwJTNEJTIwJTdCJTIya2V5JTIyJTNBJTIwYXdzX2FjY2Vzc19rZXlfaWQlMkMlMjAlMjJzZWNyZXQlMjIlM0ElMjBhd3Nfc2VjcmV0X2FjY2Vzc19rZXklN0QlMjAlMjAlMjMlMjBmb3IlMjBwcml2YXRlJTIwYnVja2V0cyUwQWltcG9ydCUyMGFpb2JvdG9jb3JlLnNlc3Npb24lMEFzM19zZXNzaW9uJTIwJTNEJTIwYWlvYm90b2NvcmUuc2Vzc2lvbi5BaW9TZXNzaW9uKHByb2ZpbGUlM0QlMjJteV9wcm9maWxlX25hbWUlMjIpJTBBc3RvcmFnZV9vcHRpb25zJTIwJTNEJTIwJTdCJTIyc2Vzc2lvbiUyMiUzQSUyMHMzX3Nlc3Npb24lN0Q=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;anon&quot;</span>: <span class="hljs-literal">True</span>}  <span class="hljs-comment"># for anonymous connection</span>
<span class="hljs-comment"># or use your credentials</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;key&quot;</span>: aws_access_key_id, <span class="hljs-string">&quot;secret&quot;</span>: aws_secret_access_key}  <span class="hljs-comment"># for private buckets</span>
<span class="hljs-comment"># or use a botocore session</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> aiobotocore.session
<span class="hljs-meta">&gt;&gt;&gt; </span>s3_session = aiobotocore.session.AioSession(profile=<span class="hljs-string">&quot;my_profile_name&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;session&quot;</span>: s3_session}`,wrap:!1}}),V=new f({props:{code:"aW1wb3J0JTIwczNmcyUwQWZzJTIwJTNEJTIwczNmcy5TM0ZpbGVTeXN0ZW0oKipzdG9yYWdlX29wdGlvbnMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> s3fs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = s3fs.S3FileSystem(**storage_options)`,wrap:!1}}),L=new M({props:{title:"Google Cloud Storage",local:"google-cloud-storage",headingTag:"h3"}}),B=new f({props:{code:"Y29uZGElMjBpbnN0YWxsJTIwLWMlMjBjb25kYS1mb3JnZSUyMGdjc2ZzJTBBcGlwJTIwaW5zdGFsbCUyMGdjc2Zz",highlighted:`<span class="hljs-meta prompt_">&gt;&gt;&gt;</span> <span class="language-python">conda install -c conda-forge gcsfs</span>
# or install with pip
<span class="hljs-meta prompt_">&gt;&gt;&gt;</span> <span class="language-python">pip install gcsfs</span>`,wrap:!1}}),N=new f({props:{code:"c3RvcmFnZV9vcHRpb25zJTNEJTdCJTIydG9rZW4lMjIlM0ElMjAlMjJhbm9uJTIyJTdEJTIwJTIwJTIzJTIwZm9yJTIwYW5vbnltb3VzJTIwY29ubmVjdGlvbiUwQXN0b3JhZ2Vfb3B0aW9ucyUzRCU3QiUyMnByb2plY3QlMjIlM0ElMjAlMjJteS1nb29nbGUtcHJvamVjdCUyMiU3RCUwQXN0b3JhZ2Vfb3B0aW9ucyUzRCU3QiUyMnByb2plY3QlMjIlM0ElMjAlMjJteS1nb29nbGUtcHJvamVjdCUyMiUyQyUyMCUyMnRva2VuJTIyJTNBJTIwVE9LRU4lN0Q=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;token&quot;</span>: <span class="hljs-string">&quot;anon&quot;</span>}  <span class="hljs-comment"># for anonymous connection</span>
<span class="hljs-comment"># or use your credentials of your default gcloud credentials or from the google metadata service</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;project&quot;</span>: <span class="hljs-string">&quot;my-google-project&quot;</span>}
<span class="hljs-comment"># or use your credentials from elsewhere, see the documentation at https://gcsfs.readthedocs.io/</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;project&quot;</span>: <span class="hljs-string">&quot;my-google-project&quot;</span>, <span class="hljs-string">&quot;token&quot;</span>: TOKEN}`,wrap:!1}}),S=new f({props:{code:"aW1wb3J0JTIwZ2NzZnMlMEFmcyUyMCUzRCUyMGdjc2ZzLkdDU0ZpbGVTeXN0ZW0oKipzdG9yYWdlX29wdGlvbnMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> gcsfs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = gcsfs.GCSFileSystem(**storage_options)`,wrap:!1}}),x=new M({props:{title:"Azure Blob Storage",local:"azure-blob-storage",headingTag:"h3"}}),E=new f({props:{code:"Y29uZGElMjBpbnN0YWxsJTIwLWMlMjBjb25kYS1mb3JnZSUyMGFkbGZzJTBBcGlwJTIwaW5zdGFsbCUyMGFkbGZz",highlighted:`<span class="hljs-meta prompt_">&gt;&gt;&gt;</span> <span class="language-python">conda install -c conda-forge adlfs</span>
# or install with pip
<span class="hljs-meta prompt_">&gt;&gt;&gt;</span> <span class="language-python">pip install adlfs</span>`,wrap:!1}}),A=new f({props:{code:"c3RvcmFnZV9vcHRpb25zJTIwJTNEJTIwJTdCJTIyYW5vbiUyMiUzQSUyMFRydWUlN0QlMjAlMjAlMjMlMjBmb3IlMjBhbm9ueW1vdXMlMjBjb25uZWN0aW9uJTBBc3RvcmFnZV9vcHRpb25zJTIwJTNEJTIwJTdCJTIyYWNjb3VudF9uYW1lJTIyJTNBJTIwQUNDT1VOVF9OQU1FJTJDJTIwJTIyYWNjb3VudF9rZXklMjIlM0ElMjBBQ0NPVU5UX0tFWSU3RCUyMCUyMCUyMyUyMGdlbiUyMDIlMjBmaWxlc3lzdGVtJTBBc3RvcmFnZV9vcHRpb25zJTNEJTdCJTIydGVuYW50X2lkJTIyJTNBJTIwVEVOQU5UX0lEJTJDJTIwJTIyY2xpZW50X2lkJTIyJTNBJTIwQ0xJRU5UX0lEJTJDJTIwJTIyY2xpZW50X3NlY3JldCUyMiUzQSUyMENMSUVOVF9TRUNSRVQlN0Q=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;anon&quot;</span>: <span class="hljs-literal">True</span>}  <span class="hljs-comment"># for anonymous connection</span>
<span class="hljs-comment"># or use your credentials</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;account_name&quot;</span>: ACCOUNT_NAME, <span class="hljs-string">&quot;account_key&quot;</span>: ACCOUNT_KEY}  <span class="hljs-comment"># gen 2 filesystem</span>
<span class="hljs-comment"># or use your credentials with the gen 1 filesystem</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options={<span class="hljs-string">&quot;tenant_id&quot;</span>: TENANT_ID, <span class="hljs-string">&quot;client_id&quot;</span>: CLIENT_ID, <span class="hljs-string">&quot;client_secret&quot;</span>: CLIENT_SECRET}`,wrap:!1}}),D=new f({props:{code:"aW1wb3J0JTIwYWRsZnMlMEFmcyUyMCUzRCUyMGFkbGZzLkF6dXJlQmxvYkZpbGVTeXN0ZW0oKipzdG9yYWdlX29wdGlvbnMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> adlfs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = adlfs.AzureBlobFileSystem(**storage_options)`,wrap:!1}}),O=new M({props:{title:"Oracle Cloud Object Storage",local:"oracle-cloud-object-storage",headingTag:"h3"}}),K=new f({props:{code:"cGlwJTIwaW5zdGFsbCUyMG9jaWZz",highlighted:'<span class="hljs-meta prompt_">&gt;&gt;&gt;</span> <span class="language-python">pip install ocifs</span>',wrap:!1}}),tt=new f({props:{code:"c3RvcmFnZV9vcHRpb25zJTIwJTNEJTIwJTdCJTIyY29uZmlnJTIyJTNBJTIwJTIyfiUyRi5vY2klMkZjb25maWclMjIlMkMlMjAlMjJyZWdpb24lMjIlM0ElMjAlMjJ1cy1hc2hidXJuLTElMjIlN0QlMjA=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>storage_options = {<span class="hljs-string">&quot;config&quot;</span>: <span class="hljs-string">&quot;~/.oci/config&quot;</span>, <span class="hljs-string">&quot;region&quot;</span>: <span class="hljs-string">&quot;us-ashburn-1&quot;</span>} ',wrap:!1}}),et=new f({props:{code:"aW1wb3J0JTIwb2NpZnMlMEFmcyUyMCUzRCUyMG9jaWZzLk9DSUZpbGVTeXN0ZW0oKipzdG9yYWdlX29wdGlvbnMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> ocifs
<span class="hljs-meta">&gt;&gt;&gt; </span>fs = ocifs.OCIFileSystem(**storage_options)`,wrap:!1}}),st=new M({props:{title:"Load and Save your datasets using your cloud storage FileSystem",local:"load-and-save-your-datasets-using-your-cloud-storage-filesystem",headingTag:"h2"}}),at=new M({props:{title:"Download and prepare a dataset into a cloud storage",local:"download-and-prepare-a-dataset-into-a-cloud-storage",headingTag:"h3"}}),pt=new f({props:{code:"b3V0cHV0X2RpciUyMCUzRCUyMCUyMnMzJTNBJTJGJTJGbXktYnVja2V0JTJGaW1kYiUyMiUwQWJ1aWxkZXIlMjAlM0QlMjBsb2FkX2RhdGFzZXRfYnVpbGRlciglMjJpbWRiJTIyKSUwQWJ1aWxkZXIuZG93bmxvYWRfYW5kX3ByZXBhcmUob3V0cHV0X2RpciUyQyUyMHN0b3JhZ2Vfb3B0aW9ucyUzRHN0b3JhZ2Vfb3B0aW9ucyUyQyUyMGZpbGVfZm9ybWF0JTNEJTIycGFycXVldCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>output_dir = <span class="hljs-string">&quot;s3://my-bucket/imdb&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>builder = load_dataset_builder(<span class="hljs-string">&quot;imdb&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>)`,wrap:!1}}),dt=new f({props:{code:"ZGF0YV9maWxlcyUyMCUzRCUyMCU3QiUyMnRyYWluJTIyJTNBJTIwJTVCJTIycGF0aCUyRnRvJTJGdHJhaW4uY3N2JTIyJTVEJTdEJTBBb3V0cHV0X2RpciUyMCUzRCUyMCUyMnMzJTNBJTJGJTJGbXktYnVja2V0JTJGaW1kYiUyMiUwQWJ1aWxkZXIlMjAlM0QlMjBsb2FkX2RhdGFzZXRfYnVpbGRlciglMjJjc3YlMjIlMkMlMjBkYXRhX2ZpbGVzJTNEZGF0YV9maWxlcyklMEFidWlsZGVyLmRvd25sb2FkX2FuZF9wcmVwYXJlKG91dHB1dF9kaXIlMkMlMjBzdG9yYWdlX29wdGlvbnMlM0RzdG9yYWdlX29wdGlvbnMlMkMlMjBmaWxlX2Zvcm1hdCUzRCUyMnBhcnF1ZXQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>data_files = {<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;path/to/train.csv&quot;</span>]}
<span class="hljs-meta">&gt;&gt;&gt; </span>output_dir = <span class="hljs-string">&quot;s3://my-bucket/imdb&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>builder = load_dataset_builder(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files)
<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>)`,wrap:!1}}),ut=new f({props:{code:"YnVpbGRlci5kb3dubG9hZF9hbmRfcHJlcGFyZShvdXRwdXRfZGlyJTJDJTIwc3RvcmFnZV9vcHRpb25zJTNEc3RvcmFnZV9vcHRpb25zJTJDJTIwZmlsZV9mb3JtYXQlM0QlMjJwYXJxdWV0JTIyJTJDJTIwbWF4X3NoYXJkX3NpemUlM0QlMjIxR0IlMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>builder.download_and_prepare(output_dir, storage_options=storage_options, file_format=<span class="hljs-string">&quot;parquet&quot;</span>, max_shard_size=<span class="hljs-string">&quot;1GB&quot;</span>)',wrap:!1}}),gt=new M({props:{title:"Dask",local:"dask",headingTag:"h4"}}),Mt=new f({props:{code:"aW1wb3J0JTIwZGFzay5kYXRhZnJhbWUlMjBhcyUyMGRkJTBBJTBBZGYlMjAlM0QlMjBkZC5yZWFkX3BhcnF1ZXQob3V0cHV0X2RpciUyQyUyMHN0b3JhZ2Vfb3B0aW9ucyUzRHN0b3JhZ2Vfb3B0aW9ucyklMEElMEElMjMlMjBvciUyMGlmJTIweW91ciUyMGRhdGFzZXQlMjBpcyUyMHNwbGl0JTIwaW50byUyMHRyYWluJTJGdmFsaWQlMkZ0ZXN0JTBBZGZfdHJhaW4lMjAlM0QlMjBkZC5yZWFkX3BhcnF1ZXQob3V0cHV0X2RpciUyMCUyQiUyMGYlMjIlMkYlN0JidWlsZGVyLm5hbWUlN0QtdHJhaW4tKi5wYXJxdWV0JTIyJTJDJTIwc3RvcmFnZV9vcHRpb25zJTNEc3RvcmFnZV9vcHRpb25zKSUwQWRmX3ZhbGlkJTIwJTNEJTIwZGQucmVhZF9wYXJxdWV0KG91dHB1dF9kaXIlMjAlMkIlMjBmJTIyJTJGJTdCYnVpbGRlci5uYW1lJTdELXZhbGlkYXRpb24tKi5wYXJxdWV0JTIyJTJDJTIwc3RvcmFnZV9vcHRpb25zJTNEc3RvcmFnZV9vcHRpb25zKSUwQWRmX3Rlc3QlMjAlM0QlMjBkZC5yZWFkX3BhcnF1ZXQob3V0cHV0X2RpciUyMCUyQiUyMGYlMjIlMkYlN0JidWlsZGVyLm5hbWUlN0QtdGVzdC0qLnBhcnF1ZXQlMjIlMkMlMjBzdG9yYWdlX29wdGlvbnMlM0RzdG9yYWdlX29wdGlvbnMp",highlighted:`<span class="hljs-keyword">import</span> dask.dataframe <span class="hljs-keyword">as</span> dd

df = dd.read_parquet(output_dir, storage_options=storage_options)

<span class="hljs-comment"># or if your dataset is split into train/valid/test</span>
df_train = dd.read_parquet(output_dir + <span class="hljs-string">f&quot;/<span class="hljs-subst">{builder.name}</span>-train-*.parquet&quot;</span>, storage_options=storage_options)
df_valid = dd.read_parquet(output_dir + <span class="hljs-string">f&quot;/<span class="hljs-subst">{builder.name}</span>-validation-*.parquet&quot;</span>, storage_options=storage_options)
df_test = dd.read_parquet(output_dir + <span class="hljs-string">f&quot;/<span class="hljs-subst">{builder.name}</span>-test-*.parquet&quot;</span>, storage_options=storage_options)`,wrap:!1}}),$t=new M({props:{title:"Saving serialized datasets",local:"saving-serialized-datasets",headingTag:"h2"}}),bt=new f({props:{code:"ZW5jb2RlZF9kYXRhc2V0LnNhdmVfdG9fZGlzayglMjJzMyUzQSUyRiUyRm15LXByaXZhdGUtZGF0YXNldHMlMkZpbWRiJTJGdHJhaW4lMjIlMkMlMjBzdG9yYWdlX29wdGlvbnMlM0RzdG9yYWdlX29wdGlvbnMpJTBBZW5jb2RlZF9kYXRhc2V0LnNhdmVfdG9fZGlzayglMjJnY3MlM0ElMkYlMkZteS1wcml2YXRlLWRhdGFzZXRzJTJGaW1kYiUyRnRyYWluJTIyJTJDJTIwc3RvcmFnZV9vcHRpb25zJTNEc3RvcmFnZV9vcHRpb25zKSUwQWVuY29kZWRfZGF0YXNldC5zYXZlX3RvX2Rpc2soJTIyYWRsJTNBJTJGJTJGbXktcHJpdmF0ZS1kYXRhc2V0cyUyRmltZGIlMkZ0cmFpbiUyMiUyQyUyMHN0b3JhZ2Vfb3B0aW9ucyUzRHN0b3JhZ2Vfb3B0aW9ucyk=",highlighted:`<span class="hljs-comment"># saves encoded_dataset to amazon s3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_dataset.save_to_disk(<span class="hljs-string">&quot;s3://my-private-datasets/imdb/train&quot;</span>, storage_options=storage_options)
<span class="hljs-comment"># saves encoded_dataset to google cloud storage</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_dataset.save_to_disk(<span class="hljs-string">&quot;gcs://my-private-datasets/imdb/train&quot;</span>, storage_options=storage_options)
<span class="hljs-comment"># saves encoded_dataset to microsoft azure blob/datalake</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_dataset.save_to_disk(<span class="hljs-string">&quot;adl://my-private-datasets/imdb/train&quot;</span>, storage_options=storage_options)`,wrap:!1}}),U=new _s({props:{$$slots:{default:[ks]},$$scope:{ctx:It}}}),Jt=new M({props:{title:"Listing serialized datasets",local:"listing-serialized-datasets",headingTag:"h2"}}),wt=new f({props:{code:"ZnMubHMoJTIybXktcHJpdmF0ZS1kYXRhc2V0cyUyRmltZGIlMkZ0cmFpbiUyMiUyQyUyMGRldGFpbCUzREZhbHNlKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>fs.ls(<span class="hljs-string">&quot;my-private-datasets/imdb/train&quot;</span>, detail=<span class="hljs-literal">False</span>)
[<span class="hljs-string">&quot;dataset_info.json.json&quot;</span>,<span class="hljs-string">&quot;dataset.arrow&quot;</span>,<span class="hljs-string">&quot;state.json&quot;</span>]`,wrap:!1}}),_t=new M({props:{title:"Load serialized datasets",local:"load-serialized-datasets",headingTag:"h3"}}),kt=new f({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9mcm9tX2Rpc2slMEFkYXRhc2V0JTIwJTNEJTIwbG9hZF9mcm9tX2Rpc2soJTIyczMlM0ElMkYlMkZhLXB1YmxpYy1kYXRhc2V0cyUyRmltZGIlMkZ0cmFpbiUyMiUyQyUyMHN0b3JhZ2Vfb3B0aW9ucyUzRHN0b3JhZ2Vfb3B0aW9ucyklMjAlMjAlMEFwcmludChsZW4oZGF0YXNldCkp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_from_disk
<span class="hljs-comment"># load encoded_dataset from cloud storage</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_from_disk(<span class="hljs-string">&quot;s3://a-public-datasets/imdb/train&quot;</span>, storage_options=storage_options)  
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(dataset))
<span class="hljs-number">25000</span>`,wrap:!1}}),Ut=new vs({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/filesystems.mdx"}}),{c(){y=c("meta"),Z=l(),h=c("p"),Zt=l(),o(z.$$.fragment),Gt=l(),I=c("p"),I.innerHTML=Le,Rt=l(),G=c("table"),G.innerHTML=Ye,Wt=l(),R=c("p"),R.textContent=Be,Ct=l(),o(W.$$.fragment),Ht=l(),o(C.$$.fragment),Xt=l(),H=c("ol"),H.innerHTML=Ne,qt=l(),o(X.$$.fragment),Ft=l(),T=c("ol"),T.innerHTML=Se,Vt=l(),q=c("p"),q.innerHTML=xe,Lt=l(),o(F.$$.fragment),Yt=l(),b=c("ol"),b.innerHTML=Qe,Bt=l(),o(V.$$.fragment),Nt=l(),o(L.$$.fragment),St=l(),Y=c("ol"),Y.innerHTML=Ee,xt=l(),o(B.$$.fragment),Qt=l(),J=c("ol"),J.innerHTML=Ae,Et=l(),o(N.$$.fragment),At=l(),j=c("ol"),j.innerHTML=De,Dt=l(),o(S.$$.fragment),Ot=l(),o(x.$$.fragment),Pt=l(),Q=c("ol"),Q.innerHTML=Oe,Kt=l(),o(E.$$.fragment),te=l(),w=c("ol"),w.innerHTML=Pe,ee=l(),o(A.$$.fragment),se=l(),_=c("ol"),_.innerHTML=Ke,ae=l(),o(D.$$.fragment),le=l(),o(O.$$.fragment),ne=l(),P=c("ol"),P.innerHTML=ts,oe=l(),o(K.$$.fragment),ie=l(),v=c("ol"),v.innerHTML=es,pe=l(),o(tt.$$.fragment),re=l(),k=c("ol"),k.innerHTML=ss,de=l(),o(et.$$.fragment),me=l(),o(st.$$.fragment),ce=l(),o(at.$$.fragment),ue=l(),lt=c("p"),lt.innerHTML=as,ge=l(),nt=c("p"),nt.innerHTML=ls,fe=l(),ot=c("ol"),ot.innerHTML=ns,ye=l(),it=c("p"),it.innerHTML=os,Me=l(),o(pt.$$.fragment),he=l(),rt=c("p"),rt.innerHTML=is,$e=l(),o(dt.$$.fragment),Te=l(),mt=c("p"),mt.innerHTML=ps,be=l(),ct=c("p"),ct.innerHTML=rs,Je=l(),o(ut.$$.fragment),je=l(),o(gt.$$.fragment),we=l(),ft=c("p"),ft.textContent=ds,_e=l(),yt=c("p"),yt.textContent=ms,ve=l(),o(Mt.$$.fragment),ke=l(),ht=c("p"),ht.innerHTML=cs,Ue=l(),o($t.$$.fragment),Ze=l(),Tt=c("p"),Tt.innerHTML=us,ze=l(),o(bt.$$.fragment),Ie=l(),o(U.$$.fragment),Ge=l(),o(Jt.$$.fragment),Re=l(),jt=c("p"),jt.innerHTML=gs,We=l(),o(wt.$$.fragment),Ce=l(),o(_t.$$.fragment),He=l(),vt=c("p"),vt.innerHTML=fs,Xe=l(),o(kt.$$.fragment),qe=l(),o(Ut.$$.fragment),Fe=l(),zt=c("p"),this.h()},l(t){const e=js("svelte-u9bgzb",document.head);y=u(e,"META",{name:!0,content:!0}),e.forEach(s),Z=n(t),h=u(t,"P",{}),Ms(h).forEach(s),Zt=n(t),i(z.$$.fragment,t),Gt=n(t),I=u(t,"P",{"data-svelte-h":!0}),g(I)!=="svelte-1xcb7pv"&&(I.innerHTML=Le),Rt=n(t),G=u(t,"TABLE",{"data-svelte-h":!0}),g(G)!=="svelte-14du81h"&&(G.innerHTML=Ye),Wt=n(t),R=u(t,"P",{"data-svelte-h":!0}),g(R)!=="svelte-jhnqvu"&&(R.textContent=Be),Ct=n(t),i(W.$$.fragment,t),Ht=n(t),i(C.$$.fragment,t),Xt=n(t),H=u(t,"OL",{"data-svelte-h":!0}),g(H)!=="svelte-16p50yk"&&(H.innerHTML=Ne),qt=n(t),i(X.$$.fragment,t),Ft=n(t),T=u(t,"OL",{start:!0,"data-svelte-h":!0}),g(T)!=="svelte-172a894"&&(T.innerHTML=Se),Vt=n(t),q=u(t,"P",{"data-svelte-h":!0}),g(q)!=="svelte-1m0nrhz"&&(q.innerHTML=xe),Lt=n(t),i(F.$$.fragment,t),Yt=n(t),b=u(t,"OL",{start:!0,"data-svelte-h":!0}),g(b)!=="svelte-zc3ks6"&&(b.innerHTML=Qe),Bt=n(t),i(V.$$.fragment,t),Nt=n(t),i(L.$$.fragment,t),St=n(t),Y=u(t,"OL",{"data-svelte-h":!0}),g(Y)!=="svelte-tzsteu"&&(Y.innerHTML=Ee),xt=n(t),i(B.$$.fragment,t),Qt=n(t),J=u(t,"OL",{start:!0,"data-svelte-h":!0}),g(J)!=="svelte-172a894"&&(J.innerHTML=Ae),Et=n(t),i(N.$$.fragment,t),At=n(t),j=u(t,"OL",{start:!0,"data-svelte-h":!0}),g(j)!=="svelte-zc3ks6"&&(j.innerHTML=De),Dt=n(t),i(S.$$.fragment,t),Ot=n(t),i(x.$$.fragment,t),Pt=n(t),Q=u(t,"OL",{"data-svelte-h":!0}),g(Q)!=="svelte-zykrn6"&&(Q.innerHTML=Oe),Kt=n(t),i(E.$$.fragment,t),te=n(t),w=u(t,"OL",{start:!0,"data-svelte-h":!0}),g(w)!=="svelte-172a894"&&(w.innerHTML=Pe),ee=n(t),i(A.$$.fragment,t),se=n(t),_=u(t,"OL",{start:!0,"data-svelte-h":!0}),g(_)!=="svelte-zc3ks6"&&(_.innerHTML=Ke),ae=n(t),i(D.$$.fragment,t),le=n(t),i(O.$$.fragment,t),ne=n(t),P=u(t,"OL",{"data-svelte-h":!0}),g(P)!=="svelte-33zejz"&&(P.innerHTML=ts),oe=n(t),i(K.$$.fragment,t),ie=n(t),v=u(t,"OL",{start:!0,"data-svelte-h":!0}),g(v)!=="svelte-172a894"&&(v.innerHTML=es),pe=n(t),i(tt.$$.fragment,t),re=n(t),k=u(t,"OL",{start:!0,"data-svelte-h":!0}),g(k)!=="svelte-zc3ks6"&&(k.innerHTML=ss),de=n(t),i(et.$$.fragment,t),me=n(t),i(st.$$.fragment,t),ce=n(t),i(at.$$.fragment,t),ue=n(t),lt=u(t,"P",{"data-svelte-h":!0}),g(lt)!=="svelte-1dq9go1"&&(lt.innerHTML=as),ge=n(t),nt=u(t,"P",{"data-svelte-h":!0}),g(nt)!=="svelte-sha7eh"&&(nt.innerHTML=ls),fe=n(t),ot=u(t,"OL",{"data-svelte-h":!0}),g(ot)!=="svelte-18vv0no"&&(ot.innerHTML=ns),ye=n(t),it=u(t,"P",{"data-svelte-h":!0}),g(it)!=="svelte-1ypj8lj"&&(it.innerHTML=os),Me=n(t),i(pt.$$.fragment,t),he=n(t),rt=u(t,"P",{"data-svelte-h":!0}),g(rt)!=="svelte-1yv9guc"&&(rt.innerHTML=is),$e=n(t),i(dt.$$.fragment,t),Te=n(t),mt=u(t,"P",{"data-svelte-h":!0}),g(mt)!=="svelte-1ksm788"&&(mt.innerHTML=ps),be=n(t),ct=u(t,"P",{"data-svelte-h":!0}),g(ct)!=="svelte-s9cwoq"&&(ct.innerHTML=rs),Je=n(t),i(ut.$$.fragment,t),je=n(t),i(gt.$$.fragment,t),we=n(t),ft=u(t,"P",{"data-svelte-h":!0}),g(ft)!=="svelte-pl160m"&&(ft.textContent=ds),_e=n(t),yt=u(t,"P",{"data-svelte-h":!0}),g(yt)!=="svelte-r3cnvj"&&(yt.textContent=ms),ve=n(t),i(Mt.$$.fragment,t),ke=n(t),ht=u(t,"P",{"data-svelte-h":!0}),g(ht)!=="svelte-e328yx"&&(ht.innerHTML=cs),Ue=n(t),i($t.$$.fragment,t),Ze=n(t),Tt=u(t,"P",{"data-svelte-h":!0}),g(Tt)!=="svelte-sxq96z"&&(Tt.innerHTML=us),ze=n(t),i(bt.$$.fragment,t),Ie=n(t),i(U.$$.fragment,t),Ge=n(t),i(Jt.$$.fragment,t),Re=n(t),jt=u(t,"P",{"data-svelte-h":!0}),g(jt)!=="svelte-19hbo6c"&&(jt.innerHTML=gs),We=n(t),i(wt.$$.fragment,t),Ce=n(t),i(_t.$$.fragment,t),He=n(t),vt=u(t,"P",{"data-svelte-h":!0}),g(vt)!=="svelte-i7jn21"&&(vt.innerHTML=fs),Xe=n(t),i(kt.$$.fragment,t),qe=n(t),i(Ut.$$.fragment,t),Fe=n(t),zt=u(t,"P",{}),Ms(zt).forEach(s),this.h()},h(){$(y,"name","hf:doc:metadata"),$(y,"content",Zs),$(T,"start","2"),$(b,"start","3"),$(J,"start","2"),$(j,"start","3"),$(w,"start","2"),$(_,"start","3"),$(v,"start","2"),$(k,"start","3")},m(t,e){ws(document.head,y),a(t,Z,e),a(t,h,e),a(t,Zt,e),p(z,t,e),a(t,Gt,e),a(t,I,e),a(t,Rt,e),a(t,G,e),a(t,Wt,e),a(t,R,e),a(t,Ct,e),p(W,t,e),a(t,Ht,e),p(C,t,e),a(t,Xt,e),a(t,H,e),a(t,qt,e),p(X,t,e),a(t,Ft,e),a(t,T,e),a(t,Vt,e),a(t,q,e),a(t,Lt,e),p(F,t,e),a(t,Yt,e),a(t,b,e),a(t,Bt,e),p(V,t,e),a(t,Nt,e),p(L,t,e),a(t,St,e),a(t,Y,e),a(t,xt,e),p(B,t,e),a(t,Qt,e),a(t,J,e),a(t,Et,e),p(N,t,e),a(t,At,e),a(t,j,e),a(t,Dt,e),p(S,t,e),a(t,Ot,e),p(x,t,e),a(t,Pt,e),a(t,Q,e),a(t,Kt,e),p(E,t,e),a(t,te,e),a(t,w,e),a(t,ee,e),p(A,t,e),a(t,se,e),a(t,_,e),a(t,ae,e),p(D,t,e),a(t,le,e),p(O,t,e),a(t,ne,e),a(t,P,e),a(t,oe,e),p(K,t,e),a(t,ie,e),a(t,v,e),a(t,pe,e),p(tt,t,e),a(t,re,e),a(t,k,e),a(t,de,e),p(et,t,e),a(t,me,e),p(st,t,e),a(t,ce,e),p(at,t,e),a(t,ue,e),a(t,lt,e),a(t,ge,e),a(t,nt,e),a(t,fe,e),a(t,ot,e),a(t,ye,e),a(t,it,e),a(t,Me,e),p(pt,t,e),a(t,he,e),a(t,rt,e),a(t,$e,e),p(dt,t,e),a(t,Te,e),a(t,mt,e),a(t,be,e),a(t,ct,e),a(t,Je,e),p(ut,t,e),a(t,je,e),p(gt,t,e),a(t,we,e),a(t,ft,e),a(t,_e,e),a(t,yt,e),a(t,ve,e),p(Mt,t,e),a(t,ke,e),a(t,ht,e),a(t,Ue,e),p($t,t,e),a(t,Ze,e),a(t,Tt,e),a(t,ze,e),p(bt,t,e),a(t,Ie,e),p(U,t,e),a(t,Ge,e),p(Jt,t,e),a(t,Re,e),a(t,jt,e),a(t,We,e),p(wt,t,e),a(t,Ce,e),p(_t,t,e),a(t,He,e),a(t,vt,e),a(t,Xe,e),p(kt,t,e),a(t,qe,e),p(Ut,t,e),a(t,Fe,e),a(t,zt,e),Ve=!0},p(t,[e]){const ys={};e&2&&(ys.$$scope={dirty:e,ctx:t}),U.$set(ys)},i(t){Ve||(r(z.$$.fragment,t),r(W.$$.fragment,t),r(C.$$.fragment,t),r(X.$$.fragment,t),r(F.$$.fragment,t),r(V.$$.fragment,t),r(L.$$.fragment,t),r(B.$$.fragment,t),r(N.$$.fragment,t),r(S.$$.fragment,t),r(x.$$.fragment,t),r(E.$$.fragment,t),r(A.$$.fragment,t),r(D.$$.fragment,t),r(O.$$.fragment,t),r(K.$$.fragment,t),r(tt.$$.fragment,t),r(et.$$.fragment,t),r(st.$$.fragment,t),r(at.$$.fragment,t),r(pt.$$.fragment,t),r(dt.$$.fragment,t),r(ut.$$.fragment,t),r(gt.$$.fragment,t),r(Mt.$$.fragment,t),r($t.$$.fragment,t),r(bt.$$.fragment,t),r(U.$$.fragment,t),r(Jt.$$.fragment,t),r(wt.$$.fragment,t),r(_t.$$.fragment,t),r(kt.$$.fragment,t),r(Ut.$$.fragment,t),Ve=!0)},o(t){d(z.$$.fragment,t),d(W.$$.fragment,t),d(C.$$.fragment,t),d(X.$$.fragment,t),d(F.$$.fragment,t),d(V.$$.fragment,t),d(L.$$.fragment,t),d(B.$$.fragment,t),d(N.$$.fragment,t),d(S.$$.fragment,t),d(x.$$.fragment,t),d(E.$$.fragment,t),d(A.$$.fragment,t),d(D.$$.fragment,t),d(O.$$.fragment,t),d(K.$$.fragment,t),d(tt.$$.fragment,t),d(et.$$.fragment,t),d(st.$$.fragment,t),d(at.$$.fragment,t),d(pt.$$.fragment,t),d(dt.$$.fragment,t),d(ut.$$.fragment,t),d(gt.$$.fragment,t),d(Mt.$$.fragment,t),d($t.$$.fragment,t),d(bt.$$.fragment,t),d(U.$$.fragment,t),d(Jt.$$.fragment,t),d(wt.$$.fragment,t),d(_t.$$.fragment,t),d(kt.$$.fragment,t),d(Ut.$$.fragment,t),Ve=!1},d(t){t&&(s(Z),s(h),s(Zt),s(Gt),s(I),s(Rt),s(G),s(Wt),s(R),s(Ct),s(Ht),s(Xt),s(H),s(qt),s(Ft),s(T),s(Vt),s(q),s(Lt),s(Yt),s(b),s(Bt),s(Nt),s(St),s(Y),s(xt),s(Qt),s(J),s(Et),s(At),s(j),s(Dt),s(Ot),s(Pt),s(Q),s(Kt),s(te),s(w),s(ee),s(se),s(_),s(ae),s(le),s(ne),s(P),s(oe),s(ie),s(v),s(pe),s(re),s(k),s(de),s(me),s(ce),s(ue),s(lt),s(ge),s(nt),s(fe),s(ot),s(ye),s(it),s(Me),s(he),s(rt),s($e),s(Te),s(mt),s(be),s(ct),s(Je),s(je),s(we),s(ft),s(_e),s(yt),s(ve),s(ke),s(ht),s(Ue),s(Ze),s(Tt),s(ze),s(Ie),s(Ge),s(Re),s(jt),s(We),s(Ce),s(He),s(vt),s(Xe),s(qe),s(Fe),s(zt)),s(y),m(z,t),m(W,t),m(C,t),m(X,t),m(F,t),m(V,t),m(L,t),m(B,t),m(N,t),m(S,t),m(x,t),m(E,t),m(A,t),m(D,t),m(O,t),m(K,t),m(tt,t),m(et,t),m(st,t),m(at,t),m(pt,t),m(dt,t),m(ut,t),m(gt,t),m(Mt,t),m($t,t),m(bt,t),m(U,t),m(Jt,t),m(wt,t),m(_t,t),m(kt,t),m(Ut,t)}}}const Zs='{"title":"Cloud storage","local":"cloud-storage","sections":[{"title":"Set up your cloud storage FileSystem","local":"set-up-your-cloud-storage-filesystem","sections":[{"title":"Amazon S3","local":"amazon-s3","sections":[],"depth":3},{"title":"Google Cloud Storage","local":"google-cloud-storage","sections":[],"depth":3},{"title":"Azure Blob Storage","local":"azure-blob-storage","sections":[],"depth":3},{"title":"Oracle Cloud Object Storage","local":"oracle-cloud-object-storage","sections":[],"depth":3}],"depth":2},{"title":"Load and Save your datasets using your cloud storage FileSystem","local":"load-and-save-your-datasets-using-your-cloud-storage-filesystem","sections":[{"title":"Download and prepare a dataset into a cloud storage","local":"download-and-prepare-a-dataset-into-a-cloud-storage","sections":[{"title":"Dask","local":"dask","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Saving serialized datasets","local":"saving-serialized-datasets","sections":[],"depth":2},{"title":"Listing serialized datasets","local":"listing-serialized-datasets","sections":[{"title":"Load serialized datasets","local":"load-serialized-datasets","sections":[],"depth":3}],"depth":2}],"depth":1}';function zs(It){return $s(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Hs extends bs{constructor(y){super(),Js(this,y,zs,Us,hs,{})}}export{Hs as component};
