import{s as L,n as z,o as P}from"../chunks/scheduler.d6170356.js";import{S as V,i as A,g as h,s as l,r as k,A as D,h as u,f as a,c as i,j as W,u as Z,x as j,k as X,y as S,a as s,v as G,d as H,t as v,w as C}from"../chunks/index.fcd4cc08.js";import{C as Y}from"../chunks/CodeBlock.7b16bdef.js";import{H as E}from"../chunks/Heading.8660bd0e.js";function I(x){let n,w,f,y,o,b,p,B=`<a href="https://github.com/webdataset/webdataset" rel="nofollow">WebDataset</a> is a library to write I/O pipelines for large datasets.
Since it supports streaming data using HTTP, you can use the Hugging Face data files URLs to stream a dataset in WebDataset format:`,M,r,F='First you need to <a href="../huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',_,m,$,c,R="And then you can stream Hugging Face datasets in WebDataset:",U,d,J,g,T;return o=new E({props:{title:"WebDataset",local:"webdataset",headingTag:"h1"}}),m=new Y({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}}),d=new Y({props:{code:"aW1wb3J0JTIwd2ViZGF0YXNldCUyMGFzJTIwd2RzJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmRm9sZGVyJTBBJTBBaGZfdG9rZW4lMjAlM0QlMjBIZkZvbGRlcigpLmdldF90b2tlbigpJTBBZGF0YXNldCUyMCUzRCUyMHdkcy5XZWJEYXRhc2V0KGYlMjJwaXBlJTNBY3VybCUyMC1zJTIwLUwlMjBodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGZGF0YXNldHMlMkZ1c2VybmFtZSUyRm15X3dkc19kYXRhc2V0JTJGcmVzb2x2ZSUyRm1haW4lMkZ0cmFpbi0wMDAwMDAudGFyJTIwLUglMjAnQXV0aG9yaXphdGlvbiUzQUJlYXJlciUyMCU3QmhmX3Rva2VuJTdEJyUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> webdataset <span class="hljs-keyword">as</span> wds
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfFolder

<span class="hljs-meta">&gt;&gt;&gt; </span>hf_token = HfFolder().get_token()
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = wds.WebDataset(<span class="hljs-string">f&quot;pipe:curl -s -L https://huggingface.co/datasets/username/my_wds_dataset/resolve/main/train-000000.tar -H &#x27;Authorization:Bearer <span class="hljs-subst">{hf_token}</span>&#x27;&quot;</span>)`,wrap:!1}}),{c(){n=h("meta"),w=l(),f=h("p"),y=l(),k(o.$$.fragment),b=l(),p=h("p"),p.innerHTML=B,M=l(),r=h("p"),r.innerHTML=F,_=l(),k(m.$$.fragment),$=l(),c=h("p"),c.textContent=R,U=l(),k(d.$$.fragment),J=l(),g=h("p"),this.h()},l(t){const e=D("svelte-u9bgzb",document.head);n=u(e,"META",{name:!0,content:!0}),e.forEach(a),w=i(t),f=u(t,"P",{}),W(f).forEach(a),y=i(t),Z(o.$$.fragment,t),b=i(t),p=u(t,"P",{"data-svelte-h":!0}),j(p)!=="svelte-1cfdl4w"&&(p.innerHTML=B),M=i(t),r=u(t,"P",{"data-svelte-h":!0}),j(r)!=="svelte-1hbzj8b"&&(r.innerHTML=F),_=i(t),Z(m.$$.fragment,t),$=i(t),c=u(t,"P",{"data-svelte-h":!0}),j(c)!=="svelte-30husn"&&(c.textContent=R),U=i(t),Z(d.$$.fragment,t),J=i(t),g=u(t,"P",{}),W(g).forEach(a),this.h()},h(){X(n,"name","hf:doc:metadata"),X(n,"content",N)},m(t,e){S(document.head,n),s(t,w,e),s(t,f,e),s(t,y,e),G(o,t,e),s(t,b,e),s(t,p,e),s(t,M,e),s(t,r,e),s(t,_,e),G(m,t,e),s(t,$,e),s(t,c,e),s(t,U,e),G(d,t,e),s(t,J,e),s(t,g,e),T=!0},p:z,i(t){T||(H(o.$$.fragment,t),H(m.$$.fragment,t),H(d.$$.fragment,t),T=!0)},o(t){v(o.$$.fragment,t),v(m.$$.fragment,t),v(d.$$.fragment,t),T=!1},d(t){t&&(a(w),a(f),a(y),a(b),a(p),a(M),a(r),a(_),a($),a(c),a(U),a(J),a(g)),a(n),C(o,t),C(m,t),C(d,t)}}}const N='{"title":"WebDataset","local":"webdataset","sections":[],"depth":1}';function Q(x){return P(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class et extends V{constructor(n){super(),A(this,n,Q,I,L,{})}}export{et as component};
