import{s as Et,n as qt,o as Kt}from"../chunks/scheduler.12fdeddc.js";import{S as Ot,i as te,g as i,s as l,r as f,A as ee,h as d,f as a,c as n,j as Bt,u as o,x as m,k as N,y as ae,a as s,v as r,d as u,t as h,w as g}from"../chunks/index.88dae322.js";import{C as Tt}from"../chunks/CodeBlock.8e54a086.js";import{H as S,E as se}from"../chunks/getInferenceSnippets.00466244.js";function le(Ut){let p,B,V,E,M,q,v,_t=`<a href="https://github.com/webdataset/webdataset" rel="nofollow">WebDataset</a> is a library for writing I/O pipelines for large datasets.
Its sequential I/O and sharding features make it especially useful for streaming large-scale datasets to a DataLoader.`,K,C,O,y,Jt=`A WebDataset file is a TAR archive containing a series of data files.
All successive data files with the same prefix are considered to be part of the same example (e.g., an image/audio file and its label or metadata):`,tt,c,Wt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-format.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-format-dark.png"/>',et,$,Gt="Labels and metadata can be in a <code>.json</code> file, in a <code>.txt</code> (for a caption, a description), or in a <code>.cls</code> (for a class index).",at,x,Lt=`A large scale WebDataset is made of many files called shards, where each shard is a TAR archive.
Each shard is often ~1GB but the full dataset can be multiple terabytes!`,st,k,lt,j,Ht="WebDataset is designed for multimodal datasets, i.e. for image, audio and/or video datasets.",nt,T,Zt="Indeed, since media files tend to be quite big, WebDataset’s sequential I/O enables large reads and buffering, resulting in the best data loading speed.",it,U,Rt="Here is a non-exhaustive list of supported data formats:",dt,_,Dt="<li>image: jpeg, png, tiff</li> <li>audio: mp3, m4a, wav, flac</li> <li>video: mp4, mov, avi</li> <li>other: npy, npz</li>",mt,J,Xt='The full list evolves over time and depends on the implementation. For example, you can find which formats the <code>webdataset</code> package supports in the source code <a href="https://github.com/webdataset/webdataset/blob/main/src/webdataset/autodecode.py" rel="nofollow">here</a>.',pt,W,ft,G,At=`Streaming TAR archives is fast because it reads contiguous chunks of data.
It can be orders of magnitude faster than reading separate data files one by one.`,ot,L,It="WebDataset streaming offers high-speed performance both when reading from disk and from cloud storage, which makes it an ideal format to feed to a DataLoader:",rt,b,zt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-stream.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-stream-dark.png"/>',ut,H,Pt='For example here is how to stream the <a href="https://huggingface.co/datasets/timm/imagenet-12k-wds" rel="nofollow">timm/imagenet-12k-wds</a> dataset directly from Hugging Face:',ht,Z,Qt='First you need to <a href="/docs/huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',gt,R,ct,D,Ft="And then you can stream the dataset with WebDataset:",bt,X,wt,A,Mt,I,Vt=`Generally, datasets in WebDataset formats are already shuffled and ready to feed to a DataLoader.
But you can still reshuffle the data with WebDataset’s approximate shuffling.`,vt,z,Yt="In addition to shuffling the list of shards, WebDataset uses a buffer to shuffle a dataset without any cost to speed:",Ct,w,Nt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-shuffle.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-shuffle-dark.png"/>',yt,P,St="To shuffle a list of sharded files and randomly sample from the shuffle buffer:",$t,Q,xt,F,kt,Y,jt;return M=new S({props:{title:"WebDataset",local:"webdataset",headingTag:"h1"}}),C=new S({props:{title:"The WebDataset format",local:"the-webdataset-format",headingTag:"h2"}}),k=new S({props:{title:"Multimodal support",local:"multimodal-support",headingTag:"h2"}}),W=new S({props:{title:"Streaming",local:"streaming",headingTag:"h2"}}),R=new Tt({props:{code:"aGYlMjBhdXRoJTIwbG9naW4=",highlighted:'<span class="hljs-attribute">hf auth login</span>',wrap:!1}}),X=new Tt({props:{code:"aW1wb3J0JTIwd2ViZGF0YXNldCUyMGFzJTIwd2RzJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGdldF90b2tlbiUwQWZyb20lMjB0b3JjaC51dGlscy5kYXRhJTIwaW1wb3J0JTIwRGF0YUxvYWRlciUwQSUwQWhmX3Rva2VuJTIwJTNEJTIwZ2V0X3Rva2VuKCklMEF1cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGZGF0YXNldHMlMkZ0aW1tJTJGaW1hZ2VuZXQtMTJrLXdkcyUyRnJlc29sdmUlMkZtYWluJTJGaW1hZ2VuZXQxMmstdHJhaW4tJTdCJTdCMDAwMC4uMTAyMyU3RCU3RC50YXIlMjIlMEF1cmwlMjAlM0QlMjBmJTIycGlwZSUzQWN1cmwlMjAtcyUyMC1MJTIwJTdCdXJsJTdEJTIwLUglMjAnQXV0aG9yaXphdGlvbiUzQUJlYXJlciUyMCU3QmhmX3Rva2VuJTdEJyUyMiUwQWRhdGFzZXQlMjAlM0QlMjB3ZHMuV2ViRGF0YXNldCh1cmwpLmRlY29kZSgpJTBBZGF0YWxvYWRlciUyMCUzRCUyMERhdGFMb2FkZXIoZGF0YXNldCUyQyUyMGJhdGNoX3NpemUlM0Q2NCUyQyUyMG51bV93b3JrZXJzJTNENCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> webdataset <span class="hljs-keyword">as</span> wds
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_token
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

<span class="hljs-meta">&gt;&gt;&gt; </span>hf_token = get_token()
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://huggingface.co/datasets/timm/imagenet-12k-wds/resolve/main/imagenet12k-train-{{0000..1023}}.tar&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">f&quot;pipe:curl -s -L <span class="hljs-subst">{url}</span> -H &#x27;Authorization:Bearer <span class="hljs-subst">{hf_token}</span>&#x27;&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = wds.WebDataset(url).decode()
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(dataset, batch_size=<span class="hljs-number">64</span>, num_workers=<span class="hljs-number">4</span>)`,wrap:!1}}),A=new S({props:{title:"Shuffle",local:"shuffle",headingTag:"h2"}}),Q=new Tt({props:{code:"YnVmZmVyX3NpemUlMjAlM0QlMjAxMDAwJTBBZGF0YXNldCUyMCUzRCUyMCglMEElMjAlMjAlMjAlMjB3ZHMuV2ViRGF0YXNldCh1cmwlMkMlMjBzaGFyZHNodWZmbGUlM0RUcnVlKSUwQSUyMCUyMCUyMCUyMC5zaHVmZmxlKGJ1ZmZlcl9zaXplKSUwQSUyMCUyMCUyMCUyMC5kZWNvZGUoKSUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>buffer_size = <span class="hljs-number">1000</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = (
<span class="hljs-meta">... </span>    wds.WebDataset(url, shardshuffle=<span class="hljs-literal">True</span>)
<span class="hljs-meta">... </span>    .shuffle(buffer_size)
<span class="hljs-meta">... </span>    .decode()
<span class="hljs-meta">... </span>)`,wrap:!1}}),F=new se({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-webdataset.md"}}),{c(){p=i("meta"),B=l(),V=i("p"),E=l(),f(M.$$.fragment),q=l(),v=i("p"),v.innerHTML=_t,K=l(),f(C.$$.fragment),O=l(),y=i("p"),y.textContent=Jt,tt=l(),c=i("div"),c.innerHTML=Wt,et=l(),$=i("p"),$.innerHTML=Gt,at=l(),x=i("p"),x.textContent=Lt,st=l(),f(k.$$.fragment),lt=l(),j=i("p"),j.textContent=Ht,nt=l(),T=i("p"),T.textContent=Zt,it=l(),U=i("p"),U.textContent=Rt,dt=l(),_=i("ul"),_.innerHTML=Dt,mt=l(),J=i("p"),J.innerHTML=Xt,pt=l(),f(W.$$.fragment),ft=l(),G=i("p"),G.textContent=At,ot=l(),L=i("p"),L.textContent=It,rt=l(),b=i("div"),b.innerHTML=zt,ut=l(),H=i("p"),H.innerHTML=Pt,ht=l(),Z=i("p"),Z.innerHTML=Qt,gt=l(),f(R.$$.fragment),ct=l(),D=i("p"),D.textContent=Ft,bt=l(),f(X.$$.fragment),wt=l(),f(A.$$.fragment),Mt=l(),I=i("p"),I.textContent=Vt,vt=l(),z=i("p"),z.textContent=Yt,Ct=l(),w=i("div"),w.innerHTML=Nt,yt=l(),P=i("p"),P.textContent=St,$t=l(),f(Q.$$.fragment),xt=l(),f(F.$$.fragment),kt=l(),Y=i("p"),this.h()},l(t){const e=ee("svelte-u9bgzb",document.head);p=d(e,"META",{name:!0,content:!0}),e.forEach(a),B=n(t),V=d(t,"P",{}),Bt(V).forEach(a),E=n(t),o(M.$$.fragment,t),q=n(t),v=d(t,"P",{"data-svelte-h":!0}),m(v)!=="svelte-1x0ilw7"&&(v.innerHTML=_t),K=n(t),o(C.$$.fragment,t),O=n(t),y=d(t,"P",{"data-svelte-h":!0}),m(y)!=="svelte-s28fzx"&&(y.textContent=Jt),tt=n(t),c=d(t,"DIV",{class:!0,"data-svelte-h":!0}),m(c)!=="svelte-1bdrcp9"&&(c.innerHTML=Wt),et=n(t),$=d(t,"P",{"data-svelte-h":!0}),m($)!=="svelte-7sb7k1"&&($.innerHTML=Gt),at=n(t),x=d(t,"P",{"data-svelte-h":!0}),m(x)!=="svelte-1pmozez"&&(x.textContent=Lt),st=n(t),o(k.$$.fragment,t),lt=n(t),j=d(t,"P",{"data-svelte-h":!0}),m(j)!=="svelte-11egvlw"&&(j.textContent=Ht),nt=n(t),T=d(t,"P",{"data-svelte-h":!0}),m(T)!=="svelte-td3uuk"&&(T.textContent=Zt),it=n(t),U=d(t,"P",{"data-svelte-h":!0}),m(U)!=="svelte-lwvb2g"&&(U.textContent=Rt),dt=n(t),_=d(t,"UL",{"data-svelte-h":!0}),m(_)!=="svelte-p3flt6"&&(_.innerHTML=Dt),mt=n(t),J=d(t,"P",{"data-svelte-h":!0}),m(J)!=="svelte-1npudm"&&(J.innerHTML=Xt),pt=n(t),o(W.$$.fragment,t),ft=n(t),G=d(t,"P",{"data-svelte-h":!0}),m(G)!=="svelte-d0lfwq"&&(G.textContent=At),ot=n(t),L=d(t,"P",{"data-svelte-h":!0}),m(L)!=="svelte-1ski7pk"&&(L.textContent=It),rt=n(t),b=d(t,"DIV",{class:!0,"data-svelte-h":!0}),m(b)!=="svelte-16x5sv3"&&(b.innerHTML=zt),ut=n(t),H=d(t,"P",{"data-svelte-h":!0}),m(H)!=="svelte-kd5fni"&&(H.innerHTML=Pt),ht=n(t),Z=d(t,"P",{"data-svelte-h":!0}),m(Z)!=="svelte-1jdzygp"&&(Z.innerHTML=Qt),gt=n(t),o(R.$$.fragment,t),ct=n(t),D=d(t,"P",{"data-svelte-h":!0}),m(D)!=="svelte-3qvfc"&&(D.textContent=Ft),bt=n(t),o(X.$$.fragment,t),wt=n(t),o(A.$$.fragment,t),Mt=n(t),I=d(t,"P",{"data-svelte-h":!0}),m(I)!=="svelte-1qsq6cj"&&(I.textContent=Vt),vt=n(t),z=d(t,"P",{"data-svelte-h":!0}),m(z)!=="svelte-e5b3kw"&&(z.textContent=Yt),Ct=n(t),w=d(t,"DIV",{class:!0,"data-svelte-h":!0}),m(w)!=="svelte-15c8cpf"&&(w.innerHTML=Nt),yt=n(t),P=d(t,"P",{"data-svelte-h":!0}),m(P)!=="svelte-1qykgdn"&&(P.textContent=St),$t=n(t),o(Q.$$.fragment,t),xt=n(t),o(F.$$.fragment,t),kt=n(t),Y=d(t,"P",{}),Bt(Y).forEach(a),this.h()},h(){N(p,"name","hf:doc:metadata"),N(p,"content",ne),N(c,"class","flex justify-center"),N(b,"class","flex justify-center"),N(w,"class","flex justify-center")},m(t,e){ae(document.head,p),s(t,B,e),s(t,V,e),s(t,E,e),r(M,t,e),s(t,q,e),s(t,v,e),s(t,K,e),r(C,t,e),s(t,O,e),s(t,y,e),s(t,tt,e),s(t,c,e),s(t,et,e),s(t,$,e),s(t,at,e),s(t,x,e),s(t,st,e),r(k,t,e),s(t,lt,e),s(t,j,e),s(t,nt,e),s(t,T,e),s(t,it,e),s(t,U,e),s(t,dt,e),s(t,_,e),s(t,mt,e),s(t,J,e),s(t,pt,e),r(W,t,e),s(t,ft,e),s(t,G,e),s(t,ot,e),s(t,L,e),s(t,rt,e),s(t,b,e),s(t,ut,e),s(t,H,e),s(t,ht,e),s(t,Z,e),s(t,gt,e),r(R,t,e),s(t,ct,e),s(t,D,e),s(t,bt,e),r(X,t,e),s(t,wt,e),r(A,t,e),s(t,Mt,e),s(t,I,e),s(t,vt,e),s(t,z,e),s(t,Ct,e),s(t,w,e),s(t,yt,e),s(t,P,e),s(t,$t,e),r(Q,t,e),s(t,xt,e),r(F,t,e),s(t,kt,e),s(t,Y,e),jt=!0},p:qt,i(t){jt||(u(M.$$.fragment,t),u(C.$$.fragment,t),u(k.$$.fragment,t),u(W.$$.fragment,t),u(R.$$.fragment,t),u(X.$$.fragment,t),u(A.$$.fragment,t),u(Q.$$.fragment,t),u(F.$$.fragment,t),jt=!0)},o(t){h(M.$$.fragment,t),h(C.$$.fragment,t),h(k.$$.fragment,t),h(W.$$.fragment,t),h(R.$$.fragment,t),h(X.$$.fragment,t),h(A.$$.fragment,t),h(Q.$$.fragment,t),h(F.$$.fragment,t),jt=!1},d(t){t&&(a(B),a(V),a(E),a(q),a(v),a(K),a(O),a(y),a(tt),a(c),a(et),a($),a(at),a(x),a(st),a(lt),a(j),a(nt),a(T),a(it),a(U),a(dt),a(_),a(mt),a(J),a(pt),a(ft),a(G),a(ot),a(L),a(rt),a(b),a(ut),a(H),a(ht),a(Z),a(gt),a(ct),a(D),a(bt),a(wt),a(Mt),a(I),a(vt),a(z),a(Ct),a(w),a(yt),a(P),a($t),a(xt),a(kt),a(Y)),a(p),g(M,t),g(C,t),g(k,t),g(W,t),g(R,t),g(X,t),g(A,t),g(Q,t),g(F,t)}}}const ne='{"title":"WebDataset","local":"webdataset","sections":[{"title":"The WebDataset format","local":"the-webdataset-format","sections":[],"depth":2},{"title":"Multimodal support","local":"multimodal-support","sections":[],"depth":2},{"title":"Streaming","local":"streaming","sections":[],"depth":2},{"title":"Shuffle","local":"shuffle","sections":[],"depth":2}],"depth":1}';function ie(Ut){return Kt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class oe extends Ot{constructor(p){super(),te(this,p,ie,le,Et,{})}}export{oe as component};
