import{s as je,n as Re,o as He}from"../chunks/scheduler.12fdeddc.js";import{S as Ye,i as xe,g as l,s as i,r as d,A as qe,h as f,f as a,c as s,j as Ie,u as r,x as u,k as Te,y as Xe,a as n,v as o,d as p,t as m,w as g}from"../chunks/index.88dae322.js";import{C as N}from"../chunks/CodeBlock.8e54a086.js";import{H as Q,E as Qe}from"../chunks/getInferenceSnippets.00466244.js";function Se(Je){let c,E,S,L,$,P,w,ye='<a href="https://daft.ai/" rel="nofollow">Daft</a> is a high-performance data engine providing simple and reliable data processing for any modality and scale. Daft has native support for reading from and writing to Hugging Face datasets.',D,h,Ze='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/daft_hf.png"/>',V,T,z,J,Me="To get started, pip install <code>daft</code> with the <code>huggingface</code> feature:",K,y,A,Z,O,M,_e='Daft is able to read datasets directly from the Hugging Face Hub using the <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_huggingface" rel="nofollow"><code>daft.read_huggingface()</code></a> function or via the <code>hf://datasets/</code> protocol.',ee,_,te,b,be='Using <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_huggingface" rel="nofollow"><code>daft.read_huggingface()</code></a>, you can easily load a dataset.',ae,v,ne,F,ve="This will read the entire dataset into a DataFrame.",ie,U,se,C,Fe='Not only can you read entire datasets, but you can also read individual files from a dataset repository. Using a read function that takes in a path (such as <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_parquet" rel="nofollow"><code>daft.read_parquet()</code></a>, <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_csv" rel="nofollow"><code>daft.read_csv()</code></a>, or <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_json" rel="nofollow"><code>daft.read_json()</code></a>), specify a Hugging Face dataset path via the <code>hf://datasets/</code> prefix:',le,W,fe,k,de,G,Ue='Daft is able to write Parquet files to a Hugging Face dataset repository using <a href="https://docs.daft.ai/en/stable/api/dataframe/#daft.DataFrame.write_deltalake" rel="nofollow"><code>daft.DataFrame.write_huggingface</code></a>. Daft supports <a href="https://huggingface.co/blog/parquet-cdc" rel="nofollow">Content-Defined Chunking</a> and <a href="https://huggingface.co/blog/xet-on-the-hub" rel="nofollow">Xet</a> for faster, deduplicated writes.',re,I,Ce="Basic usage:",oe,j,pe,R,We='See the <a href="https://docs.daft.ai/en/stable/api/dataframe/#daft.DataFrame.write_huggingface" rel="nofollow"><code>DataFrame.write_huggingface</code></a> API page for more info.',me,H,ge,Y,ke='The <code>token</code> parameter in <a href="https://docs.daft.ai/en/stable/api/config/#daft.io.HuggingFaceConfig" rel="nofollow"><code>daft.io.HuggingFaceConfig</code></a> can be used to specify a Hugging Face access token for requests that require authentication (e.g. reading private dataset repositories or writing to a dataset repository).',ue,x,Ge="Example of loading a dataset with a specified token:",ce,q,he,X,$e,B,we;return $=new Q({props:{title:"Daft",local:"daft",headingTag:"h1"}}),T=new Q({props:{title:"Getting Started",local:"getting-started",headingTag:"h2"}}),y=new N({props:{code:"cGlwJTIwaW5zdGFsbCUyMCdkYWZ0JTVCaHVnZ2luZ2ZhY2UlNUQn",highlighted:'pip install <span class="hljs-string">&#x27;daft[huggingface]&#x27;</span>',wrap:!1}}),Z=new Q({props:{title:"Read",local:"read",headingTag:"h2"}}),_=new Q({props:{title:"Reading an Entire Dataset",local:"reading-an-entire-dataset",headingTag:"h3"}}),v=new N({props:{code:"aW1wb3J0JTIwZGFmdCUwQSUwQWRmJTIwJTNEJTIwZGFmdC5yZWFkX2h1Z2dpbmdmYWNlKCUyMnVzZXJuYW1lJTJGZGF0YXNldF9uYW1lJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> daft

df = daft.read_huggingface(<span class="hljs-string">&quot;username/dataset_name&quot;</span>)`,wrap:!1}}),U=new Q({props:{title:"Reading Specific Files",local:"reading-specific-files",headingTag:"h3"}}),W=new N({props:{code:"aW1wb3J0JTIwZGFmdCUwQSUwQSUyMyUyMHJlYWQlMjBhJTIwc3BlY2lmaWMlMjBQYXJxdWV0JTIwZmlsZSUwQWRmJTIwJTNEJTIwZGFmdC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGZGF0YXNldF9uYW1lJTJGZmlsZV9uYW1lLnBhcnF1ZXQlMjIpJTBBJTBBJTIzJTIwb3IlMjBhJTIwY3N2JTIwZmlsZSUwQWRmJTIwJTNEJTIwZGFmdC5yZWFkX2NzdiglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZkYXRhc2V0X25hbWUlMkZmaWxlX25hbWUuY3N2JTIyKSUwQSUwQSUyMyUyMG9yJTIwYSUyMHNldCUyMG9mJTIwUGFycXVldCUyMGZpbGVzJTIwdXNpbmclMjBhJTIwZ2xvYiUyMHBhdHRlcm4lMEFkZiUyMCUzRCUyMGRhZnQucmVhZF9wYXJxdWV0KCUyMmhmJTNBJTJGJTJGZGF0YXNldHMlMkZ1c2VybmFtZSUyRmRhdGFzZXRfbmFtZSUyRioqJTJGKi5wYXJxdWV0JTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> daft

<span class="hljs-comment"># read a specific Parquet file</span>
df = daft.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/dataset_name/file_name.parquet&quot;</span>)

<span class="hljs-comment"># or a csv file</span>
df = daft.read_csv(<span class="hljs-string">&quot;hf://datasets/username/dataset_name/file_name.csv&quot;</span>)

<span class="hljs-comment"># or a set of Parquet files using a glob pattern</span>
df = daft.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/dataset_name/**/*.parquet&quot;</span>)`,wrap:!1}}),k=new Q({props:{title:"Write",local:"write",headingTag:"h2"}}),j=new N({props:{code:"aW1wb3J0JTIwZGFmdCUwQSUwQWRmJTNBJTIwZGFmdC5EYXRhRnJhbWUlMjAlM0QlMjAuLi4lMEElMEFkZi53cml0ZV9odWdnaW5nZmFjZSglMjJ1c2VybmFtZSUyRmRhdGFzZXRfbmFtZSUyMik=",highlighted:`<span class="hljs-keyword">import</span> daft

df: daft.DataFrame = ...

df.write_huggingface(<span class="hljs-string">&quot;username/dataset_name&quot;</span>)`,wrap:!1}}),H=new Q({props:{title:"Authentication",local:"authentication",headingTag:"h2"}}),q=new N({props:{code:"ZnJvbSUyMGRhZnQuaW8lMjBpbXBvcnQlMjBJT0NvbmZpZyUyQyUyMEh1Z2dpbmdGYWNlQ29uZmlnJTBBJTBBaW9fY29uZmlnJTIwJTNEJTIwSU9Db25maWcoaGYlM0RIdWdnaW5nRmFjZUNvbmZpZyh0b2tlbiUzRCUyMnlvdXJfdG9rZW4lMjIpKSUwQWRmJTIwJTNEJTIwZGFmdC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGZGF0YXNldF9uYW1lJTIyJTJDJTIwaW9fY29uZmlnJTNEaW9fY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">from</span> daft.io <span class="hljs-keyword">import</span> IOConfig, HuggingFaceConfig

io_config = IOConfig(hf=HuggingFaceConfig(token=<span class="hljs-string">&quot;your_token&quot;</span>))
df = daft.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/dataset_name&quot;</span>, io_config=io_config)`,wrap:!1}}),X=new Qe({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-daft.md"}}),{c(){c=l("meta"),E=i(),S=l("p"),L=i(),d($.$$.fragment),P=i(),w=l("p"),w.innerHTML=ye,D=i(),h=l("div"),h.innerHTML=Ze,V=i(),d(T.$$.fragment),z=i(),J=l("p"),J.innerHTML=Me,K=i(),d(y.$$.fragment),A=i(),d(Z.$$.fragment),O=i(),M=l("p"),M.innerHTML=_e,ee=i(),d(_.$$.fragment),te=i(),b=l("p"),b.innerHTML=be,ae=i(),d(v.$$.fragment),ne=i(),F=l("p"),F.textContent=ve,ie=i(),d(U.$$.fragment),se=i(),C=l("p"),C.innerHTML=Fe,le=i(),d(W.$$.fragment),fe=i(),d(k.$$.fragment),de=i(),G=l("p"),G.innerHTML=Ue,re=i(),I=l("p"),I.textContent=Ce,oe=i(),d(j.$$.fragment),pe=i(),R=l("p"),R.innerHTML=We,me=i(),d(H.$$.fragment),ge=i(),Y=l("p"),Y.innerHTML=ke,ue=i(),x=l("p"),x.textContent=Ge,ce=i(),d(q.$$.fragment),he=i(),d(X.$$.fragment),$e=i(),B=l("p"),this.h()},l(e){const t=qe("svelte-u9bgzb",document.head);c=f(t,"META",{name:!0,content:!0}),t.forEach(a),E=s(e),S=f(e,"P",{}),Ie(S).forEach(a),L=s(e),r($.$$.fragment,e),P=s(e),w=f(e,"P",{"data-svelte-h":!0}),u(w)!=="svelte-qmy1ip"&&(w.innerHTML=ye),D=s(e),h=f(e,"DIV",{class:!0,"data-svelte-h":!0}),u(h)!=="svelte-1xvl7v5"&&(h.innerHTML=Ze),V=s(e),r(T.$$.fragment,e),z=s(e),J=f(e,"P",{"data-svelte-h":!0}),u(J)!=="svelte-fialq8"&&(J.innerHTML=Me),K=s(e),r(y.$$.fragment,e),A=s(e),r(Z.$$.fragment,e),O=s(e),M=f(e,"P",{"data-svelte-h":!0}),u(M)!=="svelte-17u1dsn"&&(M.innerHTML=_e),ee=s(e),r(_.$$.fragment,e),te=s(e),b=f(e,"P",{"data-svelte-h":!0}),u(b)!=="svelte-1qmv9cj"&&(b.innerHTML=be),ae=s(e),r(v.$$.fragment,e),ne=s(e),F=f(e,"P",{"data-svelte-h":!0}),u(F)!=="svelte-1n7yb6"&&(F.textContent=ve),ie=s(e),r(U.$$.fragment,e),se=s(e),C=f(e,"P",{"data-svelte-h":!0}),u(C)!=="svelte-1ykvjbj"&&(C.innerHTML=Fe),le=s(e),r(W.$$.fragment,e),fe=s(e),r(k.$$.fragment,e),de=s(e),G=f(e,"P",{"data-svelte-h":!0}),u(G)!=="svelte-1eu2lag"&&(G.innerHTML=Ue),re=s(e),I=f(e,"P",{"data-svelte-h":!0}),u(I)!=="svelte-1aguo8v"&&(I.textContent=Ce),oe=s(e),r(j.$$.fragment,e),pe=s(e),R=f(e,"P",{"data-svelte-h":!0}),u(R)!=="svelte-127tqnc"&&(R.innerHTML=We),me=s(e),r(H.$$.fragment,e),ge=s(e),Y=f(e,"P",{"data-svelte-h":!0}),u(Y)!=="svelte-19rkul4"&&(Y.innerHTML=ke),ue=s(e),x=f(e,"P",{"data-svelte-h":!0}),u(x)!=="svelte-ehovs6"&&(x.textContent=Ge),ce=s(e),r(q.$$.fragment,e),he=s(e),r(X.$$.fragment,e),$e=s(e),B=f(e,"P",{}),Ie(B).forEach(a),this.h()},h(){Te(c,"name","hf:doc:metadata"),Te(c,"content",Be),Te(h,"class","flex justify-center")},m(e,t){Xe(document.head,c),n(e,E,t),n(e,S,t),n(e,L,t),o($,e,t),n(e,P,t),n(e,w,t),n(e,D,t),n(e,h,t),n(e,V,t),o(T,e,t),n(e,z,t),n(e,J,t),n(e,K,t),o(y,e,t),n(e,A,t),o(Z,e,t),n(e,O,t),n(e,M,t),n(e,ee,t),o(_,e,t),n(e,te,t),n(e,b,t),n(e,ae,t),o(v,e,t),n(e,ne,t),n(e,F,t),n(e,ie,t),o(U,e,t),n(e,se,t),n(e,C,t),n(e,le,t),o(W,e,t),n(e,fe,t),o(k,e,t),n(e,de,t),n(e,G,t),n(e,re,t),n(e,I,t),n(e,oe,t),o(j,e,t),n(e,pe,t),n(e,R,t),n(e,me,t),o(H,e,t),n(e,ge,t),n(e,Y,t),n(e,ue,t),n(e,x,t),n(e,ce,t),o(q,e,t),n(e,he,t),o(X,e,t),n(e,$e,t),n(e,B,t),we=!0},p:Re,i(e){we||(p($.$$.fragment,e),p(T.$$.fragment,e),p(y.$$.fragment,e),p(Z.$$.fragment,e),p(_.$$.fragment,e),p(v.$$.fragment,e),p(U.$$.fragment,e),p(W.$$.fragment,e),p(k.$$.fragment,e),p(j.$$.fragment,e),p(H.$$.fragment,e),p(q.$$.fragment,e),p(X.$$.fragment,e),we=!0)},o(e){m($.$$.fragment,e),m(T.$$.fragment,e),m(y.$$.fragment,e),m(Z.$$.fragment,e),m(_.$$.fragment,e),m(v.$$.fragment,e),m(U.$$.fragment,e),m(W.$$.fragment,e),m(k.$$.fragment,e),m(j.$$.fragment,e),m(H.$$.fragment,e),m(q.$$.fragment,e),m(X.$$.fragment,e),we=!1},d(e){e&&(a(E),a(S),a(L),a(P),a(w),a(D),a(h),a(V),a(z),a(J),a(K),a(A),a(O),a(M),a(ee),a(te),a(b),a(ae),a(ne),a(F),a(ie),a(se),a(C),a(le),a(fe),a(de),a(G),a(re),a(I),a(oe),a(pe),a(R),a(me),a(ge),a(Y),a(ue),a(x),a(ce),a(he),a($e),a(B)),a(c),g($,e),g(T,e),g(y,e),g(Z,e),g(_,e),g(v,e),g(U,e),g(W,e),g(k,e),g(j,e),g(H,e),g(q,e),g(X,e)}}}const Be='{"title":"Daft","local":"daft","sections":[{"title":"Getting Started","local":"getting-started","sections":[],"depth":2},{"title":"Read","local":"read","sections":[{"title":"Reading an Entire Dataset","local":"reading-an-entire-dataset","sections":[],"depth":3},{"title":"Reading Specific Files","local":"reading-specific-files","sections":[],"depth":3}],"depth":2},{"title":"Write","local":"write","sections":[],"depth":2},{"title":"Authentication","local":"authentication","sections":[],"depth":2}],"depth":1}';function Ne(Je){return He(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ve extends Ye{constructor(c){super(),xe(this,c,Ne,Se,je,{})}}export{Ve as component};
