datasets/pr_8213/en/_app/immutable/nodes/56.ec8f4698.js

import{s as Ua,n as ka,o as Ia}from"../chunks/scheduler.d75c11ed.js";import{S as ba,i as $a,e as p,s as l,c as o,h as _a,a as r,d as t,b as n,f as wa,g as c,j as i,k as ia,l as Ca,m as e,n as h,t as u,o as d,p as f}from"../chunks/index.4ec9dfe9.js";import{C as Ra,H as ra,E as Fa}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.9a52dede.js";import{C as ma}from"../chunks/CodeBlock.37bede1d.js";function Da(oa){let m,Y,Z,W,M,x,g,S,J,ca='This document is a quick introduction to using 🤗 Datasets with Spark, with a particular focus on how to load a Spark DataFrame into a <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object.',z,T,ha="From there, you have fast access to any element and you can use it as a data loader to train models.",B,j,E,w,ua=`A <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object is a wrapper of an Arrow table, which allows fast reads from arrays in the dataset to PyTorch, TensorFlow and JAX tensors.
The Arrow table is memory mapped from disk, which can load datasets bigger than your available RAM.`,X,U,da='You can get a <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Dataset">Dataset</a> from a Spark DataFrame using <code>Dataset.from_spark()</code>:',L,k,H,I,fa='The Spark workers write the dataset on disk in a cache directory as Arrow files, and the <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Dataset">Dataset</a> is loaded from there.',N,b,ya='Alternatively, you can skip materialization by using <code>IterableDataset.from_spark()</code>, which returns an <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>:',V,$,A,_,P,C,Ma=`When using <code>Dataset.from_spark()</code>, the resulting <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Dataset">Dataset</a> is cached; if you call <code>Dataset.from_spark()</code> multiple
times on the same DataFrame it won’t re-run the Spark job that writes the dataset as Arrow files on disk.`,K,R,ga=`You can set the cache location by passing <code>cache_dir=</code> to <code>Dataset.from_spark()</code>.
Make sure to use a disk that is available to both your workers and your current machine (the driver).`,O,y,Ja='<p>In a different session, a Spark DataFrame doesn’t have the same <a href="https://spark.apache.org/docs/3.2.0/api/python/reference/api/pyspark.sql.DataFrame.semanticHash.html" rel="nofollow">semantic hash</a>, and it will rerun a Spark job and store it in a new cache.</p>',aa,F,sa,D,Ta=`If your dataset is made of images, audio data or N-dimensional arrays, you can specify the <code>features=</code> argument in
<code>Dataset.from_spark()</code> (or <code>IterableDataset.from_spark()</code>):`,ta,Q,ea,q,ja='You can check the <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Features">Features</a> documentation to know about all the feature types available.',la,G,na,v,pa;return M=new Ra({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),g=new ra({props:{title:"Use with Spark",local:"use-with-spark",headingTag:"h1"}}),j=new ra({props:{title:"Load from Spark",local:"load-from-spark",headingTag:"h2"}}),k=new ma({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRmJTIwJTNEJTIwc3BhcmsuY3JlYXRlRGF0YUZyYW1lKCUwQSUyMCUyMCUyMCUyMGRhdGElM0QlNUIlNUIxJTJDJTIwJTIyRWxpYSUyMiU1RCUyQyUyMCU1QjIlMkMlMjAlMjJUZW8lMjIlNUQlMkMlMjAlNUIzJTJDJTIwJTIyRmFuZyUyMiU1RCU1RCUyQyUwQSUyMCUyMCUyMCUyMGNvbHVtbnMlM0QlNUIlMjJpZCUyMiUyQyUyMCUyMm5hbWUlMjIlNUQlMkMlMEEpJTBBZHMlMjAlM0QlMjBEYXRhc2V0LmZyb21fc3BhcmsoZGYp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>df = spark.createDataFrame(
<span class="hljs-meta">... </span>    data=[[<span class="hljs-number">1</span>, <span class="hljs-string">&quot;Elia&quot;</span>], [<span class="hljs-number">2</span>, <span class="hljs-string">&quot;Teo&quot;</span>], [<span class="hljs-number">3</span>, <span class="hljs-string">&quot;Fang&quot;</span>]],
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;name&quot;</span>],
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_spark(df)`,lang:"py",wrap:!1}}),$=new ma({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwSXRlcmFibGVEYXRhc2V0JTBBZGYlMjAlM0QlMjBzcGFyay5jcmVhdGVEYXRhRnJhbWUoJTBBJTIwJTIwJTIwJTIwZGF0YSUzRCU1QiU1QjElMkMlMjAlMjJFbGlhJTIyJTVEJTJDJTIwJTVCMiUyQyUyMCUyMlRlbyUyMiU1RCUyQyUyMCU1QjMlMkMlMjAlMjJGYW5nJTIyJTVEJTVEJTJDJTBBJTIwJTIwJTIwJTIwY29sdW1ucyUzRCU1QiUyMmlkJTIyJTJDJTIwJTIybmFtZSUyMiU1RCUyQyUwQSklMEFkcyUyMCUzRCUyMEl0ZXJhYmxlRGF0YXNldC5mcm9tX3NwYXJrKGRmKSUwQXByaW50KG5leHQoaXRlcihkcykpKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> IterableDataset
<span class="hljs-meta">&gt;&gt;&gt; </span>df = spark.createDataFrame(
<span class="hljs-meta">... </span>    data=[[<span class="hljs-number">1</span>, <span class="hljs-string">&quot;Elia&quot;</span>], [<span class="hljs-number">2</span>, <span class="hljs-string">&quot;Teo&quot;</span>], [<span class="hljs-number">3</span>, <span class="hljs-string">&quot;Fang&quot;</span>]],
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;name&quot;</span>],
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = IterableDataset.from_spark(df)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(ds)))
{<span class="hljs-string">&quot;id&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;name&quot;</span>: <span class="hljs-string">&quot;Elia&quot;</span>}`,lang:"py",wrap:!1}}),_=new ra({props:{title:"Caching",local:"caching",headingTag:"h3"}}),F=new ra({props:{title:"Feature types",local:"feature-types",headingTag:"h3"}}),Q=new ma({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwSW1hZ2UlMkMlMjBWYWx1ZSUwQWRhdGElMjAlM0QlMjAlNUIoMCUyQyUyMG9wZW4oJTIyaW1hZ2UucG5nJTIyJTJDJTIwJTIycmIlMjIpLnJlYWQoKSklNUQlMEFkZiUyMCUzRCUyMHNwYXJrLmNyZWF0ZURhdGFGcmFtZShkYXRhJTJDJTIwJTIyaWR4JTNBJTIwaW50JTJDJTIwaW1hZ2UlM0ElMjBiaW5hcnklMjIpJTBBJTIzJTIwQWxzbyUyMHdvcmtzJTIwaWYlMjB5b3UlMjBoYXZlJTIwYXJyYXlzJTBBJTIzJTIwZGF0YSUyMCUzRCUyMCU1QigwJTJDJTIwbnAuemVyb3Moc2hhcGUlM0QoMzIlMkMlMjAzMiUyQyUyMDMpJTJDJTIwZHR5cGUlM0RucC5pbnQzMikudG9saXN0KCkpJTVEJTBBJTIzJTIwZGYlMjAlM0QlMjBzcGFyay5jcmVhdGVEYXRhRnJhbWUoZGF0YSUyQyUyMCUyMmlkeCUzQSUyMGludCUyQyUyMGltYWdlJTNBJTIwYXJyYXklM0NhcnJheSUzQ2FycmF5JTNDaW50JTNFJTNFJTNFJTIyKSUwQWZlYXR1cmVzJTIwJTNEJTIwRmVhdHVyZXMoJTdCJTIyaWR4JTIyJTNBJTIwVmFsdWUoJTIyaW50NjQlMjIpJTJDJTIwJTIyaW1hZ2UlMjIlM0ElMjBJbWFnZSgpJTdEKSUwQWRhdGFzZXQlMjAlM0QlMjBEYXRhc2V0LmZyb21fc3BhcmsoZGYlMkMlMjBmZWF0dXJlcyUzRGZlYXR1cmVzKSUwQWRhdGFzZXQlNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Image, Value
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [(<span class="hljs-number">0</span>, <span class="hljs-built_in">open</span>(<span class="hljs-string">&quot;image.png&quot;</span>, <span class="hljs-string">&quot;rb&quot;</span>).read())]
<span class="hljs-meta">&gt;&gt;&gt; </span>df = spark.createDataFrame(data, <span class="hljs-string">&quot;idx: int, image: binary&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Also works if you have arrays</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># data = [(0, np.zeros(shape=(32, 32, 3), dtype=np.int32).tolist())]</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># df = spark.createDataFrame(data, &quot;idx: int, image: array&lt;array&lt;array&lt;int&gt;&gt;&gt;&quot;)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;idx&quot;</span>: Value(<span class="hljs-string">&quot;int64&quot;</span>), <span class="hljs-string">&quot;image&quot;</span>: Image()})
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = Dataset.from_spark(df, features=features)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;idx&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;image&#x27;</span>: &lt;PIL.PngImagePlugin.PngImageFile image mode=RGB size=32x32&gt;}`,lang:"py",wrap:!1}}),G=new Fa({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/use_with_spark.mdx"}}),{c(){m=p("meta"),Y=l(),Z=p("p"),W=l(),o(M.$$.fragment),x=l(),o(g.$$.fragment),S=l(),J=p("p"),J.innerHTML=ca,z=l(),T=p("p"),T.textContent=ha,B=l(),o(j.$$.fragment),E=l(),w=p("p"),w.innerHTML=ua,X=l(),U=p("p"),U.innerHTML=da,L=l(),o(k.$$.fragment),H=l(),I=p("p"),I.innerHTML=fa,N=l(),b=p("p"),b.innerHTML=ya,V=l(),o($.$$.fragment),A=l(),o(_.$$.fragment),P=l(),C=p("p"),C.innerHTML=Ma,K=l(),R=p("p"),R.innerHTML=ga,O=l(),y=p("blockquote"),y.innerHTML=Ja,aa=l(),o(F.$$.fragment),sa=l(),D=p("p"),D.innerHTML=Ta,ta=l(),o(Q.$$.fragment),ea=l(),q=p("p"),q.innerHTML=ja,la=l(),o(G.$$.fragment),na=l(),v=p("p"),this.h()},l(a){const s=_a("svelte-u9bgzb",document.head);m=r(s,"META",{name:!0,content:!0}),s.forEach(t),Y=n(a),Z=r(a,"P",{}),wa(Z).forEach(t),W=n(a),c(M.$$.fragment,a),x=n(a),c(g.$$.fragment,a),S=n(a),J=r(a,"P",{"data-svelte-h":!0}),i(J)!=="svelte-186ui0d"&&(J.innerHTML=ca),z=n(a),T=r(a,"P",{"data-svelte-h":!0}),i(T)!=="svelte-1pvlf1d"&&(T.textContent=ha),B=n(a),c(j.$$.fragment,a),E=n(a),w=r(a,"P",{"data-svelte-h":!0}),i(w)!=="svelte-124iwjf"&&(w.innerHTML=ua),X=n(a),U=r(a,"P",{"data-svelte-h":!0}),i(U)!=="svelte-8j7i"&&(U.innerHTML=da),L=n(a),c(k.$$.fragment,a),H=n(a),I=r(a,"P",{"data-svelte-h":!0}),i(I)!=="svelte-jewixu"&&(I.innerHTML=fa),N=n(a),b=r(a,"P",{"data-svelte-h":!0}),i(b)!=="svelte-l4hof8"&&(b.innerHTML=ya),V=n(a),c($.$$.fragment,a),A=n(a),c(_.$$.fragment,a),P=n(a),C=r(a,"P",{"data-svelte-h":!0}),i(C)!=="svelte-ofv2sr"&&(C.innerHTML=Ma),K=n(a),R=r(a,"P",{"data-svelte-h":!0}),i(R)!=="svelte-n6wfk7"&&(R.innerHTML=ga),O=n(a),y=r(a,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),i(y)!=="svelte-hb8r4o"&&(y.innerHTML=Ja),aa=n(a),c(F.$$.fragment,a),sa=n(a),D=r(a,"P",{"data-svelte-h":!0}),i(D)!=="svelte-9b4dmt"&&(D.innerHTML=Ta),ta=n(a),c(Q.$$.fragment,a),ea=n(a),q=r(a,"P",{"data-svelte-h":!0}),i(q)!=="svelte-1i4w2gm"&&(q.innerHTML=ja),la=n(a),c(G.$$.fragment,a),na=n(a),v=r(a,"P",{}),wa(v).forEach(t),this.h()},h(){ia(m,"name","hf:doc:metadata"),ia(m,"content",Qa),ia(y,"class","warning")},m(a,s){Ca(document.head,m),e(a,Y,s),e(a,Z,s),e(a,W,s),h(M,a,s),e(a,x,s),h(g,a,s),e(a,S,s),e(a,J,s),e(a,z,s),e(a,T,s),e(a,B,s),h(j,a,s),e(a,E,s),e(a,w,s),e(a,X,s),e(a,U,s),e(a,L,s),h(k,a,s),e(a,H,s),e(a,I,s),e(a,N,s),e(a,b,s),e(a,V,s),h($,a,s),e(a,A,s),h(_,a,s),e(a,P,s),e(a,C,s),e(a,K,s),e(a,R,s),e(a,O,s),e(a,y,s),e(a,aa,s),h(F,a,s),e(a,sa,s),e(a,D,s),e(a,ta,s),h(Q,a,s),e(a,ea,s),e(a,q,s),e(a,la,s),h(G,a,s),e(a,na,s),e(a,v,s),pa=!0},p:ka,i(a){pa||(u(M.$$.fragment,a),u(g.$$.fragment,a),u(j.$$.fragment,a),u(k.$$.fragment,a),u($.$$.fragment,a),u(_.$$.fragment,a),u(F.$$.fragment,a),u(Q.$$.fragment,a),u(G.$$.fragment,a),pa=!0)},o(a){d(M.$$.fragment,a),d(g.$$.fragment,a),d(j.$$.fragment,a),d(k.$$.fragment,a),d($.$$.fragment,a),d(_.$$.fragment,a),d(F.$$.fragment,a),d(Q.$$.fragment,a),d(G.$$.fragment,a),pa=!1},d(a){a&&(t(Y),t(Z),t(W),t(x),t(S),t(J),t(z),t(T),t(B),t(E),t(w),t(X),t(U),t(L),t(H),t(I),t(N),t(b),t(V),t(A),t(P),t(C),t(K),t(R),t(O),t(y),t(aa),t(sa),t(D),t(ta),t(ea),t(q),t(la),t(na),t(v)),t(m),f(M,a),f(g,a),f(j,a),f(k,a),f($,a),f(_,a),f(F,a),f(Q,a),f(G,a)}}}const Qa='{"title":"Use with Spark","local":"use-with-spark","sections":[{"title":"Load from Spark","local":"load-from-spark","sections":[{"title":"Caching","local":"caching","sections":[],"depth":3},{"title":"Feature types","local":"feature-types","sections":[],"depth":3}],"depth":2}],"depth":1}';function qa(oa){return Ia(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wa extends ba{constructor(m){super(),$a(this,m,qa,Da,Ua,{})}}export{Wa as component};