datasets/pr_8250/en/_app/immutable/nodes/57.bfbfc35b.js

import{s as qs,n as ks,o as As}from"../chunks/scheduler.d75c11ed.js";import{S as Cs,i as Es,e as p,s as l,c,h as xs,a as r,d as t,b as n,f as Is,g as i,j as o,k as Us,l as Hs,m as e,n as h,t as m,o as u,p as d}from"../chunks/index.4ec9dfe9.js";import{C as Ds,H as hs,E as Ps}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.0a461128.js";import{C as ms}from"../chunks/CodeBlock.93a8538d.js";function vs(us){let j,L,Q,R,f,z,M,Z,y,ds=`This document is a quick introduction to using <code>datasets</code> with PyArrow, with a particular focus on how to process
datasets using Arrow compute functions, and how to convert a dataset to PyArrow or from PyArrow.`,F,g,js="This is particularly useful as it allows fast zero-copy operations, since <code>datasets</code> uses PyArrow under the hood.",B,w,X,b,fs="By default, datasets return regular Python objects: integers, floats, strings, lists, etc.",G,T,Ms='To get PyArrow Tables or Arrays instead, you can set the format of the dataset to <code>pyarrow</code> using <a href="/docs/datasets/pr_8250/en/package_reference/main_classes#datasets.Dataset.with_format">Dataset.with_format()</a>:',S,_,W,J,ys="This also works for <code>IterableDataset</code> objects obtained e.g. using <code>load_dataset(..., streaming=True)</code>:",N,$,V,I,K,U,gs='PyArrow functions are generally faster than regular hand-written python functions, and therefore they are a good option to optimize data processing. You can use Arrow compute functions to process a dataset in <a href="/docs/datasets/pr_8250/en/package_reference/main_classes#datasets.Dataset.map">Dataset.map()</a> or <a href="/docs/datasets/pr_8250/en/package_reference/main_classes#datasets.Dataset.filter">Dataset.filter()</a>:',O,q,ss,k,ws="We use <code>batched=True</code> because it is faster to process batches of data in PyArrow rather than row by row. It’s also possible to use <code>batch_size=</code> in <code>map()</code> to set the size of each <code>table</code>.",as,A,bs='This also works for <a href="/docs/datasets/pr_8250/en/package_reference/main_classes#datasets.IterableDataset.map">IterableDataset.map()</a> and <a href="/docs/datasets/pr_8250/en/package_reference/main_classes#datasets.IterableDataset.filter">IterableDataset.filter()</a>.',ts,C,es,E,Ts='A <a href="/docs/datasets/pr_8250/en/package_reference/main_classes#datasets.Dataset">Dataset</a> is a wrapper of a PyArrow Table, you can instantiate a Dataset directly from the Table:',ls,x,ns,H,_s='You can access the PyArrow Table of a dataset using <a href="/docs/datasets/pr_8250/en/package_reference/main_classes#datasets.Dataset.data">Dataset.data</a>, which returns a <code>MemoryMappedTable</code> or a <code>InMemoryTable</code> or a <code>ConcatenationTable</code>, depending on the origin of the Arrow data and the operations that were applied.',ps,D,Js='Those objects wrap the underlying PyArrow table accessible at <code>Dataset.data.table</code>. This table contains all the data of the dataset, but there might also be an indices mapping at <code>Dataset._indices</code> which maps the dataset rows indices to the PyArrow Table rows indices. This can happen if the dataset has been shuffled with <a href="/docs/datasets/pr_8250/en/package_reference/main_classes#datasets.Dataset.shuffle">Dataset.shuffle()</a> or if only a subset of the rows are used (e.g. after a <a href="/docs/datasets/pr_8250/en/package_reference/main_classes#datasets.Dataset.select">Dataset.select()</a>).',rs,P,$s="In the general case, you can export a dataset to a PyArrow Table using <code>table = ds.with_format(&quot;arrow&quot;)[:]</code>.",os,v,cs,Y,is;return f=new Ds({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),M=new hs({props:{title:"Use with PyArrow",local:"use-with-pyarrow",headingTag:"h1"}}),w=new hs({props:{title:"Dataset format",local:"dataset-format",headingTag:"h2"}}),_=new ms({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGElMjAlM0QlMjAlN0IlMjJjb2xfMCUyMiUzQSUyMCU1QiUyMmElMjIlMkMlMjAlMjJiJTIyJTJDJTIwJTIyYyUyMiUyQyUyMCUyMmQlMjIlNUQlMkMlMjAlMjJjb2xfMSUyMiUzQSUyMCU1QjAuJTJDJTIwMC4lMkMlMjAxLiUyQyUyMDEuJTVEJTdEJTBBZHMlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdChkYXRhKSUwQWRzJTIwJTNEJTIwZHMud2l0aF9mb3JtYXQoJTIyYXJyb3clMjIpJTBBZHMlNUIwJTVEJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwcGEuVGFibGUlMEFkcyU1QiUzQTIlNUQlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBwYS5UYWJsZSUwQWRzJTVCJTIyZGF0YSUyMiU1RCUyMCUyMCUyMyUyMHBhLmFycmF5",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = {<span class="hljs-string">&quot;col_0&quot;</span>: [<span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;c&quot;</span>, <span class="hljs-string">&quot;d&quot;</span>], <span class="hljs-string">&quot;col_1&quot;</span>: [<span class="hljs-number">0.</span>, <span class="hljs-number">0.</span>, <span class="hljs-number">1.</span>, <span class="hljs-number">1.</span>]}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict(data)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;arrow&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]       <span class="hljs-comment"># pa.Table</span>
pyarrow.Table
col_0: string
col_1: double
----
col_0: [[<span class="hljs-string">&quot;a&quot;</span>]]
col_1: [[<span class="hljs-number">0</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]      <span class="hljs-comment"># pa.Table</span>
pyarrow.Table
col_0: string
col_1: double
----
col_0: [[<span class="hljs-string">&quot;a&quot;</span>,<span class="hljs-string">&quot;b&quot;</span>]]
col_1: [[<span class="hljs-number">0</span>,<span class="hljs-number">0</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-string">&quot;data&quot;</span>]  <span class="hljs-comment"># pa.array</span>
&lt;pyarrow.lib.ChunkedArray <span class="hljs-built_in">object</span> at <span class="hljs-number">0x1394312a0</span>&gt;
[
  [
    <span class="hljs-string">&quot;a&quot;</span>,
    <span class="hljs-string">&quot;b&quot;</span>,
    <span class="hljs-string">&quot;c&quot;</span>,
    <span class="hljs-string">&quot;d&quot;</span>
  ]
]`,lang:"py",wrap:!1}}),$=new ms({props:{code:"ZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJhcnJvdyUyMiklMEFmb3IlMjB0YWJsZSUyMGluJTIwZHMuaXRlcihiYXRjaF9zaXplJTNEMiklM0ElMEElMjAlMjAlMjAlMjBwcmludCh0YWJsZSklMEElMjAlMjAlMjAlMjBicmVhaw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;arrow&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> table <span class="hljs-keyword">in</span> ds.<span class="hljs-built_in">iter</span>(batch_size=<span class="hljs-number">2</span>):
<span class="hljs-meta">... </span>    <span class="hljs-built_in">print</span>(table)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">break</span>
pyarrow.Table
col_0: string
col_1: double
----
col_0: [[<span class="hljs-string">&quot;a&quot;</span>,<span class="hljs-string">&quot;b&quot;</span>]]
col_1: [[<span class="hljs-number">0</span>,<span class="hljs-number">0</span>]]`,lang:"py",wrap:!1}}),I=new hs({props:{title:"Process data",local:"process-data",headingTag:"h2"}}),q=new ms({props:{code:"aW1wb3J0JTIwcHlhcnJvdy5jb21wdXRlJTIwYXMlMjBwYyUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMERhdGFzZXQlMEFkYXRhJTIwJTNEJTIwJTdCJTIyY29sXzAlMjIlM0ElMjAlNUIlMjJhJTIyJTJDJTIwJTIyYiUyMiUyQyUyMCUyMmMlMjIlMkMlMjAlMjJkJTIyJTVEJTJDJTIwJTIyY29sXzElMjIlM0ElMjAlNUIwLiUyQyUyMDAuJTJDJTIwMS4lMkMlMjAxLiU1RCU3RCUwQWRzJTIwJTNEJTIwRGF0YXNldC5mcm9tX2RpY3QoZGF0YSklMEFkcyUyMCUzRCUyMGRzLndpdGhfZm9ybWF0KCUyMmFycm93JTIyKSUwQWRzJTIwJTNEJTIwZHMubWFwKGxhbWJkYSUyMHQlM0ElMjB0LmFwcGVuZF9jb2x1bW4oJTIyY29sXzIlMjIlMkMlMjBwYy5hZGQodCU1QiUyMmNvbF8xJTIyJTVEJTJDJTIwMSkpJTJDJTIwYmF0Y2hlZCUzRFRydWUpJTBBZHMlNUIlM0EyJTVEJTBBZHMlMjAlM0QlMjBkcy5maWx0ZXIobGFtYmRhJTIwdCUzQSUyMHBjLmVxdWFsKHQlNUIlMjJjb2xfMCUyMiU1RCUyQyUyMCUyMmIlMjIpJTJDJTIwYmF0Y2hlZCUzRFRydWUpJTBBZHMlNUIwJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyarrow.compute <span class="hljs-keyword">as</span> pc
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = {<span class="hljs-string">&quot;col_0&quot;</span>: [<span class="hljs-string">&quot;a&quot;</span>, <span class="hljs-string">&quot;b&quot;</span>, <span class="hljs-string">&quot;c&quot;</span>, <span class="hljs-string">&quot;d&quot;</span>], <span class="hljs-string">&quot;col_1&quot;</span>: [<span class="hljs-number">0.</span>, <span class="hljs-number">0.</span>, <span class="hljs-number">1.</span>, <span class="hljs-number">1.</span>]}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict(data)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;arrow&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> t: t.append_column(<span class="hljs-string">&quot;col_2&quot;</span>, pc.add(t[<span class="hljs-string">&quot;col_1&quot;</span>], <span class="hljs-number">1</span>)), batched=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
pyarrow.Table
col_0: string
col_1: double
col_2: double
----
col_0: [[<span class="hljs-string">&quot;a&quot;</span>,<span class="hljs-string">&quot;b&quot;</span>]]
col_1: [[<span class="hljs-number">0</span>,<span class="hljs-number">0</span>]]
col_2: [[<span class="hljs-number">1</span>,<span class="hljs-number">1</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> t: pc.equal(t[<span class="hljs-string">&quot;col_0&quot;</span>], <span class="hljs-string">&quot;b&quot;</span>), batched=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
pyarrow.Table
col_0: string
col_1: double
col_2: double
----
col_0: [[<span class="hljs-string">&quot;b&quot;</span>]]
col_1: [[<span class="hljs-number">0</span>]]
col_2: [[<span class="hljs-number">1</span>]]`,lang:"python",wrap:!1}}),C=new hs({props:{title:"Import or Export from PyArrow",local:"import-or-export-from-pyarrow",headingTag:"h2"}}),x=new ms({props:{code:"ZHMlMjAlM0QlMjBEYXRhc2V0KHRhYmxlKQ==",highlighted:"ds = Dataset(table)",lang:"python",wrap:!1}}),v=new Ps({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/use_with_pyarrow.mdx"}}),{c(){j=p("meta"),L=l(),Q=p("p"),R=l(),c(f.$$.fragment),z=l(),c(M.$$.fragment),Z=l(),y=p("p"),y.innerHTML=ds,F=l(),g=p("p"),g.innerHTML=js,B=l(),c(w.$$.fragment),X=l(),b=p("p"),b.textContent=fs,G=l(),T=p("p"),T.innerHTML=Ms,S=l(),c(_.$$.fragment),W=l(),J=p("p"),J.innerHTML=ys,N=l(),c($.$$.fragment),V=l(),c(I.$$.fragment),K=l(),U=p("p"),U.innerHTML=gs,O=l(),c(q.$$.fragment),ss=l(),k=p("p"),k.innerHTML=ws,as=l(),A=p("p"),A.innerHTML=bs,ts=l(),c(C.$$.fragment),es=l(),E=p("p"),E.innerHTML=Ts,ls=l(),c(x.$$.fragment),ns=l(),H=p("p"),H.innerHTML=_s,ps=l(),D=p("p"),D.innerHTML=Js,rs=l(),P=p("p"),P.innerHTML=$s,os=l(),c(v.$$.fragment),cs=l(),Y=p("p"),this.h()},l(s){const a=xs("svelte-u9bgzb",document.head);j=r(a,"META",{name:!0,content:!0}),a.forEach(t),L=n(s),Q=r(s,"P",{}),Is(Q).forEach(t),R=n(s),i(f.$$.fragment,s),z=n(s),i(M.$$.fragment,s),Z=n(s),y=r(s,"P",{"data-svelte-h":!0}),o(y)!=="svelte-o6efn3"&&(y.innerHTML=ds),F=n(s),g=r(s,"P",{"data-svelte-h":!0}),o(g)!=="svelte-syebji"&&(g.innerHTML=js),B=n(s),i(w.$$.fragment,s),X=n(s),b=r(s,"P",{"data-svelte-h":!0}),o(b)!=="svelte-ej8pz8"&&(b.textContent=fs),G=n(s),T=r(s,"P",{"data-svelte-h":!0}),o(T)!=="svelte-38iozr"&&(T.innerHTML=Ms),S=n(s),i(_.$$.fragment,s),W=n(s),J=r(s,"P",{"data-svelte-h":!0}),o(J)!=="svelte-2qljfh"&&(J.innerHTML=ys),N=n(s),i($.$$.fragment,s),V=n(s),i(I.$$.fragment,s),K=n(s),U=r(s,"P",{"data-svelte-h":!0}),o(U)!=="svelte-1bjai4b"&&(U.innerHTML=gs),O=n(s),i(q.$$.fragment,s),ss=n(s),k=r(s,"P",{"data-svelte-h":!0}),o(k)!=="svelte-4b4wdc"&&(k.innerHTML=ws),as=n(s),A=r(s,"P",{"data-svelte-h":!0}),o(A)!=="svelte-r8qc5d"&&(A.innerHTML=bs),ts=n(s),i(C.$$.fragment,s),es=n(s),E=r(s,"P",{"data-svelte-h":!0}),o(E)!=="svelte-zejnd8"&&(E.innerHTML=Ts),ls=n(s),i(x.$$.fragment,s),ns=n(s),H=r(s,"P",{"data-svelte-h":!0}),o(H)!=="svelte-53ksuo"&&(H.innerHTML=_s),ps=n(s),D=r(s,"P",{"data-svelte-h":!0}),o(D)!=="svelte-1f603e3"&&(D.innerHTML=Js),rs=n(s),P=r(s,"P",{"data-svelte-h":!0}),o(P)!=="svelte-oy3ykb"&&(P.innerHTML=$s),os=n(s),i(v.$$.fragment,s),cs=n(s),Y=r(s,"P",{}),Is(Y).forEach(t),this.h()},h(){Us(j,"name","hf:doc:metadata"),Us(j,"content",Qs)},m(s,a){Hs(document.head,j),e(s,L,a),e(s,Q,a),e(s,R,a),h(f,s,a),e(s,z,a),h(M,s,a),e(s,Z,a),e(s,y,a),e(s,F,a),e(s,g,a),e(s,B,a),h(w,s,a),e(s,X,a),e(s,b,a),e(s,G,a),e(s,T,a),e(s,S,a),h(_,s,a),e(s,W,a),e(s,J,a),e(s,N,a),h($,s,a),e(s,V,a),h(I,s,a),e(s,K,a),e(s,U,a),e(s,O,a),h(q,s,a),e(s,ss,a),e(s,k,a),e(s,as,a),e(s,A,a),e(s,ts,a),h(C,s,a),e(s,es,a),e(s,E,a),e(s,ls,a),h(x,s,a),e(s,ns,a),e(s,H,a),e(s,ps,a),e(s,D,a),e(s,rs,a),e(s,P,a),e(s,os,a),h(v,s,a),e(s,cs,a),e(s,Y,a),is=!0},p:ks,i(s){is||(m(f.$$.fragment,s),m(M.$$.fragment,s),m(w.$$.fragment,s),m(_.$$.fragment,s),m($.$$.fragment,s),m(I.$$.fragment,s),m(q.$$.fragment,s),m(C.$$.fragment,s),m(x.$$.fragment,s),m(v.$$.fragment,s),is=!0)},o(s){u(f.$$.fragment,s),u(M.$$.fragment,s),u(w.$$.fragment,s),u(_.$$.fragment,s),u($.$$.fragment,s),u(I.$$.fragment,s),u(q.$$.fragment,s),u(C.$$.fragment,s),u(x.$$.fragment,s),u(v.$$.fragment,s),is=!1},d(s){s&&(t(L),t(Q),t(R),t(z),t(Z),t(y),t(F),t(g),t(B),t(X),t(b),t(G),t(T),t(S),t(W),t(J),t(N),t(V),t(K),t(U),t(O),t(ss),t(k),t(as),t(A),t(ts),t(es),t(E),t(ls),t(ns),t(H),t(ps),t(D),t(rs),t(P),t(os),t(cs),t(Y)),t(j),d(f,s),d(M,s),d(w,s),d(_,s),d($,s),d(I,s),d(q,s),d(C,s),d(x,s),d(v,s)}}}const Qs='{"title":"Use with PyArrow","local":"use-with-pyarrow","sections":[{"title":"Dataset format","local":"dataset-format","sections":[],"depth":2},{"title":"Process data","local":"process-data","sections":[],"depth":2},{"title":"Import or Export from PyArrow","local":"import-or-export-from-pyarrow","sections":[],"depth":2}],"depth":1}';function Ys(us){return As(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fs extends Cs{constructor(j){super(),Es(this,j,Ys,vs,qs,{})}}export{Fs as component};