Buckets:

hf-doc-build
/

doc-dev

hf-doc-build/doc-dev / datasets /pr_8240 /en /_app /immutable /nodes /53.b555fc4e.js

HuggingFaceDocBuilder's picture

HuggingFaceDocBuilder

about 1 month ago

33.3 kB

	import{s as ga,n as Ma,o as ya}from"../chunks/scheduler.d75c11ed.js";import{S as Ja,i as ba,e as p,s as e,c as i,h as Ua,a as r,d as t,b as n,f as ua,g as h,j as c,k as ts,l as Ta,m as l,n as m,t as o,o as j,p as d}from"../chunks/index.4ec9dfe9.js";import{C as fa,H as ls,E as wa}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.5d95b9a7.js";import{C as u}from"../chunks/CodeBlock.275b1754.js";function Ca(Ws){let g,ps,es,rs,U,cs,T,is,f,Ls=`This document is a quick introduction to using <code>datasets</code> with JAX, with a particular focus on how to get
	<code>jax.Array</code> objects out of our datasets, and how to use them to train JAX models.`,hs,M,Ss=`<p><code>jax</code> and <code>jaxlib</code> are required to reproduce to code above, so please make sure you
	install them as <code>pip install datasets[jax]</code>.</p>`,ms,w,os,C,Ps=`By default, datasets return regular Python objects: integers, floats, strings, lists, etc., and
	string and binary objects are unchanged, since JAX only supports numbers.`,js,I,Ks="To get JAX arrays (numpy-like) instead, you can set the format of the dataset to <code>jax</code>:",ds,R,us,y,Os='<p>A <a href="/docs/datasets/pr_8240/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object is a wrapper of an Arrow table, which allows fast reads from arrays in the dataset to JAX arrays.</p>',gs,$,sa=`Note that the exact same procedure applies to <code>DatasetDict</code> objects, so that
	when setting the format of a <code>DatasetDict</code> to <code>jax</code>, all the <code>Dataset</code>s there
	will be formatted as <code>jax</code>:`,Ms,k,ys,Z,aa=`Another thing you’ll need to take into consideration is that the formatting is not applied
	until you actually access the data. So if you want to get a JAX array out of a dataset,
	you’ll need to access the data first, otherwise the format will remain the same.`,Js,x,ta=`Finally, to load the data in the device of your choice, you can specify the <code>device</code> argument,
	but note that <code>jaxlib.xla_extension.Device</code> is not supported as it’s not serializable with neither
	<code>pickle</code> not <code>dill</code>, so you’ll need to use its string identifier instead:`,bs,Q,Us,X,la=`Note that if the <code>device</code> argument is not provided to <code>with_format</code> then it will use the default
	device which is <code>jax.devices()[0]</code>.`,Ts,N,fs,q,ea="If your dataset consists of N-dimensional arrays, you will see that by default they are considered as the same tensor if the shape is fixed:",ws,v,Cs,A,Is,G,na=`However this logic often requires slow shape comparisons and data copies.
	To avoid this, you must explicitly use the <code>Array</code> feature type and specify the shape of your tensors:`,Rs,E,$s,_,ks,F,pa='<a href="/docs/datasets/pr_8240/en/package_reference/main_classes#datasets.ClassLabel">ClassLabel</a> data is properly converted to arrays:',Zs,V,xs,z,ra="String and binary objects are unchanged, since JAX only supports numbers.",Qs,Y,ca='The <a href="/docs/datasets/pr_8240/en/package_reference/main_classes#datasets.Image">Image</a> and <a href="/docs/datasets/pr_8240/en/package_reference/main_classes#datasets.Audio">Audio</a> feature types are also supported.',Xs,J,ia=`<p>To use the <a href="/docs/datasets/pr_8240/en/package_reference/main_classes#datasets.Image">Image</a> feature type, you’ll need to install the <code>vision</code> extra as
	<code>pip install datasets[vision]</code>.</p>`,Ns,B,qs,b,ha=`<p>To use the <a href="/docs/datasets/pr_8240/en/package_reference/main_classes#datasets.Audio">Audio</a> feature type, you’ll need to install the <code>audio</code> extra as
	<code>pip install datasets[audio]</code>.</p>`,vs,D,As,H,Gs,W,ma=`JAX doesn’t have any built-in data loading capabilities, so you’ll need to use a library such
	as <a href="https://pytorch.org/" rel="nofollow">PyTorch</a> to load your data using a <code>DataLoader</code> or <a href="https://www.tensorflow.org/" rel="nofollow">TensorFlow</a>
	using a <code>tf.data.Dataset</code>. Citing the <a href="https://jax.readthedocs.io/en/latest/notebooks/Neural_Network_and_Data_Loading.html#data-loading-with-pytorch" rel="nofollow">JAX documentation</a> on this topic:
	“JAX is laser-focused on program transformations and accelerator-backed NumPy, so we don’t
	include data loading or munging in the JAX library. There are already a lot of great data loaders
	out there, so let’s just use them instead of reinventing anything. We’ll grab PyTorch’s data loader,
	and make a tiny shim to make it work with NumPy arrays.”.`,Es,L,oa=`So that’s the reason why JAX-formatting in <code>datasets</code> is so useful, because it lets you use
	any model from the HuggingFace Hub with JAX, without having to worry about the data loading
	part.`,_s,S,Fs,P,ja=`The easiest way to get JAX arrays out of a dataset is to use the <code>with_format('jax')</code> method. Lets assume
	that we want to train a neural network on the <a href="http://yann.lecun.com/exdb/mnist/" rel="nofollow">MNIST dataset</a> available
	at the HuggingFace Hub at <a href="https://huggingface.co/datasets/ylecun/mnist" rel="nofollow">https://huggingface.co/datasets/ylecun/mnist</a>.`,Vs,K,zs,O,da=`Once the format is set we can feed the dataset to the JAX model in batches using the <code>Dataset.iter()</code>
	method:`,Ys,ss,Bs,as,Ds,ns,Hs;return U=new fa({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),T=new ls({props:{title:"Use with JAX",local:"use-with-jax",headingTag:"h1"}}),w=new ls({props:{title:"Dataset format",local:"dataset-format",headingTag:"h2"}}),R=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGElMjAlM0QlMjAlNUIlNUIxJTJDJTIwMiU1RCUyQyUyMCU1QjMlMkMlMjA0JTVEJTVEJTBBZHMlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglN0IlMjJkYXRhJTIyJTNBJTIwZGF0YSU3RCklMEFkcyUyMCUzRCUyMGRzLndpdGhfZm9ybWF0KCUyMmpheCUyMiklMEFkcyU1QjAlNUQlMEFkcyU1QiUzQTIlNUQ=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
	<span class="hljs-meta">>>> </span>data = [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"data"</span>: data})
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"jax"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'data'</span>: DeviceArray([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], dtype=int32)}
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">2</span>]
	{<span class="hljs-string">'data'</span>: DeviceArray([
	[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]], dtype=int32)}`,lang:"py",wrap:!1}}),k=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldERpY3QlMEFkYXRhJTIwJTNEJTIwJTdCJTIydHJhaW4lMjIlM0ElMjAlN0IlMjJkYXRhJTIyJTNBJTIwJTVCJTVCMSUyQyUyMDIlNUQlMkMlMjAlNUIzJTJDJTIwNCU1RCU1RCU3RCUyQyUyMCUyMnRlc3QlMjIlM0ElMjAlN0IlMjJkYXRhJTIyJTNBJTIwJTVCJTVCNSUyQyUyMDYlNUQlMkMlMjAlNUI3JTJDJTIwOCU1RCU1RCU3RCU3RCUwQWRkcyUyMCUzRCUyMERhdGFzZXREaWN0LmZyb21fZGljdChkYXRhKSUwQWRkcyUyMCUzRCUyMGRkcy53aXRoX2Zvcm1hdCglMjJqYXglMjIpJTBBZGRzJTVCJTIydHJhaW4lMjIlNUQlNUIlM0EyJTVE",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> DatasetDict
	<span class="hljs-meta">>>> </span>data = {<span class="hljs-string">"train"</span>: {<span class="hljs-string">"data"</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]}, <span class="hljs-string">"test"</span>: {<span class="hljs-string">"data"</span>: [[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>], [<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]}}
	<span class="hljs-meta">>>> </span>dds = DatasetDict.from_dict(data)
	<span class="hljs-meta">>>> </span>dds = dds.with_format(<span class="hljs-string">"jax"</span>)
	<span class="hljs-meta">>>> </span>dds[<span class="hljs-string">"train"</span>][:<span class="hljs-number">2</span>]
	{<span class="hljs-string">'data'</span>: DeviceArray([
	[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]], dtype=int32)}`,lang:"py",wrap:!1}}),Q=new u({props:{code:"aW1wb3J0JTIwamF4JTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGElMjAlM0QlMjAlNUIlNUIxJTJDJTIwMiU1RCUyQyUyMCU1QjMlMkMlMjA0JTVEJTVEJTBBZHMlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglN0IlMjJkYXRhJTIyJTNBJTIwZGF0YSU3RCklMEFkZXZpY2UlMjAlM0QlMjBzdHIoamF4LmRldmljZXMoKSU1QjAlNUQpJTIwJTIwJTIzJTIwTm90JTIwY2FzdGluZyUyMHRvJTIwJTYwc3RyJTYwJTIwYmVmb3JlJTIwcGFzc2luZyUyMGl0JTIwdG8lMjAlNjB3aXRoX2Zvcm1hdCU2MCUyMHdpbGwlMjByYWlzZSUyMGElMjAlNjBWYWx1ZUVycm9yJTYwJTBBZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJqYXglMjIlMkMlMjBkZXZpY2UlM0RkZXZpY2UpJTBBZHMlNUIwJTVEJTBBZHMlNUIwJTVEJTVCJTIyZGF0YSUyMiU1RC5kZXZpY2UoKSUwQWFzc2VydCUyMGRzJTVCMCU1RCU1QiUyMmRhdGElMjIlNUQuZGV2aWNlKCklMjAlM0QlM0QlMjBqYXguZGV2aWNlcygpJTVCMCU1RA==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> jax
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
	<span class="hljs-meta">>>> </span>data = [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"data"</span>: data})
	<span class="hljs-meta">>>> </span>device = <span class="hljs-built_in">str</span>(jax.devices()[<span class="hljs-number">0</span>]) <span class="hljs-comment"># Not casting to \`str\` before passing it to \`with_format\` will raise a \`ValueError\`</span>
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"jax"</span>, device=device)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'data'</span>: DeviceArray([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], dtype=int32)}
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">"data"</span>].device()
	TFRT_CPU_0
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">assert</span> ds[<span class="hljs-number">0</span>][<span class="hljs-string">"data"</span>].device() == jax.devices()[<span class="hljs-number">0</span>]
	<span class="hljs-literal">True</span>`,lang:"py",wrap:!1}}),N=new ls({props:{title:"N-dimensional arrays",local:"n-dimensional-arrays",headingTag:"h3"}}),v=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGElMjAlM0QlMjAlNUIlNUIlNUIxJTJDJTIwMiU1RCUyQyU1QjMlMkMlMjA0JTVEJTVEJTJDJTIwJTVCJTVCNSUyQyUyMDYlNUQlMkMlNUI3JTJDJTIwOCU1RCU1RCU1RCUyMCUyMCUyMyUyMGZpeGVkJTIwc2hhcGUlMEFkcyUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCU3QiUyMmRhdGElMjIlM0ElMjBkYXRhJTdEKSUwQWRzJTIwJTNEJTIwZHMud2l0aF9mb3JtYXQoJTIyamF4JTIyKSUwQWRzJTVCMCU1RA==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
	<span class="hljs-meta">>>> </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]], [[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]] <span class="hljs-comment"># fixed shape</span>
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"data"</span>: data})
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"jax"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'data'</span>: Array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]], dtype=int32)}`,lang:"py",wrap:!1}}),A=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGElMjAlM0QlMjAlNUIlNUIlNUIxJTJDJTIwMiU1RCUyQyU1QjMlNUQlNUQlMkMlMjAlNUIlNUI0JTJDJTIwNSUyQyUyMDYlNUQlMkMlNUI3JTJDJTIwOCU1RCU1RCU1RCUyMCUyMCUyMyUyMHZhcnlpbmclMjBzaGFwZSUwQWRzJTIwJTNEJTIwRGF0YXNldC5mcm9tX2RpY3QoJTdCJTIyZGF0YSUyMiUzQSUyMGRhdGElN0QpJTBBZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJqYXglMjIpJTBBZHMlNUIwJTVE",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
	<span class="hljs-meta">>>> </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>]], [[<span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]] <span class="hljs-comment"># varying shape</span>
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"data"</span>: data})
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"jax"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'data'</span>: [Array([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], dtype=int32), Array([<span class="hljs-number">3</span>], dtype=int32)]}`,lang:"py",wrap:!1}}),E=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwQXJyYXkyRCUwQWRhdGElMjAlM0QlMjAlNUIlNUIlNUIxJTJDJTIwMiU1RCUyQyU1QjMlMkMlMjA0JTVEJTVEJTJDJTVCJTVCNSUyQyUyMDYlNUQlMkMlNUI3JTJDJTIwOCU1RCU1RCU1RCUwQWZlYXR1cmVzJTIwJTNEJTIwRmVhdHVyZXMoJTdCJTIyZGF0YSUyMiUzQSUyMEFycmF5MkQoc2hhcGUlM0QoMiUyQyUyMDIpJTJDJTIwZHR5cGUlM0QnaW50MzInKSU3RCklMEFkcyUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCU3QiUyMmRhdGElMjIlM0ElMjBkYXRhJTdEJTJDJTIwZmVhdHVyZXMlM0RmZWF0dXJlcyklMEFkcyUyMCUzRCUyMGRzLndpdGhfZm9ybWF0KCUyMmpheCUyMiklMEFkcyU1QjAlNUQlMEFkcyU1QiUzQTIlNUQ=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Array2D
	<span class="hljs-meta">>>> </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
	<span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">"data"</span>: Array2D(shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=<span class="hljs-string">'int32'</span>)})
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"data"</span>: data}, features=features)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"jax"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'data'</span>: Array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]], dtype=int32)}
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">2</span>]
	{<span class="hljs-string">'data'</span>: Array([[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],

	[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],
	[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]], dtype=int32)}`,lang:"py",wrap:!1}}),_=new ls({props:{title:"Other feature types",local:"other-feature-types",headingTag:"h3"}}),V=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwQ2xhc3NMYWJlbCUwQWxhYmVscyUyMCUzRCUyMCU1QjAlMkMlMjAwJTJDJTIwMSU1RCUwQWZlYXR1cmVzJTIwJTNEJTIwRmVhdHVyZXMoJTdCJTIybGFiZWwlMjIlM0ElMjBDbGFzc0xhYmVsKG5hbWVzJTNEJTVCJTIybmVnYXRpdmUlMjIlMkMlMjAlMjJwb3NpdGl2ZSUyMiU1RCklN0QpJTBBZHMlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglN0IlMjJsYWJlbCUyMiUzQSUyMGxhYmVscyU3RCUyQyUyMGZlYXR1cmVzJTNEZmVhdHVyZXMpJTBBZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJqYXglMjIpJTBBZHMlNUIlM0EzJTVE",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, ClassLabel
	<span class="hljs-meta">>>> </span>labels = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
	<span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">"label"</span>: ClassLabel(names=[<span class="hljs-string">"negative"</span>, <span class="hljs-string">"positive"</span>])})
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"label"</span>: labels}, features=features)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"jax"</span>)
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">3</span>]
	{<span class="hljs-string">'label'</span>: DeviceArray([<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>], dtype=int32)}`,lang:"py",wrap:!1}}),B=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwSW1hZ2UlMEFpbWFnZXMlMjAlM0QlMjAlNUIlMjJwYXRoJTJGdG8lMkZpbWFnZS5wbmclMjIlNUQlMjAqJTIwMTAlMEFmZWF0dXJlcyUyMCUzRCUyMEZlYXR1cmVzKCU3QiUyMmltYWdlJTIyJTNBJTIwSW1hZ2UoKSU3RCklMEFkcyUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCU3QiUyMmltYWdlJTIyJTNBJTIwaW1hZ2VzJTdEJTJDJTIwZmVhdHVyZXMlM0RmZWF0dXJlcyklMEFkcyUyMCUzRCUyMGRzLndpdGhfZm9ybWF0KCUyMmpheCUyMiklMEFkcyU1QjAlNUQlNUIlMjJpbWFnZSUyMiU1RC5zaGFwZSUwQWRzJTVCMCU1RCUwQWRzJTVCJTNBMiU1RCU1QiUyMmltYWdlJTIyJTVELnNoYXBlJTBBZHMlNUIlM0EyJTVE",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Image
	<span class="hljs-meta">>>> </span>images = [<span class="hljs-string">"path/to/image.png"</span>] * <span class="hljs-number">10</span>
	<span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">"image"</span>: Image()})
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"image"</span>: images}, features=features)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"jax"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">"image"</span>].shape
	(<span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">3</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'image'</span>: DeviceArray([[[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	...,
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]], dtype=uint8)}
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">2</span>][<span class="hljs-string">"image"</span>].shape
	(<span class="hljs-number">2</span>, <span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">3</span>)
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">2</span>]
	{<span class="hljs-string">'image'</span>: DeviceArray([[[[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	...,
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]]], dtype=uint8)}`,lang:"py",wrap:!1}}),D=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwQXVkaW8lMEFhdWRpbyUyMCUzRCUyMCU1QiUyMnBhdGglMkZ0byUyRmF1ZGlvLndhdiUyMiU1RCUyMColMjAxMCUwQWZlYXR1cmVzJTIwJTNEJTIwRmVhdHVyZXMoJTdCJTIyYXVkaW8lMjIlM0ElMjBBdWRpbygpJTdEKSUwQWRzJTIwJTNEJTIwRGF0YXNldC5mcm9tX2RpY3QoJTdCJTIyYXVkaW8lMjIlM0ElMjBhdWRpbyU3RCUyQyUyMGZlYXR1cmVzJTNEZmVhdHVyZXMpJTBBZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJqYXglMjIpJTBBZHMlNUIwJTVEJTVCJTIyYXVkaW8lMjIlNUQlNUIlMjJhcnJheSUyMiU1RCUwQWRzJTVCMCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTVCJTIyc2FtcGxpbmdfcmF0ZSUyMiU1RA==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio
	<span class="hljs-meta">>>> </span>audio = [<span class="hljs-string">"path/to/audio.wav"</span>] * <span class="hljs-number">10</span>
	<span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">"audio"</span>: Audio()})
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"audio"</span>: audio}, features=features)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"jax"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">"audio"</span>][<span class="hljs-string">"array"</span>]
	DeviceArray([-<span class="hljs-number">0.059021</span> , -<span class="hljs-number">0.03894043</span>, -<span class="hljs-number">0.00735474</span>, ..., <span class="hljs-number">0.0133667</span> ,
	<span class="hljs-number">0.01809692</span>, <span class="hljs-number">0.00268555</span>], dtype=float32)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">"audio"</span>][<span class="hljs-string">"sampling_rate"</span>]
	DeviceArray(<span class="hljs-number">44100</span>, dtype=int32, weak_type=<span class="hljs-literal">True</span>)`,lang:"py",wrap:!1}}),H=new ls({props:{title:"Data loading",local:"data-loading",headingTag:"h2"}}),S=new ls({props:{title:"Using with_format('jax')",local:"using-withformatjax",headingTag:"h3"}}),K=new u({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyeWxlY3VuJTJGbW5pc3QlMjIpJTBBZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJqYXglMjIpJTBBZHMlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQ=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
	<span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">"ylecun/mnist"</span>)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"jax"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-string">"train"</span>][<span class="hljs-number">0</span>]
	{<span class="hljs-string">'image'</span>: DeviceArray([[ <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, ...],
	[ <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, ...],
	...,
	[ <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, ...],
	[ <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, ...]], dtype=uint8),
	<span class="hljs-string">'label'</span>: DeviceArray(<span class="hljs-number">5</span>, dtype=int32)}`,lang:"py",wrap:!1}}),ss=new u({props:{code:"Zm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKGVwb2NocyklM0ElMEElMjAlMjAlMjAlMjBmb3IlMjBiYXRjaCUyMGluJTIwZHMlNUIlMjJ0cmFpbiUyMiU1RC5pdGVyKGJhdGNoX3NpemUlM0QzMiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB4JTJDJTIweSUyMCUzRCUyMGJhdGNoJTVCJTIyaW1hZ2UlMjIlNUQlMkMlMjBiYXRjaCU1QiUyMmxhYmVsJTIyJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwLi4u",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(epochs):
	<span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> ds[<span class="hljs-string">"train"</span>].<span class="hljs-built_in">iter</span>(batch_size=<span class="hljs-number">32</span>):
	<span class="hljs-meta">... </span> x, y = batch[<span class="hljs-string">"image"</span>], batch[<span class="hljs-string">"label"</span>]
	<span class="hljs-meta">... </span> ...`,lang:"py",wrap:!1}}),as=new wa({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/use_with_jax.mdx"}}),{c(){g=p("meta"),ps=e(),es=p("p"),rs=e(),i(U.$$.fragment),cs=e(),i(T.$$.fragment),is=e(),f=p("p"),f.innerHTML=Ls,hs=e(),M=p("blockquote"),M.innerHTML=Ss,ms=e(),i(w.$$.fragment),os=e(),C=p("p"),C.textContent=Ps,js=e(),I=p("p"),I.innerHTML=Ks,ds=e(),i(R.$$.fragment),us=e(),y=p("blockquote"),y.innerHTML=Os,gs=e(),$=p("p"),$.innerHTML=sa,Ms=e(),i(k.$$.fragment),ys=e(),Z=p("p"),Z.textContent=aa,Js=e(),x=p("p"),x.innerHTML=ta,bs=e(),i(Q.$$.fragment),Us=e(),X=p("p"),X.innerHTML=la,Ts=e(),i(N.$$.fragment),fs=e(),q=p("p"),q.textContent=ea,ws=e(),i(v.$$.fragment),Cs=e(),i(A.$$.fragment),Is=e(),G=p("p"),G.innerHTML=na,Rs=e(),i(E.$$.fragment),$s=e(),i(_.$$.fragment),ks=e(),F=p("p"),F.innerHTML=pa,Zs=e(),i(V.$$.fragment),xs=e(),z=p("p"),z.textContent=ra,Qs=e(),Y=p("p"),Y.innerHTML=ca,Xs=e(),J=p("blockquote"),J.innerHTML=ia,Ns=e(),i(B.$$.fragment),qs=e(),b=p("blockquote"),b.innerHTML=ha,vs=e(),i(D.$$.fragment),As=e(),i(H.$$.fragment),Gs=e(),W=p("p"),W.innerHTML=ma,Es=e(),L=p("p"),L.innerHTML=oa,_s=e(),i(S.$$.fragment),Fs=e(),P=p("p"),P.innerHTML=ja,Vs=e(),i(K.$$.fragment),zs=e(),O=p("p"),O.innerHTML=da,Ys=e(),i(ss.$$.fragment),Bs=e(),i(as.$$.fragment),Ds=e(),ns=p("p"),this.h()},l(s){const a=Ua("svelte-u9bgzb",document.head);g=r(a,"META",{name:!0,content:!0}),a.forEach(t),ps=n(s),es=r(s,"P",{}),ua(es).forEach(t),rs=n(s),h(U.$$.fragment,s),cs=n(s),h(T.$$.fragment,s),is=n(s),f=r(s,"P",{"data-svelte-h":!0}),c(f)!=="svelte-1a2kcgx"&&(f.innerHTML=Ls),hs=n(s),M=r(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),c(M)!=="svelte-ls9qpj"&&(M.innerHTML=Ss),ms=n(s),h(w.$$.fragment,s),os=n(s),C=r(s,"P",{"data-svelte-h":!0}),c(C)!=="svelte-1ix49d8"&&(C.textContent=Ps),js=n(s),I=r(s,"P",{"data-svelte-h":!0}),c(I)!=="svelte-1upv66g"&&(I.innerHTML=Ks),ds=n(s),h(R.$$.fragment,s),us=n(s),y=r(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),c(y)!=="svelte-n3nte3"&&(y.innerHTML=Os),gs=n(s),$=r(s,"P",{"data-svelte-h":!0}),c($)!=="svelte-1b7i338"&&($.innerHTML=sa),Ms=n(s),h(k.$$.fragment,s),ys=n(s),Z=r(s,"P",{"data-svelte-h":!0}),c(Z)!=="svelte-1c29vp0"&&(Z.textContent=aa),Js=n(s),x=r(s,"P",{"data-svelte-h":!0}),c(x)!=="svelte-1ooxy7c"&&(x.innerHTML=ta),bs=n(s),h(Q.$$.fragment,s),Us=n(s),X=r(s,"P",{"data-svelte-h":!0}),c(X)!=="svelte-hj2cy7"&&(X.innerHTML=la),Ts=n(s),h(N.$$.fragment,s),fs=n(s),q=r(s,"P",{"data-svelte-h":!0}),c(q)!=="svelte-smjp9l"&&(q.textContent=ea),ws=n(s),h(v.$$.fragment,s),Cs=n(s),h(A.$$.fragment,s),Is=n(s),G=r(s,"P",{"data-svelte-h":!0}),c(G)!=="svelte-1gw41y9"&&(G.innerHTML=na),Rs=n(s),h(E.$$.fragment,s),$s=n(s),h(_.$$.fragment,s),ks=n(s),F=r(s,"P",{"data-svelte-h":!0}),c(F)!=="svelte-1tiplh"&&(F.innerHTML=pa),Zs=n(s),h(V.$$.fragment,s),xs=n(s),z=r(s,"P",{"data-svelte-h":!0}),c(z)!=="svelte-gkri0z"&&(z.textContent=ra),Qs=n(s),Y=r(s,"P",{"data-svelte-h":!0}),c(Y)!=="svelte-a3kpt6"&&(Y.innerHTML=ca),Xs=n(s),J=r(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),c(J)!=="svelte-1qfd0sn"&&(J.innerHTML=ia),Ns=n(s),h(B.$$.fragment,s),qs=n(s),b=r(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),c(b)!=="svelte-1mqqik9"&&(b.innerHTML=ha),vs=n(s),h(D.$$.fragment,s),As=n(s),h(H.$$.fragment,s),Gs=n(s),W=r(s,"P",{"data-svelte-h":!0}),c(W)!=="svelte-1msw6w0"&&(W.innerHTML=ma),Es=n(s),L=r(s,"P",{"data-svelte-h":!0}),c(L)!=="svelte-1xmbhz7"&&(L.innerHTML=oa),_s=n(s),h(S.$$.fragment,s),Fs=n(s),P=r(s,"P",{"data-svelte-h":!0}),c(P)!=="svelte-14kpvqo"&&(P.innerHTML=ja),Vs=n(s),h(K.$$.fragment,s),zs=n(s),O=r(s,"P",{"data-svelte-h":!0}),c(O)!=="svelte-lnmbh3"&&(O.innerHTML=da),Ys=n(s),h(ss.$$.fragment,s),Bs=n(s),h(as.$$.fragment,s),Ds=n(s),ns=r(s,"P",{}),ua(ns).forEach(t),this.h()},h(){ts(g,"name","hf:doc:metadata"),ts(g,"content",Ia),ts(M,"class","tip"),ts(y,"class","tip"),ts(J,"class","tip"),ts(b,"class","tip")},m(s,a){Ta(document.head,g),l(s,ps,a),l(s,es,a),l(s,rs,a),m(U,s,a),l(s,cs,a),m(T,s,a),l(s,is,a),l(s,f,a),l(s,hs,a),l(s,M,a),l(s,ms,a),m(w,s,a),l(s,os,a),l(s,C,a),l(s,js,a),l(s,I,a),l(s,ds,a),m(R,s,a),l(s,us,a),l(s,y,a),l(s,gs,a),l(s,$,a),l(s,Ms,a),m(k,s,a),l(s,ys,a),l(s,Z,a),l(s,Js,a),l(s,x,a),l(s,bs,a),m(Q,s,a),l(s,Us,a),l(s,X,a),l(s,Ts,a),m(N,s,a),l(s,fs,a),l(s,q,a),l(s,ws,a),m(v,s,a),l(s,Cs,a),m(A,s,a),l(s,Is,a),l(s,G,a),l(s,Rs,a),m(E,s,a),l(s,$s,a),m(_,s,a),l(s,ks,a),l(s,F,a),l(s,Zs,a),m(V,s,a),l(s,xs,a),l(s,z,a),l(s,Qs,a),l(s,Y,a),l(s,Xs,a),l(s,J,a),l(s,Ns,a),m(B,s,a),l(s,qs,a),l(s,b,a),l(s,vs,a),m(D,s,a),l(s,As,a),m(H,s,a),l(s,Gs,a),l(s,W,a),l(s,Es,a),l(s,L,a),l(s,_s,a),m(S,s,a),l(s,Fs,a),l(s,P,a),l(s,Vs,a),m(K,s,a),l(s,zs,a),l(s,O,a),l(s,Ys,a),m(ss,s,a),l(s,Bs,a),m(as,s,a),l(s,Ds,a),l(s,ns,a),Hs=!0},p:Ma,i(s){Hs\|\|(o(U.$$.fragment,s),o(T.$$.fragment,s),o(w.$$.fragment,s),o(R.$$.fragment,s),o(k.$$.fragment,s),o(Q.$$.fragment,s),o(N.$$.fragment,s),o(v.$$.fragment,s),o(A.$$.fragment,s),o(E.$$.fragment,s),o(_.$$.fragment,s),o(V.$$.fragment,s),o(B.$$.fragment,s),o(D.$$.fragment,s),o(H.$$.fragment,s),o(S.$$.fragment,s),o(K.$$.fragment,s),o(ss.$$.fragment,s),o(as.$$.fragment,s),Hs=!0)},o(s){j(U.$$.fragment,s),j(T.$$.fragment,s),j(w.$$.fragment,s),j(R.$$.fragment,s),j(k.$$.fragment,s),j(Q.$$.fragment,s),j(N.$$.fragment,s),j(v.$$.fragment,s),j(A.$$.fragment,s),j(E.$$.fragment,s),j(_.$$.fragment,s),j(V.$$.fragment,s),j(B.$$.fragment,s),j(D.$$.fragment,s),j(H.$$.fragment,s),j(S.$$.fragment,s),j(K.$$.fragment,s),j(ss.$$.fragment,s),j(as.$$.fragment,s),Hs=!1},d(s){s&&(t(ps),t(es),t(rs),t(cs),t(is),t(f),t(hs),t(M),t(ms),t(os),t(C),t(js),t(I),t(ds),t(us),t(y),t(gs),t($),t(Ms),t(ys),t(Z),t(Js),t(x),t(bs),t(Us),t(X),t(Ts),t(fs),t(q),t(ws),t(Cs),t(Is),t(G),t(Rs),t($s),t(ks),t(F),t(Zs),t(xs),t(z),t(Qs),t(Y),t(Xs),t(J),t(Ns),t(qs),t(b),t(vs),t(As),t(Gs),t(W),t(Es),t(L),t(_s),t(Fs),t(P),t(Vs),t(zs),t(O),t(Ys),t(Bs),t(Ds),t(ns)),t(g),d(U,s),d(T,s),d(w,s),d(R,s),d(k,s),d(Q,s),d(N,s),d(v,s),d(A,s),d(E,s),d(_,s),d(V,s),d(B,s),d(D,s),d(H,s),d(S,s),d(K,s),d(ss,s),d(as,s)}}}const Ia=`{"title":"Use with JAX","local":"use-with-jax","sections":[{"title":"Dataset format","local":"dataset-format","sections":[{"title":"N-dimensional arrays","local":"n-dimensional-arrays","sections":[],"depth":3},{"title":"Other feature types","local":"other-feature-types","sections":[],"depth":3}],"depth":2},{"title":"Data loading","local":"data-loading","sections":[{"title":"Using with_format('jax')","local":"using-withformatjax","sections":[],"depth":3}],"depth":2}],"depth":1}`;function Ra(Ws){return ya(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Qa extends Ja{constructor(g){super(),ba(this,g,Ra,Ca,ga,{})}}export{Qa as component};

Xet Storage Details

Size:: 33.3 kB
Xet hash:: 68199f67500087c6bfc41aebde6c811fe3486ca0ab51e5a7e89abe61dff904ed

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.