Buckets:

hf-doc-build
/

doc-dev

hf-doc-build/doc-dev / datasets /pr_8213 /en /_app /immutable /nodes /51.abf8eaee.js

HuggingFaceDocBuilder's picture

HuggingFaceDocBuilder

about 2 months ago

28.2 kB

	import{s as Os,n as sa,o as aa}from"../chunks/scheduler.d75c11ed.js";import{S as ta,i as la,e as p,s as n,c as r,h as na,a as m,d as t,b as e,f as Ks,g as h,j as o,k as O,l as ea,m as l,n as c,t as u,o as i,p as j}from"../chunks/index.4ec9dfe9.js";import{C as pa,H as L,E as ra}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.9a52dede.js";import{C as d}from"../chunks/CodeBlock.37bede1d.js";function ma(Gs){let y,ss,P,as,J,ts,U,ls,T,xs=`This document is a quick introduction to using <code>datasets</code> with NumPy, with a particular focus on how to get
	<code>numpy.ndarray</code> objects out of our datasets, and how to use them to train models based on NumPy such as <code>scikit-learn</code> models.`,ns,f,es,w,Vs="By default, datasets return regular Python objects: integers, floats, strings, lists, etc..",ps,C,_s="To get NumPy arrays instead, you can set the format of the dataset to <code>numpy</code>:",rs,I,ms,g,zs='<p>A <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object is a wrapper of an Arrow table, which allows fast reads from arrays in the dataset to NumPy arrays.</p>',hs,R,Fs=`Note that the exact same procedure applies to <code>DatasetDict</code> objects, so that
	when setting the format of a <code>DatasetDict</code> to <code>numpy</code>, all the <code>Dataset</code>s there
	will be formatted as <code>numpy</code>:`,cs,$,us,k,is,Q,vs="If your dataset consists of N-dimensional arrays, you will see that by default they are considered as the same array if the shape is fixed:",js,N,os,Z,ds,q,Ws=`However this logic often requires slow shape comparisons and data copies.
	To avoid this, you must explicitly use the <code>Array</code> feature type and specify the shape of your tensors:`,ys,E,gs,X,Ms,G,Ys='<a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.ClassLabel">ClassLabel</a> data is properly converted to arrays:',bs,x,Js,V,Bs="String and binary objects are unchanged, since NumPy only supports numbers.",Us,_,Ds='The <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Image">Image</a> and <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Audio">Audio</a> feature types are also supported.',Ts,M,As=`<p>To use the <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Image">Image</a> feature type, you’ll need to install the <code>vision</code> extra as
	<code>pip install datasets[vision]</code>.</p>`,fs,z,ws,b,Hs=`<p>To use the <a href="/docs/datasets/pr_8213/en/package_reference/main_classes#datasets.Audio">Audio</a> feature type, you’ll need to install the <code>audio</code> extra as
	<code>pip install datasets[audio]</code>.</p>`,Cs,F,Is,v,Rs,W,Ss='NumPy doesn’t have any built-in data loading capabilities, so you’ll either need to materialize the NumPy arrays like <code>X, y</code> to use in <code>scikit-learn</code> or use a library such as <a href="https://pytorch.org/" rel="nofollow">PyTorch</a> to load your data using a <code>DataLoader</code>.',$s,Y,ks,B,Ls=`The easiest way to get NumPy arrays out of a dataset is to use the <code>with_format('numpy')</code> method. Lets assume
	that we want to train a neural network on the <a href="http://yann.lecun.com/exdb/mnist/" rel="nofollow">MNIST dataset</a> available
	at the HuggingFace Hub at <a href="https://huggingface.co/datasets/mnist" rel="nofollow">https://huggingface.co/datasets/mnist</a>.`,Qs,D,Ns,A,Ps=`Once the format is set we can feed the dataset to the model based on NumPy in batches using the <code>Dataset.iter()</code>
	method:`,Zs,H,qs,S,Es,K,Xs;return J=new pa({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new L({props:{title:"Use with NumPy",local:"use-with-numpy",headingTag:"h1"}}),f=new L({props:{title:"Dataset format",local:"dataset-format",headingTag:"h2"}}),I=new d({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGElMjAlM0QlMjAlNUIlNUIxJTJDJTIwMiU1RCUyQyUyMCU1QjMlMkMlMjA0JTVEJTVEJTBBZHMlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglN0IlMjJkYXRhJTIyJTNBJTIwZGF0YSU3RCklMEFkcyUyMCUzRCUyMGRzLndpdGhfZm9ybWF0KCUyMm51bXB5JTIyKSUwQWRzJTVCMCU1RCUwQWRzJTVCJTNBMiU1RA==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
	<span class="hljs-meta">>>> </span>data = [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"data"</span>: data})
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"numpy"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'data'</span>: array([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>])}
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">2</span>]
	{<span class="hljs-string">'data'</span>: array([
	[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])}`,lang:"py",wrap:!1}}),$=new d({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldERpY3QlMEFkYXRhJTIwJTNEJTIwJTdCJTIydHJhaW4lMjIlM0ElMjAlN0IlMjJkYXRhJTIyJTNBJTIwJTVCJTVCMSUyQyUyMDIlNUQlMkMlMjAlNUIzJTJDJTIwNCU1RCU1RCU3RCUyQyUyMCUyMnRlc3QlMjIlM0ElMjAlN0IlMjJkYXRhJTIyJTNBJTIwJTVCJTVCNSUyQyUyMDYlNUQlMkMlMjAlNUI3JTJDJTIwOCU1RCU1RCU3RCU3RCUwQWRkcyUyMCUzRCUyMERhdGFzZXREaWN0LmZyb21fZGljdChkYXRhKSUwQWRkcyUyMCUzRCUyMGRkcy53aXRoX2Zvcm1hdCglMjJudW1weSUyMiklMEFkZHMlNUIlMjJ0cmFpbiUyMiU1RCU1QiUzQTIlNUQ=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> DatasetDict
	<span class="hljs-meta">>>> </span>data = {<span class="hljs-string">"train"</span>: {<span class="hljs-string">"data"</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]}, <span class="hljs-string">"test"</span>: {<span class="hljs-string">"data"</span>: [[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>], [<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]}}
	<span class="hljs-meta">>>> </span>dds = DatasetDict.from_dict(data)
	<span class="hljs-meta">>>> </span>dds = dds.with_format(<span class="hljs-string">"numpy"</span>)
	<span class="hljs-meta">>>> </span>dds[<span class="hljs-string">"train"</span>][:<span class="hljs-number">2</span>]
	{<span class="hljs-string">'data'</span>: array([
	[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])}`,lang:"py",wrap:!1}}),k=new L({props:{title:"N-dimensional arrays",local:"n-dimensional-arrays",headingTag:"h3"}}),N=new d({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGElMjAlM0QlMjAlNUIlNUIlNUIxJTJDJTIwMiU1RCUyQyU1QjMlMkMlMjA0JTVEJTVEJTJDJTIwJTVCJTVCNSUyQyUyMDYlNUQlMkMlNUI3JTJDJTIwOCU1RCU1RCU1RCUyMCUyMCUyMyUyMGZpeGVkJTIwc2hhcGUlMEFkcyUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCU3QiUyMmRhdGElMjIlM0ElMjBkYXRhJTdEKSUwQWRzJTIwJTNEJTIwZHMud2l0aF9mb3JtYXQoJTIybnVtcHklMjIpJTBBZHMlNUIwJTVE",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
	<span class="hljs-meta">>>> </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]], [[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]] <span class="hljs-comment"># fixed shape</span>
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"data"</span>: data})
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"numpy"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'data'</span>: array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])}`,lang:"py",wrap:!1}}),Z=new d({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUwQWRhdGElMjAlM0QlMjAlNUIlNUIlNUIxJTJDJTIwMiU1RCUyQyU1QjMlNUQlNUQlMkMlMjAlNUIlNUI0JTJDJTIwNSUyQyUyMDYlNUQlMkMlNUI3JTJDJTIwOCU1RCU1RCU1RCUyMCUyMCUyMyUyMHZhcnlpbmclMjBzaGFwZSUwQWRzJTIwJTNEJTIwRGF0YXNldC5mcm9tX2RpY3QoJTdCJTIyZGF0YSUyMiUzQSUyMGRhdGElN0QpJTBBZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJudW1weSUyMiklMEFkcyU1QjAlNUQ=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
	<span class="hljs-meta">>>> </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>]], [[<span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]] <span class="hljs-comment"># varying shape</span>
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"data"</span>: data})
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"numpy"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'data'</span>: array([array([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>]), array([<span class="hljs-number">3</span>])], dtype=<span class="hljs-built_in">object</span>)}`,lang:"py",wrap:!1}}),E=new d({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwQXJyYXkyRCUwQWRhdGElMjAlM0QlMjAlNUIlNUIlNUIxJTJDJTIwMiU1RCUyQyU1QjMlMkMlMjA0JTVEJTVEJTJDJTVCJTVCNSUyQyUyMDYlNUQlMkMlNUI3JTJDJTIwOCU1RCU1RCU1RCUwQWZlYXR1cmVzJTIwJTNEJTIwRmVhdHVyZXMoJTdCJTIyZGF0YSUyMiUzQSUyMEFycmF5MkQoc2hhcGUlM0QoMiUyQyUyMDIpJTJDJTIwZHR5cGUlM0QnaW50MzInKSU3RCklMEFkcyUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCU3QiUyMmRhdGElMjIlM0ElMjBkYXRhJTdEJTJDJTIwZmVhdHVyZXMlM0RmZWF0dXJlcyklMEFkcyUyMCUzRCUyMGRzLndpdGhfZm9ybWF0KCUyMm51bXB5JTIyKSUwQWRzJTVCMCU1RCUwQWRzJTVCJTNBMiU1RA==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Array2D
	<span class="hljs-meta">>>> </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
	<span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">"data"</span>: Array2D(shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=<span class="hljs-string">'int32'</span>)})
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"data"</span>: data}, features=features)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"numpy"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'data'</span>: array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])}
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">2</span>]
	{<span class="hljs-string">'data'</span>: array([[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
	[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],

	[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],
	[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]])}`,lang:"py",wrap:!1}}),X=new L({props:{title:"Other feature types",local:"other-feature-types",headingTag:"h3"}}),x=new d({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwQ2xhc3NMYWJlbCUwQWxhYmVscyUyMCUzRCUyMCU1QjAlMkMlMjAwJTJDJTIwMSU1RCUwQWZlYXR1cmVzJTIwJTNEJTIwRmVhdHVyZXMoJTdCJTIybGFiZWwlMjIlM0ElMjBDbGFzc0xhYmVsKG5hbWVzJTNEJTVCJTIybmVnYXRpdmUlMjIlMkMlMjAlMjJwb3NpdGl2ZSUyMiU1RCklN0QpJTBBZHMlMjAlM0QlMjBEYXRhc2V0LmZyb21fZGljdCglN0IlMjJsYWJlbCUyMiUzQSUyMGxhYmVscyU3RCUyQyUyMGZlYXR1cmVzJTNEZmVhdHVyZXMpJTBBZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJudW1weSUyMiklMEFkcyU1QiUzQTMlNUQ=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, ClassLabel
	<span class="hljs-meta">>>> </span>labels = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
	<span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">"label"</span>: ClassLabel(names=[<span class="hljs-string">"negative"</span>, <span class="hljs-string">"positive"</span>])})
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"label"</span>: labels}, features=features)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"numpy"</span>)
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">3</span>]
	{<span class="hljs-string">'label'</span>: array([<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>])}`,lang:"py",wrap:!1}}),z=new d({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwSW1hZ2UlMEFpbWFnZXMlMjAlM0QlMjAlNUIlMjJwYXRoJTJGdG8lMkZpbWFnZS5wbmclMjIlNUQlMjAqJTIwMTAlMEFmZWF0dXJlcyUyMCUzRCUyMEZlYXR1cmVzKCU3QiUyMmltYWdlJTIyJTNBJTIwSW1hZ2UoKSU3RCklMEFkcyUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCU3QiUyMmltYWdlJTIyJTNBJTIwaW1hZ2VzJTdEJTJDJTIwZmVhdHVyZXMlM0RmZWF0dXJlcyklMEFkcyUyMCUzRCUyMGRzLndpdGhfZm9ybWF0KCUyMm51bXB5JTIyKSUwQWRzJTVCMCU1RCU1QiUyMmltYWdlJTIyJTVELnNoYXBlJTBBZHMlNUIwJTVEJTBBZHMlNUIlM0EyJTVEJTVCJTIyaW1hZ2UlMjIlNUQuc2hhcGUlMEFkcyU1QiUzQTIlNUQ=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Image
	<span class="hljs-meta">>>> </span>images = [<span class="hljs-string">"path/to/image.png"</span>] * <span class="hljs-number">10</span>
	<span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">"image"</span>: Image()})
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"image"</span>: images}, features=features)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"numpy"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">"image"</span>].shape
	(<span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">3</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>]
	{<span class="hljs-string">'image'</span>: array([[[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	...,
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]], dtype=uint8)}
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">2</span>][<span class="hljs-string">"image"</span>].shape
	(<span class="hljs-number">2</span>, <span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">3</span>)
	<span class="hljs-meta">>>> </span>ds[:<span class="hljs-number">2</span>]
	{<span class="hljs-string">'image'</span>: array([[[[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	...,
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
	[ <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]]], dtype=uint8)}`,lang:"py",wrap:!1}}),F=new d({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwRGF0YXNldCUyQyUyMEZlYXR1cmVzJTJDJTIwQXVkaW8lMEFhdWRpbyUyMCUzRCUyMCU1QiUyMnBhdGglMkZ0byUyRmF1ZGlvLndhdiUyMiU1RCUyMColMjAxMCUwQWZlYXR1cmVzJTIwJTNEJTIwRmVhdHVyZXMoJTdCJTIyYXVkaW8lMjIlM0ElMjBBdWRpbygpJTdEKSUwQWRzJTIwJTNEJTIwRGF0YXNldC5mcm9tX2RpY3QoJTdCJTIyYXVkaW8lMjIlM0ElMjBhdWRpbyU3RCUyQyUyMGZlYXR1cmVzJTNEZmVhdHVyZXMpJTBBZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJudW1weSUyMiklMEFkcyU1QjAlNUQlNUIlMjJhdWRpbyUyMiU1RCU1QiUyMmFycmF5JTIyJTVEJTBBZHMlNUIwJTVEJTVCJTIyYXVkaW8lMjIlNUQlNUIlMjJzYW1wbGluZ19yYXRlJTIyJTVE",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio
	<span class="hljs-meta">>>> </span>audio = [<span class="hljs-string">"path/to/audio.wav"</span>] * <span class="hljs-number">10</span>
	<span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">"audio"</span>: Audio()})
	<span class="hljs-meta">>>> </span>ds = Dataset.from_dict({<span class="hljs-string">"audio"</span>: audio}, features=features)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"numpy"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">"audio"</span>][<span class="hljs-string">"array"</span>]
	array([-<span class="hljs-number">0.059021</span> , -<span class="hljs-number">0.03894043</span>, -<span class="hljs-number">0.00735474</span>, ..., <span class="hljs-number">0.0133667</span> ,
	<span class="hljs-number">0.01809692</span>, <span class="hljs-number">0.00268555</span>], dtype=float32)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">"audio"</span>][<span class="hljs-string">"sampling_rate"</span>]
	array(<span class="hljs-number">44100</span>, weak_type=<span class="hljs-literal">True</span>)`,lang:"py",wrap:!1}}),v=new L({props:{title:"Data loading",local:"data-loading",headingTag:"h2"}}),Y=new L({props:{title:"Using with_format('numpy')",local:"using-withformatnumpy",headingTag:"h3"}}),D=new d({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyeWxlY3VuJTJGbW5pc3QlMjIpJTBBZHMlMjAlM0QlMjBkcy53aXRoX2Zvcm1hdCglMjJudW1weSUyMiklMEFkcyU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
	<span class="hljs-meta">>>> </span>ds = load_dataset(<span class="hljs-string">"ylecun/mnist"</span>)
	<span class="hljs-meta">>>> </span>ds = ds.with_format(<span class="hljs-string">"numpy"</span>)
	<span class="hljs-meta">>>> </span>ds[<span class="hljs-string">"train"</span>][<span class="hljs-number">0</span>]
	{<span class="hljs-string">'image'</span>: array([[ <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, ...],
	[ <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, ...],
	...,
	[ <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, ...],
	[ <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, ...]], dtype=uint8),
	<span class="hljs-string">'label'</span>: array(<span class="hljs-number">5</span>)}`,lang:"py",wrap:!1}}),H=new d({props:{code:"Zm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKGVwb2NocyklM0ElMEElMjAlMjAlMjAlMjBmb3IlMjBiYXRjaCUyMGluJTIwZHMlNUIlMjJ0cmFpbiUyMiU1RC5pdGVyKGJhdGNoX3NpemUlM0QzMiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB4JTJDJTIweSUyMCUzRCUyMGJhdGNoJTVCJTIyaW1hZ2UlMjIlNUQlMkMlMjBiYXRjaCU1QiUyMmxhYmVsJTIyJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwLi4u",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(epochs):
	<span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> ds[<span class="hljs-string">"train"</span>].<span class="hljs-built_in">iter</span>(batch_size=<span class="hljs-number">32</span>):
	<span class="hljs-meta">... </span> x, y = batch[<span class="hljs-string">"image"</span>], batch[<span class="hljs-string">"label"</span>]
	<span class="hljs-meta">... </span> ...`,lang:"py",wrap:!1}}),S=new ra({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/use_with_numpy.mdx"}}),{c(){y=p("meta"),ss=n(),P=p("p"),as=n(),r(J.$$.fragment),ts=n(),r(U.$$.fragment),ls=n(),T=p("p"),T.innerHTML=xs,ns=n(),r(f.$$.fragment),es=n(),w=p("p"),w.textContent=Vs,ps=n(),C=p("p"),C.innerHTML=_s,rs=n(),r(I.$$.fragment),ms=n(),g=p("blockquote"),g.innerHTML=zs,hs=n(),R=p("p"),R.innerHTML=Fs,cs=n(),r($.$$.fragment),us=n(),r(k.$$.fragment),is=n(),Q=p("p"),Q.textContent=vs,js=n(),r(N.$$.fragment),os=n(),r(Z.$$.fragment),ds=n(),q=p("p"),q.innerHTML=Ws,ys=n(),r(E.$$.fragment),gs=n(),r(X.$$.fragment),Ms=n(),G=p("p"),G.innerHTML=Ys,bs=n(),r(x.$$.fragment),Js=n(),V=p("p"),V.textContent=Bs,Us=n(),_=p("p"),_.innerHTML=Ds,Ts=n(),M=p("blockquote"),M.innerHTML=As,fs=n(),r(z.$$.fragment),ws=n(),b=p("blockquote"),b.innerHTML=Hs,Cs=n(),r(F.$$.fragment),Is=n(),r(v.$$.fragment),Rs=n(),W=p("p"),W.innerHTML=Ss,$s=n(),r(Y.$$.fragment),ks=n(),B=p("p"),B.innerHTML=Ls,Qs=n(),r(D.$$.fragment),Ns=n(),A=p("p"),A.innerHTML=Ps,Zs=n(),r(H.$$.fragment),qs=n(),r(S.$$.fragment),Es=n(),K=p("p"),this.h()},l(s){const a=na("svelte-u9bgzb",document.head);y=m(a,"META",{name:!0,content:!0}),a.forEach(t),ss=e(s),P=m(s,"P",{}),Ks(P).forEach(t),as=e(s),h(J.$$.fragment,s),ts=e(s),h(U.$$.fragment,s),ls=e(s),T=m(s,"P",{"data-svelte-h":!0}),o(T)!=="svelte-movp6t"&&(T.innerHTML=xs),ns=e(s),h(f.$$.fragment,s),es=e(s),w=m(s,"P",{"data-svelte-h":!0}),o(w)!=="svelte-jsr2ey"&&(w.textContent=Vs),ps=e(s),C=m(s,"P",{"data-svelte-h":!0}),o(C)!=="svelte-tgh4v8"&&(C.innerHTML=_s),rs=e(s),h(I.$$.fragment,s),ms=e(s),g=m(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(g)!=="svelte-lnoxx3"&&(g.innerHTML=zs),hs=e(s),R=m(s,"P",{"data-svelte-h":!0}),o(R)!=="svelte-19vwxv0"&&(R.innerHTML=Fs),cs=e(s),h($.$$.fragment,s),us=e(s),h(k.$$.fragment,s),is=e(s),Q=m(s,"P",{"data-svelte-h":!0}),o(Q)!=="svelte-n2lxj9"&&(Q.textContent=vs),js=e(s),h(N.$$.fragment,s),os=e(s),h(Z.$$.fragment,s),ds=e(s),q=m(s,"P",{"data-svelte-h":!0}),o(q)!=="svelte-1gw41y9"&&(q.innerHTML=Ws),ys=e(s),h(E.$$.fragment,s),gs=e(s),h(X.$$.fragment,s),Ms=e(s),G=m(s,"P",{"data-svelte-h":!0}),o(G)!=="svelte-kgslq1"&&(G.innerHTML=Ys),bs=e(s),h(x.$$.fragment,s),Js=e(s),V=m(s,"P",{"data-svelte-h":!0}),o(V)!=="svelte-1i7nqnr"&&(V.textContent=Bs),Us=e(s),_=m(s,"P",{"data-svelte-h":!0}),o(_)!=="svelte-xxqqhu"&&(_.innerHTML=Ds),Ts=e(s),M=m(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(M)!=="svelte-1qczjnv"&&(M.innerHTML=As),fs=e(s),h(z.$$.fragment,s),ws=e(s),b=m(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),o(b)!=="svelte-13278d5"&&(b.innerHTML=Hs),Cs=e(s),h(F.$$.fragment,s),Is=e(s),h(v.$$.fragment,s),Rs=e(s),W=m(s,"P",{"data-svelte-h":!0}),o(W)!=="svelte-14evha6"&&(W.innerHTML=Ss),$s=e(s),h(Y.$$.fragment,s),ks=e(s),B=m(s,"P",{"data-svelte-h":!0}),o(B)!=="svelte-kj9r2i"&&(B.innerHTML=Ls),Qs=e(s),h(D.$$.fragment,s),Ns=e(s),A=m(s,"P",{"data-svelte-h":!0}),o(A)!=="svelte-117alrv"&&(A.innerHTML=Ps),Zs=e(s),h(H.$$.fragment,s),qs=e(s),h(S.$$.fragment,s),Es=e(s),K=m(s,"P",{}),Ks(K).forEach(t),this.h()},h(){O(y,"name","hf:doc:metadata"),O(y,"content",ha),O(g,"class","tip"),O(M,"class","tip"),O(b,"class","tip")},m(s,a){ea(document.head,y),l(s,ss,a),l(s,P,a),l(s,as,a),c(J,s,a),l(s,ts,a),c(U,s,a),l(s,ls,a),l(s,T,a),l(s,ns,a),c(f,s,a),l(s,es,a),l(s,w,a),l(s,ps,a),l(s,C,a),l(s,rs,a),c(I,s,a),l(s,ms,a),l(s,g,a),l(s,hs,a),l(s,R,a),l(s,cs,a),c($,s,a),l(s,us,a),c(k,s,a),l(s,is,a),l(s,Q,a),l(s,js,a),c(N,s,a),l(s,os,a),c(Z,s,a),l(s,ds,a),l(s,q,a),l(s,ys,a),c(E,s,a),l(s,gs,a),c(X,s,a),l(s,Ms,a),l(s,G,a),l(s,bs,a),c(x,s,a),l(s,Js,a),l(s,V,a),l(s,Us,a),l(s,_,a),l(s,Ts,a),l(s,M,a),l(s,fs,a),c(z,s,a),l(s,ws,a),l(s,b,a),l(s,Cs,a),c(F,s,a),l(s,Is,a),c(v,s,a),l(s,Rs,a),l(s,W,a),l(s,$s,a),c(Y,s,a),l(s,ks,a),l(s,B,a),l(s,Qs,a),c(D,s,a),l(s,Ns,a),l(s,A,a),l(s,Zs,a),c(H,s,a),l(s,qs,a),c(S,s,a),l(s,Es,a),l(s,K,a),Xs=!0},p:sa,i(s){Xs\|\|(u(J.$$.fragment,s),u(U.$$.fragment,s),u(f.$$.fragment,s),u(I.$$.fragment,s),u($.$$.fragment,s),u(k.$$.fragment,s),u(N.$$.fragment,s),u(Z.$$.fragment,s),u(E.$$.fragment,s),u(X.$$.fragment,s),u(x.$$.fragment,s),u(z.$$.fragment,s),u(F.$$.fragment,s),u(v.$$.fragment,s),u(Y.$$.fragment,s),u(D.$$.fragment,s),u(H.$$.fragment,s),u(S.$$.fragment,s),Xs=!0)},o(s){i(J.$$.fragment,s),i(U.$$.fragment,s),i(f.$$.fragment,s),i(I.$$.fragment,s),i($.$$.fragment,s),i(k.$$.fragment,s),i(N.$$.fragment,s),i(Z.$$.fragment,s),i(E.$$.fragment,s),i(X.$$.fragment,s),i(x.$$.fragment,s),i(z.$$.fragment,s),i(F.$$.fragment,s),i(v.$$.fragment,s),i(Y.$$.fragment,s),i(D.$$.fragment,s),i(H.$$.fragment,s),i(S.$$.fragment,s),Xs=!1},d(s){s&&(t(ss),t(P),t(as),t(ts),t(ls),t(T),t(ns),t(es),t(w),t(ps),t(C),t(rs),t(ms),t(g),t(hs),t(R),t(cs),t(us),t(is),t(Q),t(js),t(os),t(ds),t(q),t(ys),t(gs),t(Ms),t(G),t(bs),t(Js),t(V),t(Us),t(_),t(Ts),t(M),t(fs),t(ws),t(b),t(Cs),t(Is),t(Rs),t(W),t($s),t(ks),t(B),t(Qs),t(Ns),t(A),t(Zs),t(qs),t(Es),t(K)),t(y),j(J,s),j(U,s),j(f,s),j(I,s),j($,s),j(k,s),j(N,s),j(Z,s),j(E,s),j(X,s),j(x,s),j(z,s),j(F,s),j(v,s),j(Y,s),j(D,s),j(H,s),j(S,s)}}}const ha=`{"title":"Use with NumPy","local":"use-with-numpy","sections":[{"title":"Dataset format","local":"dataset-format","sections":[{"title":"N-dimensional arrays","local":"n-dimensional-arrays","sections":[],"depth":3},{"title":"Other feature types","local":"other-feature-types","sections":[],"depth":3}],"depth":2},{"title":"Data loading","local":"data-loading","sections":[{"title":"Using with_format('numpy')","local":"using-withformatnumpy","sections":[],"depth":3}],"depth":2}],"depth":1}`;function ca(Gs){return aa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class da extends ta{constructor(y){super(),la(this,y,ca,ma,Os,{})}}export{da as component};

Xet Storage Details

Size:: 28.2 kB
Xet hash:: 87cbb828def95b50e7826602a655236e9281d2068cdd700a2b8266529d0a8bd0

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.