Buckets:

hf-doc-build
/

doc

Files

xet

hf-doc-build/doc / datasets /v2.2.2 /en /about_dataset_features.html

rtrm

about 2 months ago

download

raw

15.6 kB

	<meta charset="utf-8" /><meta http-equiv="content-security-policy" content=""><meta name="hf:doc:metadata" content="{"local":"dataset-features","title":"Dataset features"}" data-svelte="svelte-1phssyn">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/assets/pages/__layout.svelte-efc77dbd.css">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/start-0f8c1da7.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/vendor-8138ceec.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/paths-4b3c6e7e.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/pages/__layout.svelte-efb8e839.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/pages/about_dataset_features.mdx-23297db7.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/Tip-12722dfc.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/IconCopyLink-2dd3a6ac.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/CodeBlock-fc89709f.js">





	<h1 class="relative group"><a id="dataset-features" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dataset-features"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Dataset features
	</span></h1>

	<p><a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Features">Features</a> defines the internal structure of a dataset. It is used to specify the underlying serialization format. What’s more interesting to you though is that <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Features">Features</a> contains high-level information about everything from the column names and types, to the <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.ClassLabel">ClassLabel</a>. You can think of <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Features">Features</a> as the backbone of a dataset.</p>
	<p>The <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Features">Features</a> format is simple: <code>dict[column_name, column_type]</code>. It is a dictionary of column name and column type pairs. The column type provides a wide range of options for describing the type of data you have.</p>
	<p>Let’s have a look at the features of the MRPC dataset from the GLUE benchmark:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
	<span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'glue'</span>, <span class="hljs-string">'mrpc'</span>, split=<span class="hljs-string">'train'</span>)
	<span class="hljs-meta">>>> </span>dataset.features
	{<span class="hljs-string">'idx'</span>: Value(dtype=<span class="hljs-string">'int32'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
	<span class="hljs-string">'label'</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">'not_equivalent'</span>, <span class="hljs-string">'equivalent'</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
	<span class="hljs-string">'sentence1'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
	<span class="hljs-string">'sentence2'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
	}<!-- HTML_TAG_END --></pre></div>
	<p>The <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Value">Value</a> feature tells 🤗 Datasets:</p>
	<ul><li>The <code>idx</code> data type is <code>int32</code>.</li>
	<li>The <code>sentence1</code> and <code>sentence2</code> data types are <code>string</code>.</li></ul>
	<p>🤗 Datasets supports many other data types such as <code>bool</code>, <code>float32</code> and <code>binary</code> to name just a few.</p>


	<div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p>Refer to <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Value">Value</a> for a full list of supported data types.</p></div>
	<p>The <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.ClassLabel">ClassLabel</a> feature informs 🤗 Datasets the <code>label</code> column contains two classes. The classes are labeled <code>not_equivalent</code> and <code>equivalent</code>. Labels are stored as integers in the dataset. When you retrieve the labels, <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.ClassLabel.int2str">ClassLabel.int2str()</a> and <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.ClassLabel.str2int">ClassLabel.str2int()</a> carries out the conversion from integer value to label name, and vice versa.</p>
	<p>If your data type contains a list of objects, then you want to use the <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Sequence">Sequence</a> feature. Remember the SQuAD dataset?</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
	<span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'squad'</span>, split=<span class="hljs-string">'train'</span>)
	<span class="hljs-meta">>>> </span>dataset.features
	{<span class="hljs-string">'answers'</span>: <span class="hljs-type">Sequence</span>(feature={<span class="hljs-string">'text'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), <span class="hljs-string">'answer_start'</span>: Value(dtype=<span class="hljs-string">'int32'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}, length=-<span class="hljs-number">1</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
	<span class="hljs-string">'context'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
	<span class="hljs-string">'id'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
	<span class="hljs-string">'question'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>),
	<span class="hljs-string">'title'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}<!-- HTML_TAG_END --></pre></div>
	<p>The <code>answers</code> field is constructed using the <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Sequence">Sequence</a> feature because it contains two subfields, <code>text</code> and <code>answer_start</code>, which are lists of <code>string</code> and <code>int32</code>, respectively.</p>


	<div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p>See the <a href="./process#flatten">flatten</a> section to learn how you can extract the nested subfields as their own independent columns.</p></div>
	<p>The array feature type is useful for creating arrays of various sizes. You can create arrays with two dimensions using <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Array2D">Array2D</a>, and even arrays with five dimensions using <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.Array5D">Array5D</a>. </p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">'a'</span>: Array2D(shape=(<span class="hljs-number">1</span>, <span class="hljs-number">3</span>), dtype=<span class="hljs-string">'int32'</span>))<!-- HTML_TAG_END --></pre></div>
	<p>The array type also allows the first dimension of the array to be dynamic. This is useful for handling sequences with variable lengths such as sentences, without having to pad or truncate the input to a uniform shape.</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>features = Features({<span class="hljs-string">'a'</span>: Array3D(shape=(<span class="hljs-literal">None</span>, <span class="hljs-number">5</span>, <span class="hljs-number">2</span>), dtype=<span class="hljs-string">'int32'</span>)})<!-- HTML_TAG_END --></pre></div>


	<script type="module" data-hydrate="1y82rzk">
	import { start } from "/docs/datasets/v2.2.2/en/_app/start-0f8c1da7.js";
	start({
	target: document.querySelector('[data-hydrate="1y82rzk"]').parentNode,
	paths: {"base":"/docs/datasets/v2.2.2/en","assets":"/docs/datasets/v2.2.2/en"},
	session: {},
	route: false,
	spa: false,
	trailing_slash: "never",
	hydrate: {
	status: 200,
	error: null,
	nodes: [
	import("/docs/datasets/v2.2.2/en/_app/pages/__layout.svelte-efb8e839.js"),
	import("/docs/datasets/v2.2.2/en/_app/pages/about_dataset_features.mdx-23297db7.js")
	],
	params: {}
	}
	});
	</script>

Xet Storage Details

Size:: 15.6 kB
Xet hash:: 761411fb39816241d4a1fe6a74ce84e95f1f2848640c90ed3e53d7751fc98b26

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.