Buckets:

hf-doc-build
/

doc

Files

xet

hf-doc-build/doc / datasets /v2.2.2 /en /load_hub.html

rtrm

about 2 months ago

download

raw

26.6 kB

	<meta charset="utf-8" /><meta http-equiv="content-security-policy" content=""><meta name="hf:doc:metadata" content="{"local":"hugging-face-hub","sections":[{"local":"load-a-dataset","title":"Load a dataset"},{"local":"select-a-configuration","title":"Select a configuration"},{"local":"select-a-split","title":"Select a split"}],"title":"Hugging Face Hub"}" data-svelte="svelte-1phssyn">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/assets/pages/__layout.svelte-efc77dbd.css">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/start-0f8c1da7.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/vendor-8138ceec.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/paths-4b3c6e7e.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/pages/__layout.svelte-efb8e839.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/pages/load_hub.mdx-966ed259.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/Tip-12722dfc.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/IconCopyLink-2dd3a6ac.js">
	<link rel="modulepreload" href="/docs/datasets/v2.2.2/en/_app/chunks/CodeBlock-fc89709f.js">





	<h1 class="relative group"><a id="hugging-face-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#hugging-face-hub"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Hugging Face Hub
	</span></h1>

	<p>Now that you are all setup, the first step is to load a dataset. The easiest way to load a dataset is from the <a href="https://huggingface.co/datasets" rel="nofollow">Hugging Face Hub</a>. There are already over 6000 datasets in over 100 languages on the Hub. Choose from a wide category of datasets to use for NLP tasks like question answering, summarization, machine translation, and language modeling.</p>
	<h2 class="relative group"><a id="load-a-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#load-a-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Load a dataset
	</span></h2>

	<p>Before you take the time to download a dataset, it is often helpful to quickly get all the relevant information about a dataset. The <a href="/docs/datasets/v2.2.2/en/package_reference/loading_methods#datasets.load_dataset_builder">load_dataset_builder()</a> method allows you to inspect the attributes of a dataset without downloading it.</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset_builder
	<span class="hljs-meta">>>> </span>dataset_builder = load_dataset_builder(<span class="hljs-string">'imdb'</span>)
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(dataset_builder.cache_dir)
	/Users/thomwolf/.cache/huggingface/datasets/imdb/plain_text/<span class="hljs-number">1.0</span><span class="hljs-number">.0</span>/fdc76b18d5506f14b0646729b8d371880ef1bc48a26d00835a7f3da44004b676
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(dataset_builder.info.features)
	{<span class="hljs-string">'text'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), <span class="hljs-string">'label'</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">'neg'</span>, <span class="hljs-string">'pos'</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(dataset_builder.info.splits)
	{<span class="hljs-string">'train'</span>: SplitInfo(name=<span class="hljs-string">'train'</span>, num_bytes=<span class="hljs-number">33432835</span>, num_examples=<span class="hljs-number">25000</span>, dataset_name=<span class="hljs-string">'imdb'</span>), <span class="hljs-string">'test'</span>: SplitInfo(name=<span class="hljs-string">'test'</span>, num_bytes=<span class="hljs-number">32650697</span>, num_examples=<span class="hljs-number">25000</span>, dataset_name=<span class="hljs-string">'imdb'</span>), <span class="hljs-string">'unsupervised'</span>: SplitInfo(name=<span class="hljs-string">'unsupervised'</span>, num_bytes=<span class="hljs-number">67106814</span>, num_examples=<span class="hljs-number">50000</span>, dataset_name=<span class="hljs-string">'imdb'</span>)}<!-- HTML_TAG_END --></pre></div>


	<div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p>Take a look at <a href="/docs/datasets/v2.2.2/en/package_reference/main_classes#datasets.DatasetInfo">DatasetInfo</a> for a full list of attributes you can use with <code>dataset_builder</code>.</p></div>
	<p>Once you are happy with the dataset you want, load it in a single line with <a href="/docs/datasets/v2.2.2/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a>:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
	<span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'glue'</span>, <span class="hljs-string">'mrpc'</span>, split=<span class="hljs-string">'train'</span>)<!-- HTML_TAG_END --></pre></div>
	<h2 class="relative group"><a id="select-a-configuration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#select-a-configuration"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Select a configuration
	</span></h2>

	<p>Some datasets, like the <a href="https://huggingface.co/datasets/glue" rel="nofollow">General Language Understanding Evaluation (GLUE)</a> benchmark, are actually made up of several datasets. These sub-datasets are called <strong>configurations</strong>, and you must explicitly select one when you load the dataset. If you don’t provide a configuration name, 🤗 Datasets will raise a <code>ValueError</code> and remind you to select a configuration.</p>
	<p>Use the <a href="/docs/datasets/v2.2.2/en/package_reference/loading_methods#datasets.get_dataset_config_names">get_dataset_config_names()</a> function to retrieve a list of all the possible configurations available to your dataset:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> get_dataset_config_names

	configs = get_dataset_config_names(<span class="hljs-string">"glue"</span>)
	<span class="hljs-built_in">print</span>(configs)
	<span class="hljs-comment"># ['cola', 'sst2', 'mrpc', 'qqp', 'stsb', 'mnli', 'mnli_mismatched', 'mnli_matched', 'qnli', 'rte', 'wnli', 'ax']</span><!-- HTML_TAG_END --></pre></div>
	<p>❌ Incorrect way to load a configuration:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
	<span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'glue'</span>)
	ValueError: Config name <span class="hljs-keyword">is</span> missing.
	Please pick one among the available configs: [<span class="hljs-string">'cola'</span>, <span class="hljs-string">'sst2'</span>, <span class="hljs-string">'mrpc'</span>, <span class="hljs-string">'qqp'</span>, <span class="hljs-string">'stsb'</span>, <span class="hljs-string">'mnli'</span>, <span class="hljs-string">'mnli_mismatched'</span>, <span class="hljs-string">'mnli_matched'</span>, <span class="hljs-string">'qnli'</span>, <span class="hljs-string">'rte'</span>, <span class="hljs-string">'wnli'</span>, <span class="hljs-string">'ax'</span>]
	Example of usage:
	load_dataset(<span class="hljs-string">'glue'</span>, <span class="hljs-string">'cola'</span>)<!-- HTML_TAG_END --></pre></div>
	<p>✅ Correct way to load a configuration:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'glue'</span>, <span class="hljs-string">'sst2'</span>)
	Downloading <span class="hljs-keyword">and</span> preparing dataset glue/sst2 (download: <span class="hljs-number">7.09</span> MiB, generated: <span class="hljs-number">4.81</span> MiB, total: <span class="hljs-number">11.90</span> MiB) to /Users/thomwolf/.cache/huggingface/datasets/glue/sst2/<span class="hljs-number">1.0</span><span class="hljs-number">.0</span>...
	Downloading: <span class="hljs-number">100</span>%\|██████████████████████████████████████████████████████████████\| <span class="hljs-number">7.44</span>M/<span class="hljs-number">7.44</span>M [<span class="hljs-number">00</span>:01<<span class="hljs-number">00</span>:<span class="hljs-number">00</span>, <span class="hljs-number">7.03</span>MB/s]
	Dataset glue downloaded <span class="hljs-keyword">and</span> prepared to /Users/thomwolf/.cache/huggingface/datasets/glue/sst2/<span class="hljs-number">1.0</span><span class="hljs-number">.0</span>. Subsequent calls will reuse this data.
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(dataset)
	{<span class="hljs-string">'train'</span>: Dataset(schema: {<span class="hljs-string">'sentence'</span>: <span class="hljs-string">'string'</span>, <span class="hljs-string">'label'</span>: <span class="hljs-string">'int64'</span>, <span class="hljs-string">'idx'</span>: <span class="hljs-string">'int32'</span>}, num_rows: <span class="hljs-number">67349</span>),
	<span class="hljs-string">'validation'</span>: Dataset(schema: {<span class="hljs-string">'sentence'</span>: <span class="hljs-string">'string'</span>, <span class="hljs-string">'label'</span>: <span class="hljs-string">'int64'</span>, <span class="hljs-string">'idx'</span>: <span class="hljs-string">'int32'</span>}, num_rows: <span class="hljs-number">872</span>),
	<span class="hljs-string">'test'</span>: Dataset(schema: {<span class="hljs-string">'sentence'</span>: <span class="hljs-string">'string'</span>, <span class="hljs-string">'label'</span>: <span class="hljs-string">'int64'</span>, <span class="hljs-string">'idx'</span>: <span class="hljs-string">'int32'</span>}, num_rows: <span class="hljs-number">1821</span>)
	}<!-- HTML_TAG_END --></pre></div>
	<h2 class="relative group"><a id="select-a-split" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#select-a-split"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a>
	<span>Select a split
	</span></h2>

	<p>A split is a specific subset of the dataset like <code>train</code> and <code>test</code>. Make sure you select a split when you load a dataset. If you don’t supply a <code>split</code> argument, 🤗 Datasets will only return a dictionary containing the subsets of the dataset.</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
	<span class="hljs-meta">>>> </span>datasets = load_dataset(<span class="hljs-string">'glue'</span>, <span class="hljs-string">'mrpc'</span>)
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(datasets)
	{train: Dataset({
	features: [<span class="hljs-string">'idx'</span>, <span class="hljs-string">'label'</span>, <span class="hljs-string">'sentence1'</span>, <span class="hljs-string">'sentence2'</span>],
	num_rows: <span class="hljs-number">3668</span>
	})
	validation: Dataset({
	features: [<span class="hljs-string">'idx'</span>, <span class="hljs-string">'label'</span>, <span class="hljs-string">'sentence1'</span>, <span class="hljs-string">'sentence2'</span>],
	num_rows: <span class="hljs-number">408</span>
	})
	test: Dataset({
	features: [<span class="hljs-string">'idx'</span>, <span class="hljs-string">'label'</span>, <span class="hljs-string">'sentence1'</span>, <span class="hljs-string">'sentence2'</span>],
	num_rows: <span class="hljs-number">1725</span>
	})
	}<!-- HTML_TAG_END --></pre></div>
	<p>You can list the split names for a dataset, or a specific configuration, with the <a href="/docs/datasets/v2.2.2/en/package_reference/loading_methods#datasets.get_dataset_split_names">get_dataset_split_names()</a> method:</p>

	<div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg>
	<div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div>
	Copied</div></button></div>
	<pre><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> get_dataset_split_names
	<span class="hljs-meta">>>> </span>get_dataset_split_names(<span class="hljs-string">'sent_comp'</span>)
	[<span class="hljs-string">'validation'</span>, <span class="hljs-string">'train'</span>]
	<span class="hljs-meta">>>> </span>get_dataset_split_names(<span class="hljs-string">'glue'</span>, <span class="hljs-string">'cola'</span>)
	[<span class="hljs-string">'test'</span>, <span class="hljs-string">'train'</span>, <span class="hljs-string">'validation'</span>]<!-- HTML_TAG_END --></pre></div>


	<script type="module" data-hydrate="1f529iz">
	import { start } from "/docs/datasets/v2.2.2/en/_app/start-0f8c1da7.js";
	start({
	target: document.querySelector('[data-hydrate="1f529iz"]').parentNode,
	paths: {"base":"/docs/datasets/v2.2.2/en","assets":"/docs/datasets/v2.2.2/en"},
	session: {},
	route: false,
	spa: false,
	trailing_slash: "never",
	hydrate: {
	status: 200,
	error: null,
	nodes: [
	import("/docs/datasets/v2.2.2/en/_app/pages/__layout.svelte-efb8e839.js"),
	import("/docs/datasets/v2.2.2/en/_app/pages/load_hub.mdx-966ed259.js")
	],
	params: {}
	}
	});
	</script>

Xet Storage Details

Size:: 26.6 kB
Xet hash:: c768136cc67549303150bc7acdc6062be294bc3ee7ca9862eb8d94e50c29c435

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.