Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Stream","local":"stream","sections":[{"title":"Convert from a Dataset","local":"convert-from-a-dataset","sections":[],"depth":2},{"title":"Shuffle","local":"shuffle","sections":[],"depth":2},{"title":"Reshuffle","local":"reshuffle","sections":[],"depth":2},{"title":"Split dataset","local":"split-dataset","sections":[],"depth":2},{"title":"Interleave","local":"interleave","sections":[],"depth":2},{"title":"Rename, remove, and cast","local":"rename-remove-and-cast","sections":[{"title":"Rename","local":"rename","sections":[],"depth":3},{"title":"Remove","local":"remove","sections":[],"depth":3},{"title":"Cast","local":"cast","sections":[],"depth":3}],"depth":2},{"title":"Map","local":"map","sections":[{"title":"Batch processing","local":"batch-processing","sections":[{"title":"Tokenization","local":"tokenization","sections":[],"depth":4}],"depth":3},{"title":"Filter","local":"filter","sections":[],"depth":3}],"depth":2},{"title":"Batch","local":"batch","sections":[],"depth":2},{"title":"Stream in a training loop","local":"stream-in-a-training-loop","sections":[{"title":"Save a dataset checkpoint and resume iteration","local":"save-a-dataset-checkpoint-and-resume-iteration","sections":[],"depth":3}],"depth":2}],"depth":1}"> | |
| <link href="/docs/datasets/main/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/entry/start.4d44eea4.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/scheduler.bdbef820.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/singletons.36b689ad.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/index.8a885b74.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/paths.27092e28.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/entry/app.d83067e8.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/index.c0aea24a.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/nodes/0.bfb01985.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/nodes/42.8654077a.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/Tip.31005f7d.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/CodeBlock.6ccca92e.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/Markdown.1f17db59.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/globals.7f7f1b26.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/IconTensorflow.bdd96fa9.js"> | |
| <link rel="modulepreload" href="/docs/datasets/main/en/_app/immutable/chunks/EditOnGithub.725ee0c1.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Stream","local":"stream","sections":[{"title":"Convert from a Dataset","local":"convert-from-a-dataset","sections":[],"depth":2},{"title":"Shuffle","local":"shuffle","sections":[],"depth":2},{"title":"Reshuffle","local":"reshuffle","sections":[],"depth":2},{"title":"Split dataset","local":"split-dataset","sections":[],"depth":2},{"title":"Interleave","local":"interleave","sections":[],"depth":2},{"title":"Rename, remove, and cast","local":"rename-remove-and-cast","sections":[{"title":"Rename","local":"rename","sections":[],"depth":3},{"title":"Remove","local":"remove","sections":[],"depth":3},{"title":"Cast","local":"cast","sections":[],"depth":3}],"depth":2},{"title":"Map","local":"map","sections":[{"title":"Batch processing","local":"batch-processing","sections":[{"title":"Tokenization","local":"tokenization","sections":[],"depth":4}],"depth":3},{"title":"Filter","local":"filter","sections":[],"depth":3}],"depth":2},{"title":"Batch","local":"batch","sections":[],"depth":2},{"title":"Stream in a training loop","local":"stream-in-a-training-loop","sections":[{"title":"Save a dataset checkpoint and resume iteration","local":"save-a-dataset-checkpoint-and-resume-iteration","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="stream" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#stream"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Stream</span></h1> <p data-svelte-h="svelte-1sf84jt">Dataset streaming lets you work with a dataset without downloading it. | |
| The data is streamed as you iterate over the dataset. | |
| This is especially helpful when:</p> <ul data-svelte-h="svelte-1y6qvic"><li>You don’t want to wait for an extremely large dataset to download.</li> <li>The dataset size exceeds the amount of available disk space on your computer.</li> <li>You want to quickly explore just a few samples of a dataset.</li></ul> <div class="flex justify-center" data-svelte-h="svelte-p91s8d"><img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/streaming.gif"> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/streaming-dark.gif"></div> <p data-svelte-h="svelte-1gdgj03">For example, the English split of the <a href="https://huggingface.co/datasets/oscar-corpus/OSCAR-2201" rel="nofollow">oscar-corpus/OSCAR-2201</a> dataset is 1.2 terabytes, but you can use it instantly with streaming. Stream a dataset by setting <code>streaming=True</code> in <a href="/docs/datasets/main/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a> as shown below:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'oscar-corpus/OSCAR-2201'</span>, <span class="hljs-string">'en'</span>, split=<span class="hljs-string">'train'</span>, streaming=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(dataset))) | |
| {<span class="hljs-string">'id'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'Founded in 2015, Golden Bees is a leading programmatic recruitment platform dedicated to employers, HR agencies and job boards. The company has developed unique HR-custom technologies and predictive algorithms to identify and attract the best candidates for a job opportunity.'</span>, ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-xdwdpa">Dataset streaming also lets you work with a dataset made of local files without doing any conversion. | |
| In this case, the data is streamed from the local files as you iterate over the dataset. | |
| This is especially helpful when:</p> <ul data-svelte-h="svelte-lffs3"><li>You don’t want to wait for an extremely large local dataset to be converted to Arrow.</li> <li>The converted files size would exceed the amount of available disk space on your computer.</li> <li>You want to quickly explore just a few samples of a dataset.</li></ul> <p data-svelte-h="svelte-1newt0m">For example, you can stream a local dataset of hundreds of compressed JSONL files like <a href="https://huggingface.co/datasets/oscar-corpus/OSCAR-2201" rel="nofollow">oscar-corpus/OSCAR-2201</a> to use it instantly:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>data_files = {<span class="hljs-string">'train'</span>: <span class="hljs-string">'path/to/OSCAR-2201/compressed/en_meta/*.jsonl.gz'</span>} | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'json'</span>, data_files=data_files, split=<span class="hljs-string">'train'</span>, streaming=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(<span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(dataset))) | |
| {<span class="hljs-string">'id'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'Founded in 2015, Golden Bees is a leading programmatic recruitment platform dedicated to employers, HR agencies and job boards. The company has developed unique HR-custom technologies and predictive algorithms to identify and attract the best candidates for a job opportunity.'</span>, ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1v9hbju">Loading a dataset in streaming mode creates a new dataset type instance (instead of the classic <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object), known as an <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>. | |
| This special type of dataset has its own set of processing methods shown below.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-b07mzo">An <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> is useful for iterative jobs like training a model. | |
| You shouldn’t use a <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> for jobs that require random access to examples because you have to iterate all over it using a for loop. Getting the last example in an iterable dataset would require you to iterate over all the previous examples. | |
| You can find more details in the <a href="./about_mapstyle_vs_iterable">Dataset vs. IterableDataset guide</a>.</p></div> <h2 class="relative group"><a id="convert-from-a-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#convert-from-a-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Convert from a Dataset</span></h2> <p data-svelte-h="svelte-9d16wo">If you have an existing <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object, you can convert it to an <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> with the <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.to_iterable_dataset">to_iterable_dataset()</a> function. This is actually faster than setting the <code>streaming=True</code> argument in <a href="/docs/datasets/main/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a> because the data is streamed from local files.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-comment"># faster 🐇</span> | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"food101"</span>) | |
| <span class="hljs-meta">>>> </span>iterable_dataset = dataset.to_iterable_dataset() | |
| <span class="hljs-comment"># slower 🐢</span> | |
| <span class="hljs-meta">>>> </span>iterable_dataset = load_dataset(<span class="hljs-string">"food101"</span>, streaming=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1oppubr">The <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.to_iterable_dataset">to_iterable_dataset()</a> function supports sharding when the <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> is instantiated. This is useful when working with big datasets, and you’d like to shuffle the dataset or to enable fast parallel loading with a PyTorch DataLoader.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"food101"</span>) | |
| <span class="hljs-meta">>>> </span>iterable_dataset = dataset.to_iterable_dataset(num_shards=<span class="hljs-number">64</span>) <span class="hljs-comment"># shard the dataset</span> | |
| <span class="hljs-meta">>>> </span>iterable_dataset = iterable_dataset.shuffle(buffer_size=<span class="hljs-number">10_000</span>) <span class="hljs-comment"># shuffles the shards order and use a shuffle buffer when you start iterating</span> | |
| dataloader = torch.utils.data.DataLoader(iterable_dataset, num_workers=<span class="hljs-number">4</span>) <span class="hljs-comment"># assigns 64 / 4 = 16 shards from the shuffled list of shards to each worker when you start iterating</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="shuffle" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#shuffle"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Shuffle</span></h2> <p data-svelte-h="svelte-1pw5r1t">Like a regular <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object, you can also shuffle a <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> with <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.shuffle">IterableDataset.shuffle()</a>.</p> <p data-svelte-h="svelte-pt67rh">The <code>buffer_size</code> argument controls the size of the buffer to randomly sample examples from. Let’s say your dataset has one million examples, and you set the <code>buffer_size</code> to ten thousand. <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.shuffle">IterableDataset.shuffle()</a> will randomly select examples from the first ten thousand examples in the buffer. Selected examples in the buffer are replaced with new examples. By default, the buffer size is 1,000.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'oscar'</span>, <span class="hljs-string">"unshuffled_deduplicated_en"</span>, split=<span class="hljs-string">'train'</span>, streaming=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>shuffled_dataset = dataset.shuffle(seed=<span class="hljs-number">42</span>, buffer_size=<span class="hljs-number">10_000</span>)<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-msn2o0"><a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.shuffle">IterableDataset.shuffle()</a> will also shuffle the order of the shards if the dataset is sharded into multiple files.</p></div> <h2 class="relative group"><a id="reshuffle" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#reshuffle"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Reshuffle</span></h2> <p data-svelte-h="svelte-l7znh1">Sometimes you may want to reshuffle the dataset after each epoch. This will require you to set a different seed for each epoch. Use <code>IterableDataset.set_epoch()</code> in between epochs to tell the dataset what epoch you’re on.</p> <p data-svelte-h="svelte-flznjz">Your seed effectively becomes: <code>initial seed + current epoch</code>.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(epochs): | |
| <span class="hljs-meta">... </span> shuffled_dataset.set_epoch(epoch) | |
| <span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> shuffled_dataset: | |
| <span class="hljs-meta">... </span> ...<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="split-dataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#split-dataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Split dataset</span></h2> <p data-svelte-h="svelte-1y9p9l9">You can split your dataset one of two ways:</p> <ul data-svelte-h="svelte-1jm6l4z"><li><a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.take">IterableDataset.take()</a> returns the first <code>n</code> examples in a dataset:</li></ul> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'oscar'</span>, <span class="hljs-string">"unshuffled_deduplicated_en"</span>, split=<span class="hljs-string">'train'</span>, streaming=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>dataset_head = dataset.take(<span class="hljs-number">2</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">list</span>(dataset_head) | |
| [{<span class="hljs-string">'id'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'Mtendere Village was...'</span>}, {<span class="hljs-string">'id'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'Lily James cannot fight the music...'</span>}]<!-- HTML_TAG_END --></pre></div> <ul data-svelte-h="svelte-1tg1950"><li><a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.skip">IterableDataset.skip()</a> omits the first <code>n</code> examples in a dataset and returns the remaining examples:</li></ul> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>train_dataset = shuffled_dataset.skip(<span class="hljs-number">1000</span>)<!-- HTML_TAG_END --></pre></div> <div class="course-tip course-tip-orange bg-gradient-to-br dark:bg-gradient-to-r before:border-orange-500 dark:before:border-orange-800 from-orange-50 dark:from-gray-900 to-white dark:to-gray-950 border border-orange-50 text-orange-700 dark:text-gray-400"><p data-svelte-h="svelte-kg825l"><code>take</code> and <code>skip</code> prevent future calls to <code>shuffle</code> because they lock in the order of the shards. You should <code>shuffle</code> your dataset before splitting it.</p></div> <a id="interleave_datasets"></a> <h2 class="relative group"><a id="interleave" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#interleave"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Interleave</span></h2> <p data-svelte-h="svelte-upyohm"><a href="/docs/datasets/main/en/package_reference/main_classes#datasets.interleave_datasets">interleave_datasets()</a> can combine an <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> with other datasets. The combined dataset returns alternating examples from each of the original datasets.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> interleave_datasets | |
| <span class="hljs-meta">>>> </span>en_dataset = load_dataset(<span class="hljs-string">'oscar'</span>, <span class="hljs-string">"unshuffled_deduplicated_en"</span>, split=<span class="hljs-string">'train'</span>, streaming=<span class="hljs-literal">True</span>, trust_remote_code=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>fr_dataset = load_dataset(<span class="hljs-string">'oscar'</span>, <span class="hljs-string">"unshuffled_deduplicated_fr"</span>, split=<span class="hljs-string">'train'</span>, streaming=<span class="hljs-literal">True</span>, trust_remote_code=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>multilingual_dataset = interleave_datasets([en_dataset, fr_dataset]) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">list</span>(multilingual_dataset.take(<span class="hljs-number">2</span>)) | |
| [{<span class="hljs-string">'text'</span>: <span class="hljs-string">'Mtendere Village was inspired by the vision...'</span>}, {<span class="hljs-string">'text'</span>: <span class="hljs-string">"Média de débat d'idées, de culture et de littérature..."</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1t5oa18">Define sampling probabilities from each of the original datasets for more control over how each of them are sampled and combined. Set the <code>probabilities</code> argument with your desired sampling probabilities:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>multilingual_dataset_with_oversampling = interleave_datasets([en_dataset, fr_dataset], probabilities=[<span class="hljs-number">0.8</span>, <span class="hljs-number">0.2</span>], seed=<span class="hljs-number">42</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">list</span>(multilingual_dataset_with_oversampling.take(<span class="hljs-number">2</span>)) | |
| [{<span class="hljs-string">'text'</span>: <span class="hljs-string">'Mtendere Village was inspired by the vision...'</span>}, {<span class="hljs-string">'text'</span>: <span class="hljs-string">'Lily James cannot fight the music...'</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1oi4bk">Around 80% of the final dataset is made of the <code>en_dataset</code>, and 20% of the <code>fr_dataset</code>.</p> <p data-svelte-h="svelte-13euv56">You can also specify the <code>stopping_strategy</code>. The default strategy, <code>first_exhausted</code>, is a subsampling strategy, i.e the dataset construction is stopped as soon one of the dataset runs out of samples. | |
| You can specify <code>stopping_strategy=all_exhausted</code> to execute an oversampling strategy. In this case, the dataset construction is stopped as soon as every samples in every dataset has been added at least once. In practice, it means that if a dataset is exhausted, it will return to the beginning of this dataset until the stop criterion has been reached. | |
| Note that if no sampling probabilities are specified, the new dataset will have <code>max_length_datasets*nb_dataset samples</code>.</p> <h2 class="relative group"><a id="rename-remove-and-cast" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rename-remove-and-cast"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Rename, remove, and cast</span></h2> <p data-svelte-h="svelte-mlzv10">The following methods allow you to modify the columns of a dataset. These methods are useful for renaming or removing columns and changing columns to a new set of features.</p> <h3 class="relative group"><a id="rename" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#rename"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Rename</span></h3> <p data-svelte-h="svelte-1981esk">Use <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.rename_column">IterableDataset.rename_column()</a> when you need to rename a column in your dataset. Features associated with the original column are actually moved under the new column name, instead of just replacing the original column in-place.</p> <p data-svelte-h="svelte-chfhij">Provide <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.rename_column">IterableDataset.rename_column()</a> with the name of the original column, and the new column name:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'mc4'</span>, <span class="hljs-string">'en'</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">'train'</span>, trust_remote_code=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>dataset = dataset.rename_column(<span class="hljs-string">"text"</span>, <span class="hljs-string">"content"</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="remove" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#remove"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Remove</span></h3> <p data-svelte-h="svelte-1r5asza">When you need to remove one or more columns, give <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.remove_columns">IterableDataset.remove_columns()</a> the name of the column to remove. Remove more than one column by providing a list of column names:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'mc4'</span>, <span class="hljs-string">'en'</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">'train'</span>, trust_remote_code=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>dataset = dataset.remove_columns(<span class="hljs-string">'timestamp'</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="cast" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cast"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Cast</span></h3> <p data-svelte-h="svelte-1xittgo"><a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.cast">IterableDataset.cast()</a> changes the feature type of one or more columns. This method takes your new <code>Features</code> as its argument. The following sample code shows how to change the feature types of <code>ClassLabel</code> and <code>Value</code>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'glue'</span>, <span class="hljs-string">'mrpc'</span>, split=<span class="hljs-string">'train'</span>, streaming=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>dataset.features | |
| {<span class="hljs-string">'sentence1'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), | |
| <span class="hljs-string">'sentence2'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), | |
| <span class="hljs-string">'label'</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">'not_equivalent'</span>, <span class="hljs-string">'equivalent'</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), | |
| <span class="hljs-string">'idx'</span>: Value(dtype=<span class="hljs-string">'int32'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)} | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> ClassLabel, Value | |
| <span class="hljs-meta">>>> </span>new_features = dataset.features.copy() | |
| <span class="hljs-meta">>>> </span>new_features[<span class="hljs-string">"label"</span>] = ClassLabel(names=[<span class="hljs-string">'negative'</span>, <span class="hljs-string">'positive'</span>]) | |
| <span class="hljs-meta">>>> </span>new_features[<span class="hljs-string">"idx"</span>] = Value(<span class="hljs-string">'int64'</span>) | |
| <span class="hljs-meta">>>> </span>dataset = dataset.cast(new_features) | |
| <span class="hljs-meta">>>> </span>dataset.features | |
| {<span class="hljs-string">'sentence1'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), | |
| <span class="hljs-string">'sentence2'</span>: Value(dtype=<span class="hljs-string">'string'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), | |
| <span class="hljs-string">'label'</span>: ClassLabel(num_classes=<span class="hljs-number">2</span>, names=[<span class="hljs-string">'negative'</span>, <span class="hljs-string">'positive'</span>], names_file=<span class="hljs-literal">None</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>), | |
| <span class="hljs-string">'idx'</span>: Value(dtype=<span class="hljs-string">'int64'</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-19jja92">Casting only works if the original feature type and new feature type are compatible. For example, you can cast a column with the feature type <code>Value('int32')</code> to <code>Value('bool')</code> if the original column only contains ones and zeros.</p></div> <p data-svelte-h="svelte-w6xmqw">Use <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.cast_column">IterableDataset.cast_column()</a> to change the feature type of just one column. Pass the column name and its new feature type as arguments:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>dataset.features | |
| {<span class="hljs-string">'audio'</span>: Audio(sampling_rate=<span class="hljs-number">44100</span>, mono=<span class="hljs-literal">True</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)} | |
| <span class="hljs-meta">>>> </span>dataset = dataset.cast_column(<span class="hljs-string">"audio"</span>, Audio(sampling_rate=<span class="hljs-number">16000</span>)) | |
| <span class="hljs-meta">>>> </span>dataset.features | |
| {<span class="hljs-string">'audio'</span>: Audio(sampling_rate=<span class="hljs-number">16000</span>, mono=<span class="hljs-literal">True</span>, <span class="hljs-built_in">id</span>=<span class="hljs-literal">None</span>)}<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="map" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#map"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Map</span></h2> <p data-svelte-h="svelte-1h1g2or">Similar to the <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.map">Dataset.map()</a> function for a regular <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset">Dataset</a>, 🤗 Datasets features <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.map">IterableDataset.map()</a> for processing an <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>. | |
| <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.map">IterableDataset.map()</a> applies processing on-the-fly when examples are streamed.</p> <p data-svelte-h="svelte-1rfoeeh">It allows you to apply a processing function to each example in a dataset, independently or in batches. This function can even create new rows and columns.</p> <p data-svelte-h="svelte-zszb9u">The following example demonstrates how to tokenize a <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>. The function needs to accept and output a <code>dict</code>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">add_prefix</span>(<span class="hljs-params">example</span>): | |
| <span class="hljs-meta">... </span> example[<span class="hljs-string">'text'</span>] = <span class="hljs-string">'My text: '</span> + example[<span class="hljs-string">'text'</span>] | |
| <span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> example<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1v8p29n">Next, apply this function to the dataset with <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.map">IterableDataset.map()</a>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'oscar'</span>, <span class="hljs-string">'unshuffled_deduplicated_en'</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">'train'</span>, trust_remote_code=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>updated_dataset = dataset.<span class="hljs-built_in">map</span>(add_prefix) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">list</span>(updated_dataset.take(<span class="hljs-number">3</span>)) | |
| [{<span class="hljs-string">'id'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'My text: Mtendere Village was inspired by...'</span>}, | |
| {<span class="hljs-string">'id'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'My text: Lily James cannot fight the music...'</span>}, | |
| {<span class="hljs-string">'id'</span>: <span class="hljs-number">2</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'My text: "I\'d love to help kickstart...'</span>}]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1f12xbe">Let’s take a look at another example, except this time, you will remove a column with <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.map">IterableDataset.map()</a>. When you remove a column, it is only removed after the example has been provided to the mapped function. This allows the mapped function to use the content of the columns before they are removed.</p> <p data-svelte-h="svelte-zr1qst">Specify the column to remove with the <code>remove_columns</code> argument in <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.map">IterableDataset.map()</a>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>updated_dataset = dataset.<span class="hljs-built_in">map</span>(add_prefix, remove_columns=[<span class="hljs-string">"id"</span>]) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">list</span>(updated_dataset.take(<span class="hljs-number">3</span>)) | |
| [{<span class="hljs-string">'text'</span>: <span class="hljs-string">'My text: Mtendere Village was inspired by...'</span>}, | |
| {<span class="hljs-string">'text'</span>: <span class="hljs-string">'My text: Lily James cannot fight the music...'</span>}, | |
| {<span class="hljs-string">'text'</span>: <span class="hljs-string">'My text: "I\'d love to help kickstart...'</span>}]<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="batch-processing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#batch-processing"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Batch processing</span></h3> <p data-svelte-h="svelte-cewjpj"><a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.map">IterableDataset.map()</a> also supports working with batches of examples. Operate on batches by setting <code>batched=True</code>. The default batch size is 1000, but you can adjust it with the <code>batch_size</code> argument. This opens the door to many interesting applications such as tokenization, splitting long sentences into shorter chunks, and data augmentation.</p> <h4 class="relative group"><a id="tokenization" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tokenization"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Tokenization</span></h4> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">"mc4"</span>, <span class="hljs-string">"en"</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">"train"</span>, trust_remote_code=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">'distilbert-base-uncased'</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">encode</span>(<span class="hljs-params">examples</span>): | |
| <span class="hljs-meta">... </span> <span class="hljs-keyword">return</span> tokenizer(examples[<span class="hljs-string">'text'</span>], truncation=<span class="hljs-literal">True</span>, padding=<span class="hljs-string">'max_length'</span>) | |
| <span class="hljs-meta">>>> </span>dataset = dataset.<span class="hljs-built_in">map</span>(encode, batched=<span class="hljs-literal">True</span>, remove_columns=[<span class="hljs-string">"text"</span>, <span class="hljs-string">"timestamp"</span>, <span class="hljs-string">"url"</span>]) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(dataset)) | |
| {<span class="hljs-string">'input_ids'</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">8466</span>, <span class="hljs-number">1018</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">4029</span>, <span class="hljs-number">2475</span>, <span class="hljs-number">2062</span>, <span class="hljs-number">18558</span>, <span class="hljs-number">3100</span>, <span class="hljs-number">2061</span>, ...,<span class="hljs-number">1106</span>, <span class="hljs-number">3739</span>, <span class="hljs-number">102</span>], | |
| <span class="hljs-string">'attention_mask'</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, ..., <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-csithm">See other examples of batch processing in the <a href="./process#batch-processing">batched map processing</a> documentation. They work the same for iterable datasets.</p></div> <h3 class="relative group"><a id="filter" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#filter"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Filter</span></h3> <p data-svelte-h="svelte-86ngw3">You can filter rows in the dataset based on a predicate function using <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.filter">Dataset.filter()</a>. It returns rows that match a specified condition:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-meta">>>> </span>dataset = load_dataset(<span class="hljs-string">'oscar'</span>, <span class="hljs-string">'unshuffled_deduplicated_en'</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">'train'</span>, trust_remote_code=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span>start_with_ar = dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> example: example[<span class="hljs-string">'text'</span>].startswith(<span class="hljs-string">'Ar'</span>)) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(start_with_ar)) | |
| {<span class="hljs-string">'id'</span>: <span class="hljs-number">4</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'Are you looking for Number the Stars (Essential Modern Classics)?...'</span>}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-19qhh0v"><a href="/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.filter">Dataset.filter()</a> can also filter by indices if you set <code>with_indices=True</code>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>even_dataset = dataset.<span class="hljs-built_in">filter</span>(<span class="hljs-keyword">lambda</span> example, idx: idx % <span class="hljs-number">2</span> == <span class="hljs-number">0</span>, with_indices=<span class="hljs-literal">True</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">list</span>(even_dataset.take(<span class="hljs-number">3</span>)) | |
| [{<span class="hljs-string">'id'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'Mtendere Village was inspired by the vision of Chief Napoleon Dzombe, ...'</span>}, | |
| {<span class="hljs-string">'id'</span>: <span class="hljs-number">2</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'"I\'d love to help kickstart continued development! And 0 EUR/month...'</span>}, | |
| {<span class="hljs-string">'id'</span>: <span class="hljs-number">4</span>, <span class="hljs-string">'text'</span>: <span class="hljs-string">'Are you looking for Number the Stars (Essential Modern Classics)? Normally, ...'</span>}]<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="batch" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#batch"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Batch</span></h2> <p data-svelte-h="svelte-144pwpf">The <code>batch</code> method transforms your <code>IterableDataset</code> into an iterable of batches. This is particularly useful when you want to work with batches in your training loop or when using frameworks that expect batched inputs.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-f0d9pr">There is also a “Batch Processing” option when using the <code>map</code> function to apply a function to batches of data, which is discussed in the <a href="#map">Map section</a> above. The <code>batch</code> method described here is different and provides a more direct way to create batches from your dataset.</p></div> <p data-svelte-h="svelte-la94za">You can use the <code>batch</code> method like this:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset | |
| <span class="hljs-comment"># Load a dataset in streaming mode</span> | |
| dataset = load_dataset(<span class="hljs-string">"some_dataset"</span>, split=<span class="hljs-string">"train"</span>, streaming=<span class="hljs-literal">True</span>) | |
| <span class="hljs-comment"># Create batches of 32 samples</span> | |
| batched_dataset = dataset.batch(batch_size=<span class="hljs-number">32</span>) | |
| <span class="hljs-comment"># Iterate over the batched dataset</span> | |
| <span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> batched_dataset: | |
| <span class="hljs-built_in">print</span>(batch) | |
| <span class="hljs-keyword">break</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-z19uc5">In this example, batched_dataset is still an IterableDataset, but each item yielded is now a batch of 32 samples instead of a single sample. | |
| This batching is done on-the-fly as you iterate over the dataset, preserving the memory-efficient nature of IterableDataset.</p> <p data-svelte-h="svelte-1srxqyd">The batch method also provides a drop_last_batch parameter. | |
| When set to True, it will discard the last batch if it’s smaller than the specified batch_size. | |
| This can be useful in scenarios where your downstream processing requires all batches to be of the same size:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->batched_dataset = dataset.batch(batch_size=<span class="hljs-number">32</span>, drop_last_batch=<span class="hljs-literal">True</span>)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="stream-in-a-training-loop" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#stream-in-a-training-loop"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Stream in a training loop</span></h2> <p data-svelte-h="svelte-3lon84"><a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> can be integrated into a training loop. First, shuffle the dataset:</p> <div class="space-y-10 py-6 2xl:py-8 2xl:-mx-4"> <div class="border border-gray-200 rounded-xl px-4 relative"><div class="flex h-[22px] mt-[-12.5px] justify-between leading-none"><div class="flex px-1 items-center space-x-1 bg-white dark:bg-gray-950"><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><defs><clipPath id="a"><rect x="3.05" y="0.5" width="25.73" height="31" fill="none"></rect></clipPath></defs><g clip-path="url(#a)"><path d="M24.94,9.51a12.81,12.81,0,0,1,0,18.16,12.68,12.68,0,0,1-18,0,12.81,12.81,0,0,1,0-18.16l9-9V5l-.84.83-6,6a9.58,9.58,0,1,0,13.55,0ZM20.44,9a1.68,1.68,0,1,1,1.67-1.67A1.68,1.68,0,0,1,20.44,9Z" fill="#ee4c2c"></path></g></svg> <span>Pytorch</span></div> <div class="cursor-pointer flex items-center justify-center space-x-1 text-sm px-2 bg-white dark:bg-gray-950 hover:underline leading-none"><svg class="" width="0.9em" height="0.9em" viewBox="0 0 10 9" fill="currentColor" xmlns="http://www.w3.org/2000/svg"><path d="M1.39125 1.9725L0.0883333 0.669997L0.677917 0.0804138L8.9275 8.33041L8.33792 8.91958L6.95875 7.54041C6.22592 8.00523 5.37572 8.25138 4.50792 8.25C2.26125 8.25 0.392083 6.63333 0 4.5C0.179179 3.52946 0.667345 2.64287 1.39167 1.9725H1.39125ZM5.65667 6.23833L5.04667 5.62833C4.81335 5.73996 4.55116 5.77647 4.29622 5.73282C4.04129 5.68918 3.80617 5.56752 3.62328 5.38463C3.44039 5.20175 3.31874 4.96663 3.27509 4.71169C3.23144 4.45676 3.26795 4.19456 3.37958 3.96125L2.76958 3.35125C2.50447 3.75187 2.38595 4.2318 2.4341 4.70978C2.48225 5.18777 2.6941 5.63442 3.0338 5.97411C3.37349 6.31381 3.82015 6.52567 4.29813 6.57382C4.77611 6.62197 5.25605 6.50345 5.65667 6.23833ZM2.83042 1.06666C3.35 0.862497 3.91625 0.749997 4.50792 0.749997C6.75458 0.749997 8.62375 2.36666 9.01583 4.5C8.88816 5.19404 8.60119 5.84899 8.1775 6.41333L6.56917 4.805C6.61694 4.48317 6.58868 4.15463 6.48664 3.84569C6.3846 3.53675 6.21162 3.256 5.98156 3.02594C5.7515 2.79588 5.47075 2.6229 5.16181 2.52086C4.85287 2.41882 4.52433 2.39056 4.2025 2.43833L2.83042 1.06708V1.06666Z" fill="currentColor"></path></svg> <span>Hide Pytorch content</span></div></div> <div class="framework-content"> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>seed, buffer_size = <span class="hljs-number">42</span>, <span class="hljs-number">10_000</span> | |
| <span class="hljs-meta">>>> </span>dataset = dataset.shuffle(seed, buffer_size=buffer_size)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-uyzjf">Lastly, create a simple training loop and start training:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForMaskedLM, DataCollatorForLanguageModeling | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm | |
| <span class="hljs-meta">>>> </span>dataset = dataset.with_format(<span class="hljs-string">"torch"</span>) | |
| <span class="hljs-meta">>>> </span>dataloader = DataLoader(dataset, collate_fn=DataCollatorForLanguageModeling(tokenizer)) | |
| <span class="hljs-meta">>>> </span>device = <span class="hljs-string">'cuda'</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">'cpu'</span> | |
| <span class="hljs-meta">>>> </span>model = AutoModelForMaskedLM.from_pretrained(<span class="hljs-string">"distilbert-base-uncased"</span>) | |
| <span class="hljs-meta">>>> </span>model.train().to(device) | |
| <span class="hljs-meta">>>> </span>optimizer = torch.optim.AdamW(params=model.parameters(), lr=<span class="hljs-number">1e-5</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>): | |
| <span class="hljs-meta">... </span> dataset.set_epoch(epoch) | |
| <span class="hljs-meta">... </span> <span class="hljs-keyword">for</span> i, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(tqdm(dataloader, total=<span class="hljs-number">5</span>)): | |
| <span class="hljs-meta">... </span> <span class="hljs-keyword">if</span> i == <span class="hljs-number">5</span>: | |
| <span class="hljs-meta">... </span> <span class="hljs-keyword">break</span> | |
| <span class="hljs-meta">... </span> batch = {k: v.to(device) <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> batch.items()} | |
| <span class="hljs-meta">... </span> outputs = model(**batch) | |
| <span class="hljs-meta">... </span> loss = outputs[<span class="hljs-number">0</span>] | |
| <span class="hljs-meta">... </span> loss.backward() | |
| <span class="hljs-meta">... </span> optimizer.step() | |
| <span class="hljs-meta">... </span> optimizer.zero_grad() | |
| <span class="hljs-meta">... </span> <span class="hljs-keyword">if</span> i % <span class="hljs-number">10</span> == <span class="hljs-number">0</span>: | |
| <span class="hljs-meta">... </span> <span class="hljs-built_in">print</span>(<span class="hljs-string">f"loss: <span class="hljs-subst">{loss}</span>"</span>)<!-- HTML_TAG_END --></pre></div></div></div> </div> <h3 class="relative group"><a id="save-a-dataset-checkpoint-and-resume-iteration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#save-a-dataset-checkpoint-and-resume-iteration"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Save a dataset checkpoint and resume iteration</span></h3> <p data-svelte-h="svelte-qoizba">If your training loop stops, you may want to restart the training from where it was. To do so you can save a checkpoint of your model and optimizers, as well as your data loader.</p> <p data-svelte-h="svelte-hrq78r">Iterable datasets don’t provide random access to a specific example index to resume from, but you can use <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.state_dict">IterableDataset.state_dict()</a> and <a href="/docs/datasets/main/en/package_reference/main_classes#datasets.IterableDataset.load_state_dict">IterableDataset.load_state_dict()</a> to resume from a checkpoint instead, similarly to what you can do for models and optimizers:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span>iterable_dataset = Dataset.from_dict({<span class="hljs-string">"a"</span>: <span class="hljs-built_in">range</span>(<span class="hljs-number">6</span>)}).to_iterable_dataset(num_shards=<span class="hljs-number">3</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> idx, example <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(iterable_dataset): | |
| <span class="hljs-meta">... </span> <span class="hljs-built_in">print</span>(example) | |
| <span class="hljs-meta">... </span> <span class="hljs-keyword">if</span> idx == <span class="hljs-number">2</span>: | |
| <span class="hljs-meta">... </span> state_dict = iterable_dataset.state_dict() | |
| <span class="hljs-meta">... </span> <span class="hljs-built_in">print</span>(<span class="hljs-string">"checkpoint"</span>) | |
| <span class="hljs-meta">... </span> <span class="hljs-keyword">break</span> | |
| <span class="hljs-meta">>>> </span>iterable_dataset.load_state_dict(state_dict) | |
| <span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(<span class="hljs-string">f"restart from checkpoint"</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> iterable_dataset: | |
| <span class="hljs-meta">... </span> <span class="hljs-built_in">print</span>(example)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-f816x">Returns:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->{<span class="hljs-string">'a'</span>: <span class="hljs-number">0</span>} | |
| {<span class="hljs-string">'a'</span>: <span class="hljs-number">1</span>} | |
| {<span class="hljs-string">'a'</span>: <span class="hljs-number">2</span>} | |
| <span class="hljs-keyword">checkpoint</span> | |
| <span class="hljs-keyword">restart</span> <span class="hljs-keyword">from</span> <span class="hljs-keyword">checkpoint</span> | |
| {<span class="hljs-string">'a'</span>: <span class="hljs-number">3</span>} | |
| {<span class="hljs-string">'a'</span>: <span class="hljs-number">4</span>} | |
| {<span class="hljs-string">'a'</span>: <span class="hljs-number">5</span>}<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ul64r3">Under the hood, the iterable dataset keeps track of the current shard being read and the example index in the current shard and it stores this info in the <code>state_dict</code>.</p> <p data-svelte-h="svelte-fghrva">To resume from a checkpoint, the dataset skips all the shards that were previously read to restart from the current shard. | |
| Then it reads the shard and skips examples until it reaches the exact example from the checkpoint.</p> <p data-svelte-h="svelte-1i92o70">Therefore restarting a dataset is quite fast, since it will not re-read the shards that have already been iterated on. Still, resuming a dataset is generally not instantaneous since it has to restart reading from the beginning of the current shard and skip examples until it reaches the checkpoint location.</p> <p data-svelte-h="svelte-10pdp7c">This can be used with the <code>StatefulDataLoader</code> from <code>torchdata</code>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> torchdata.stateful_dataloader <span class="hljs-keyword">import</span> StatefulDataLoader | |
| <span class="hljs-meta">>>> </span>iterable_dataset = load_dataset(<span class="hljs-string">"deepmind/code_contests"</span>, streaming=<span class="hljs-literal">True</span>, split=<span class="hljs-string">"train"</span>) | |
| <span class="hljs-meta">>>> </span>dataloader = StatefulDataLoader(iterable_dataset, batch_size=<span class="hljs-number">32</span>, num_workers=<span class="hljs-number">4</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># checkpoint</span> | |
| <span class="hljs-meta">>>> </span>state_dict = dataloader.state_dict() <span class="hljs-comment"># uses iterable_dataset.state_dict() under the hood</span> | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># resume from checkpoint</span> | |
| <span class="hljs-meta">>>> </span>dataloader.load_state_dict(state_dict) <span class="hljs-comment"># uses iterable_dataset.load_state_dict() under the hood</span><!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1aaqobe">Resuming returns exactly where the checkpoint was saved except if <code>.shuffle()</code> is used: examples from shuffle buffers are lost when resuming and the buffers are refilled with new data.</p></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/datasets/blob/main/docs/source/stream.mdx" target="_blank"><span data-svelte-h="svelte-1kd6by1"><</span> <span data-svelte-h="svelte-x0xyl0">></span> <span data-svelte-h="svelte-1dajgef"><span class="underline ml-1.5">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_w3org2 = { | |
| assets: "/docs/datasets/main/en", | |
| base: "/docs/datasets/main/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/datasets/main/en/_app/immutable/entry/start.4d44eea4.js"), | |
| import("/docs/datasets/main/en/_app/immutable/entry/app.d83067e8.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 42], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 109 kB
- Xet hash:
- 432be94c548a182cc3e2b94872c723b511b993a530c61da425937b51e77e9612
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.