datasets/pr_8055/en/_app/immutable/nodes/7.aaa08e6f.js

import{s as Ms,n as us,o as bs}from"../chunks/scheduler.d75c11ed.js";import{S as js,i as ws,e as i,s as l,c as d,h as _s,a as r,d as s,b as n,f as ys,g as o,j as p,k as Ra,l as Ts,m as t,n as c,t as h,o as m,p as f}from"../chunks/index.4ec9dfe9.js";import{C as Js,H as b,E as gs}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.bf3c6a50.js";import{C as y}from"../chunks/CodeBlock.ed5a8c30.js";function Us(Ba){let M,Ze,ge,Xe,j,Ge,w,Ie,_,Fa=`There are two types of dataset objects, a <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> and an <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>.
Whichever type of dataset you choose to use or create depends on the size of the dataset.
In general, an <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> is ideal for big datasets (think hundreds of GBs!) due to its lazy behavior and speed advantages, while a <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> is great for everything else.
This page will compare the differences between a <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> and an <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> to help you pick the right dataset object for you.`,ke,T,$e,J,xa=`When you have a regular <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a>, you can access it using <code>my_dataset[0]</code>. This provides random access to the rows.
Such datasets are also called “map-style” datasets.
For example you can download ImageNet-1k like this and access any row:`,Ce,g,Ve,U,Ya=`But one caveat is that you must have the entire dataset stored on your disk or in memory, which blocks you from accessing datasets bigger than the disk.
Because it can become inconvenient for big datasets, there exists another type of dataset, the <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>.
When you have an <code>IterableDataset</code>, you can access it using a <code>for</code> loop to load the data progressively as you iterate over the dataset.
This way, only a small fraction of examples is loaded in memory, and you don’t write anything on disk.`,ve,Z,za="For example, you can stream the ImageNet-1k dataset without downloading it on disk:",Re,X,Be,G,Wa=`Streaming can read online data without writing any file to disk.
For example, you can stream datasets made out of multiple shards, each of which is hundreds of gigabytes like <a href="https://huggingface.co/datasets/c4" rel="nofollow">C4</a>  or <a href="https://huggingface.co/datasets/laion/laion2B-en" rel="nofollow">LAION-2B</a>.
Learn more about how to stream a dataset in the <a href="./stream">Dataset Streaming Guide</a>.`,Fe,I,Na="This is not the only difference though, because the “lazy” behavior of an <code>IterableDataset</code> is also present when it comes to dataset creation and processing.",xe,k,Ye,$,Qa='You can create a <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> using lists or dictionaries, and the data is entirely converted to Arrow so you can easily access any row:',ze,C,We,V,Da=`To create an <code>IterableDataset</code> on the other hand, you must provide a “lazy” way to load the data.
In Python, we generally use generator functions. These functions <code>yield</code> one example at a time, which means you can’t access a row by slicing it like a regular <code>Dataset</code>:`,Ne,v,Qe,R,De,B,Ha='It is possible to convert local or remote data files to an Arrow <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> using <a href="/docs/datasets/pr_8055/en/package_reference/loading_methods#datasets.load_dataset">load_dataset()</a>:',He,F,Ae,x,Aa="However, this requires a conversion step from CSV to Arrow format, which takes time and disk space if your dataset is big.",Ee,Y,Ea=`To save disk space and skip the conversion step, you can define an <code>IterableDataset</code> by streaming from the local files directly.
This way, the data is read progressively from the local files as you iterate over the dataset:`,Se,z,Le,W,Sa=`Many file formats are supported, like CSV, JSONL, and Parquet, as well as image and audio files.
You can find more information in the corresponding guides for loading <a href="./tabular_load">tabular</a>, <a href="./nlp_load">text</a>, <a href="./image_load">vision</a>, and <a href="./audio_load%5D">audio</a> datasets.`,qe,N,Pe,Q,La=`When you process a <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> object using <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset.map">Dataset.map()</a>, the entire dataset is processed immediately and returned.
This is similar to how <code>pandas</code> works for example.`,Ke,D,Oe,H,qa=`On the other hand, due to the “lazy” nature of an <code>IterableDataset</code>, calling <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset.map">IterableDataset.map()</a> does not apply your <code>map</code> function over the full dataset.
Instead, your <code>map</code> function is applied on-the-fly.`,ea,A,Pa="Because of that, you can chain multiple processing steps and they will all run at once when you start iterating over the dataset:",aa,E,sa,S,ta,L,Ka=`When you shuffle a <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> using <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset.shuffle">Dataset.shuffle()</a>, you apply an exact shuffling of the dataset.
It works by taking a list of indices <code>[0, 1, 2, ... len(my_dataset) - 1]</code> and shuffling this list.
Then, accessing <code>my_dataset[0]</code> returns the row and index defined by the first element of the indices mapping that has been shuffled:`,la,q,na,P,Oa=`Since we don’t have random access to the rows in the case of an <code>IterableDataset</code>, we can’t use a shuffled list of indices and access a row at an arbitrary position.
This prevents the use of exact shuffling.
Instead, a fast approximate shuffling is used in <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset.shuffle">IterableDataset.shuffle()</a>.
It uses a shuffle buffer to sample random examples iteratively from the dataset.
Since the dataset is still read iteratively, it provides excellent speed performance:`,ia,K,ra,O,es='But using a shuffle buffer is not enough to provide a satisfactory shuffling for machine learning model training. So <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset.shuffle">IterableDataset.shuffle()</a> also shuffles the dataset shards if your dataset is made of multiple files or sources. Additionally, it fills the buffer using up to <code>max_buffer_input_shards</code> shards at a time (default is 10), which greatly improves the quality of the shuffling:',pa,ee,da,u,as='<p>If your dataset has few files, you can improve shuffling quality by calling <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset.reshard">reshard()</a> before shuffling. For example, for Parquet datasets this reshards using row groups instead of having one file per shard, giving you many more shards to shuffle from.</p>',oa,ae,ca,se,ss=`Regular <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> objects are based on Arrow which provides fast random access to the rows.
Thanks to memory mapping and the fact that Arrow is an in-memory format, reading data from disk doesn’t do expensive system calls and deserialization.
It provides even faster data loading when iterating using a <code>for</code> loop by iterating on contiguous Arrow record batches.`,ha,te,ts=`However as soon as your <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> has an indices mapping (via <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset.shuffle">Dataset.shuffle()</a> for example), the speed can become 10x slower.
This is because there is an extra step to get the row index to read using the indices mapping, and most importantly, you aren’t reading contiguous chunks of data anymore.
To restore the speed, you’d need to rewrite the entire dataset on your disk again using <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset.flatten_indices">Dataset.flatten_indices()</a>, which removes the indices mapping.
This may take a lot of time depending on the size of your dataset though:`,ma,le,fa,ne,ls=`In this case, we recommend switching to an <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> and leveraging its fast approximate shuffling method <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset.shuffle">IterableDataset.shuffle()</a>.
It only shuffles the shards order and adds a shuffle buffer to your dataset, which keeps the speed of your dataset optimal.
You can also reshuffle the dataset easily:`,ya,ie,Ma,re,ns=`If you’re using your dataset on multiple epochs, the effective seed to shuffle the shards order in the shuffle buffer is <code>seed + epoch</code>.
It makes it easy to reshuffle a dataset between epochs:`,ua,pe,ba,de,is="To restart the iteration of a map-style dataset, you can simply skip the first examples:",ja,oe,wa,ce,rs="But if you use a <code>DataLoader</code> with a <code>Sampler</code>, you should instead save the state of your sampler (you might have written a custom sampler that allows resuming).",_a,he,ps='On the other hand, iterable datasets don’t provide random access to a specific example index to resume from. But you can use <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset.state_dict">IterableDataset.state_dict()</a> and <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset.load_state_dict">IterableDataset.load_state_dict()</a> to resume from a checkpoint instead, similarly to what you can do for models and optimizers:',Ta,me,Ja,fe,ds="Under the hood, the iterable dataset keeps track of the current shard being read and the example index in the current shard and it stores this info in the <code>state_dict</code>.",ga,ye,os=`To resume from a checkpoint, the dataset skips all the shards that were previously read to restart from the current shard.
Then it reads the shard and skips examples until it reaches the exact example from the checkpoint.`,Ua,Me,cs="Therefore restarting a dataset is quite fast, since it will not re-read the shards that have already been iterated on. Still, resuming a dataset is generally not instantaneous since it has to restart reading from the beginning of the current shard and skip examples until it reaches the checkpoint location.",Za,ue,hs='This can be used with the <code>StatefulDataLoader</code> from <code>torchdata</code>, see <a href="./use_with_pytorch#stream-data">streaming with a PyTorch DataLoader</a>.',Xa,be,Ga,je,ms='If you want to benefit from the “lazy” behavior of an <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a> or their speed advantages, you can switch your map-style <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.Dataset">Dataset</a> to an <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>:',Ia,we,ka,_e,fs='If you want to shuffle your dataset or <a href="./use_with_pytorch#stream-data">use it with a PyTorch DataLoader</a>, we recommend generating a sharded <a href="/docs/datasets/pr_8055/en/package_reference/main_classes#datasets.IterableDataset">IterableDataset</a>:',$a,Te,Ca,Je,Va,Ue,va;return j=new Js({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new b({props:{title:"Differences between Dataset and IterableDataset",local:"differences-between-dataset-and-iterabledataset",headingTag:"h1"}}),T=new b({props:{title:"Downloading and streaming",local:"downloading-and-streaming",headingTag:"h2"}}),g=new y({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBaW1hZ2VuZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIydGltbSUyRmltYWdlbmV0LTFrLXdkcyUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIpJTIwJTIwJTIzJTIwZG93bmxvYWRzJTIwdGhlJTIwZnVsbCUyMGRhdGFzZXQlMEFwcmludChpbWFnZW5ldCU1QjAlNUQp",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

imagenet = load_dataset(<span class="hljs-string">&quot;timm/imagenet-1k-wds&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)  <span class="hljs-comment"># downloads the full dataset</span>
<span class="hljs-built_in">print</span>(imagenet[<span class="hljs-number">0</span>])`,lang:"python",wrap:!1}}),X=new y({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBaW1hZ2VuZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIydGltbSUyRmltYWdlbmV0LTFrLXdkcyUyMiUyQyUyMHNwbGl0JTNEJTIydHJhaW4lMjIlMkMlMjBzdHJlYW1pbmclM0RUcnVlKSUyMCUyMCUyMyUyMHdpbGwlMjBzdGFydCUyMGxvYWRpbmclMjB0aGUlMjBkYXRhJTIwd2hlbiUyMGl0ZXJhdGVkJTIwb3ZlciUwQWZvciUyMGV4YW1wbGUlMjBpbiUyMGltYWdlbmV0JTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZXhhbXBsZSklMEElMjAlMjAlMjAlMjBicmVhaw==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

imagenet = load_dataset(<span class="hljs-string">&quot;timm/imagenet-1k-wds&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>)  <span class="hljs-comment"># will start loading the data when iterated over</span>
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> imagenet:
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`,lang:"python",wrap:!1}}),k=new b({props:{title:"Creating map-style datasets and iterable datasets",local:"creating-map-style-datasets-and-iterable-datasets",headingTag:"h2"}}),C=new y({props:{code:"bXlfZGF0YXNldCUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCU3QiUyMmNvbF8xJTIyJTNBJTIwJTVCMCUyQyUyMDElMkMlMjAyJTJDJTIwMyUyQyUyMDQlMkMlMjA1JTJDJTIwNiUyQyUyMDclMkMlMjA4JTJDJTIwOSU1RCU3RCklMEFwcmludChteV9kYXRhc2V0JTVCMCU1RCk=",highlighted:`my_dataset = Dataset.from_dict({<span class="hljs-string">&quot;col_1&quot;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">4</span>, <span class="hljs-number">5</span>, <span class="hljs-number">6</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">9</span>]})
<span class="hljs-built_in">print</span>(my_dataset[<span class="hljs-number">0</span>])`,lang:"python",wrap:!1}}),v=new y({props:{code:"ZGVmJTIwbXlfZ2VuZXJhdG9yKG4pJTNBJTBBJTIwJTIwJTIwJTIwZm9yJTIwaSUyMGluJTIwcmFuZ2UobiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB5aWVsZCUyMCU3QiUyMmNvbF8xJTIyJTNBJTIwaSU3RCUwQSUwQW15X2l0ZXJhYmxlX2RhdGFzZXQlMjAlM0QlMjBJdGVyYWJsZURhdGFzZXQuZnJvbV9nZW5lcmF0b3IobXlfZ2VuZXJhdG9yJTJDJTIwZ2VuX2t3YXJncyUzRCU3QiUyMm4lMjIlM0ElMjAxMCU3RCklMEFmb3IlMjBleGFtcGxlJTIwaW4lMjBteV9pdGVyYWJsZV9kYXRhc2V0JTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZXhhbXBsZSklMEElMjAlMjAlMjAlMjBicmVhaw==",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">my_generator</span>(<span class="hljs-params">n</span>):
    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n):
        <span class="hljs-keyword">yield</span> {<span class="hljs-string">&quot;col_1&quot;</span>: i}

my_iterable_dataset = IterableDataset.from_generator(my_generator, gen_kwargs={<span class="hljs-string">&quot;n&quot;</span>: <span class="hljs-number">10</span>})
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`,lang:"python",wrap:!1}}),R=new b({props:{title:"Loading local files entirely and progressively",local:"loading-local-files-entirely-and-progressively",headingTag:"h2"}}),F=new y({props:{code:"ZGF0YV9maWxlcyUyMCUzRCUyMCU3QiUyMnRyYWluJTIyJTNBJTIwJTVCJTIycGF0aCUyRnRvJTJGZGF0YS5jc3YlMjIlNUQlN0QlMEFteV9kYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmNzdiUyMiUyQyUyMGRhdGFfZmlsZXMlM0RkYXRhX2ZpbGVzJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEFwcmludChteV9kYXRhc2V0JTVCMCU1RCk=",highlighted:`data_files = {<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;path/to/data.csv&quot;</span>]}
my_dataset = load_dataset(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-built_in">print</span>(my_dataset[<span class="hljs-number">0</span>])`,lang:"python",wrap:!1}}),z=new y({props:{code:"ZGF0YV9maWxlcyUyMCUzRCUyMCU3QiUyMnRyYWluJTIyJTNBJTIwJTVCJTIycGF0aCUyRnRvJTJGZGF0YS5jc3YlMjIlNUQlN0QlMEFteV9pdGVyYWJsZV9kYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmNzdiUyMiUyQyUyMGRhdGFfZmlsZXMlM0RkYXRhX2ZpbGVzJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiUyQyUyMHN0cmVhbWluZyUzRFRydWUpJTBBZm9yJTIwZXhhbXBsZSUyMGluJTIwbXlfaXRlcmFibGVfZGF0YXNldCUzQSUyMCUyMCUyMyUyMHRoaXMlMjByZWFkcyUyMHRoZSUyMENTViUyMGZpbGUlMjBwcm9ncmVzc2l2ZWx5JTIwYXMlMjB5b3UlMjBpdGVyYXRlJTIwb3ZlciUyMHRoZSUyMGRhdGFzZXQlMEElMjAlMjAlMjAlMjBwcmludChleGFtcGxlKSUwQSUyMCUyMCUyMCUyMGJyZWFr",highlighted:`data_files = {<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;path/to/data.csv&quot;</span>]}
my_iterable_dataset = load_dataset(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>)
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:  <span class="hljs-comment"># this reads the CSV file progressively as you iterate over the dataset</span>
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`,lang:"python",wrap:!1}}),N=new b({props:{title:"Eager data processing and lazy data processing",local:"eager-data-processing-and-lazy-data-processing",headingTag:"h2"}}),D=new y({props:{code:"bXlfZGF0YXNldCUyMCUzRCUyMG15X2RhdGFzZXQubWFwKHByb2Nlc3NfZm4pJTIwJTIwJTIzJTIwcHJvY2Vzc19mbiUyMGlzJTIwYXBwbGllZCUyMG9uJTIwYWxsJTIwdGhlJTIwZXhhbXBsZXMlMjBvZiUyMHRoZSUyMGRhdGFzZXQlMEFwcmludChteV9kYXRhc2V0JTVCMCU1RCk=",highlighted:`my_dataset = my_dataset.<span class="hljs-built_in">map</span>(process_fn)  <span class="hljs-comment"># process_fn is applied on all the examples of the dataset</span>
<span class="hljs-built_in">print</span>(my_dataset[<span class="hljs-number">0</span>])`,lang:"python",wrap:!1}}),E=new y({props:{code:"bXlfaXRlcmFibGVfZGF0YXNldCUyMCUzRCUyMG15X2l0ZXJhYmxlX2RhdGFzZXQubWFwKHByb2Nlc3NfZm5fMSklMEFteV9pdGVyYWJsZV9kYXRhc2V0JTIwJTNEJTIwbXlfaXRlcmFibGVfZGF0YXNldC5maWx0ZXIoZmlsdGVyX2ZuKSUwQW15X2l0ZXJhYmxlX2RhdGFzZXQlMjAlM0QlMjBteV9pdGVyYWJsZV9kYXRhc2V0Lm1hcChwcm9jZXNzX2ZuXzIpJTBBJTBBJTIzJTIwcHJvY2Vzc19mbl8xJTJDJTIwZmlsdGVyX2ZuJTIwYW5kJTIwcHJvY2Vzc19mbl8yJTIwYXJlJTIwYXBwbGllZCUyMG9uLXRoZS1mbHklMjB3aGVuJTIwaXRlcmF0aW5nJTIwb3ZlciUyMHRoZSUyMGRhdGFzZXQlMEFmb3IlMjBleGFtcGxlJTIwaW4lMjBteV9pdGVyYWJsZV9kYXRhc2V0JTNBJTIwJTIwJTBBJTIwJTIwJTIwJTIwcHJpbnQoZXhhbXBsZSklMEElMjAlMjAlMjAlMjBicmVhaw==",highlighted:`my_iterable_dataset = my_iterable_dataset.<span class="hljs-built_in">map</span>(process_fn_1)
my_iterable_dataset = my_iterable_dataset.<span class="hljs-built_in">filter</span>(filter_fn)
my_iterable_dataset = my_iterable_dataset.<span class="hljs-built_in">map</span>(process_fn_2)

<span class="hljs-comment"># process_fn_1, filter_fn and process_fn_2 are applied on-the-fly when iterating over the dataset</span>
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:  
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`,lang:"python",wrap:!1}}),S=new b({props:{title:"Exact and fast approximate shuffling",local:"exact-and-fast-approximate-shuffling",headingTag:"h2"}}),q=new y({props:{code:"bXlfZGF0YXNldCUyMCUzRCUyMG15X2RhdGFzZXQuc2h1ZmZsZShzZWVkJTNENDIpJTBBcHJpbnQobXlfZGF0YXNldCU1QjAlNUQp",highlighted:`my_dataset = my_dataset.shuffle(seed=<span class="hljs-number">42</span>)
<span class="hljs-built_in">print</span>(my_dataset[<span class="hljs-number">0</span>])`,lang:"python",wrap:!1}}),K=new y({props:{code:"bXlfaXRlcmFibGVfZGF0YXNldCUyMCUzRCUyMG15X2l0ZXJhYmxlX2RhdGFzZXQuc2h1ZmZsZShzZWVkJTNENDIlMkMlMjBidWZmZXJfc2l6ZSUzRDEwMCklMEFmb3IlMjBleGFtcGxlJTIwaW4lMjBteV9pdGVyYWJsZV9kYXRhc2V0JTNBJTBBJTIwJTIwJTIwJTIwcHJpbnQoZXhhbXBsZSklMEElMjAlMjAlMjAlMjBicmVhaw==",highlighted:`my_iterable_dataset = my_iterable_dataset.shuffle(seed=<span class="hljs-number">42</span>, buffer_size=<span class="hljs-number">100</span>)
<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:
    <span class="hljs-built_in">print</span>(example)
    <span class="hljs-keyword">break</span>`,lang:"python",wrap:!1}}),ee=new y({props:{code:"JTIzJTIwU3RyZWFtJTIwZnJvbSUyMHRoZSUyMGludGVybmV0JTBBbXlfaXRlcmFibGVfZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJkZWVwbWluZCUyRmNvZGVfY29udGVzdHMlMjIlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyJTJDJTIwc3RyZWFtaW5nJTNEVHJ1ZSklMEFteV9pdGVyYWJsZV9kYXRhc2V0Lm51bV9zaGFyZHMlMjAlMjAlMjMlMjAzOSUwQSUwQSUyMyUyMFN0cmVhbSUyMGZyb20lMjBsb2NhbCUyMGZpbGVzJTBBZGF0YV9maWxlcyUyMCUzRCUyMCU3QiUyMnRyYWluJTIyJTNBJTIwJTVCZiUyMnBhdGglMkZ0byUyRmRhdGFfJTdCaSU3RC5jc3YlMjIlMjBmb3IlMjBpJTIwaW4lMjByYW5nZSgxMDI0KSU1RCU3RCUwQW15X2l0ZXJhYmxlX2RhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyY3N2JTIyJTJDJTIwZGF0YV9maWxlcyUzRGRhdGFfZmlsZXMlMkMlMjBzcGxpdCUzRCUyMnRyYWluJTIyJTJDJTIwc3RyZWFtaW5nJTNEVHJ1ZSklMEFteV9pdGVyYWJsZV9kYXRhc2V0Lm51bV9zaGFyZHMlMjAlMjAlMjMlMjAxMDI0JTBBJTBBJTIzJTIwRnJvbSUyMGElMjBnZW5lcmF0b3IlMjBmdW5jdGlvbiUwQWRlZiUyMG15X2dlbmVyYXRvcihuJTJDJTIwc291cmNlcyklM0ElMEElMjAlMjAlMjAlMjBmb3IlMjBzb3VyY2UlMjBpbiUyMHNvdXJjZXMlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBleGFtcGxlX2lkX2Zvcl9jdXJyZW50X3NvdXJjZSUyMGluJTIwcmFuZ2UobiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB5aWVsZCUyMCU3QiUyMmV4YW1wbGVfaWQlMjIlM0ElMjBmJTIyJTdCc291cmNlJTdEXyU3QmV4YW1wbGVfaWRfZm9yX2N1cnJlbnRfc291cmNlJTdEJTIyJTdEJTBBJTBBZ2VuX2t3YXJncyUyMCUzRCUyMCU3QiUyMm4lMjIlM0ElMjAxMCUyQyUyMCUyMnNvdXJjZXMlMjIlM0ElMjAlNUJmJTIycGF0aCUyRnRvJTJGZGF0YV8lN0JpJTdEJTIyJTIwZm9yJTIwaSUyMGluJTIwcmFuZ2UoMTAyNCklNUQlN0QlMEFteV9pdGVyYWJsZV9kYXRhc2V0JTIwJTNEJTIwSXRlcmFibGVEYXRhc2V0LmZyb21fZ2VuZXJhdG9yKG15X2dlbmVyYXRvciUyQyUyMGdlbl9rd2FyZ3MlM0RnZW5fa3dhcmdzKSUwQW15X2l0ZXJhYmxlX2RhdGFzZXQubnVtX3NoYXJkcyUyMCUyMCUyMyUyMDEwMjQ=",highlighted:`<span class="hljs-comment"># Stream from the internet</span>
my_iterable_dataset = load_dataset(<span class="hljs-string">&quot;deepmind/code_contests&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>)
my_iterable_dataset.num_shards  <span class="hljs-comment"># 39</span>

<span class="hljs-comment"># Stream from local files</span>
data_files = {<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">f&quot;path/to/data_<span class="hljs-subst">{i}</span>.csv&quot;</span> <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1024</span>)]}
my_iterable_dataset = load_dataset(<span class="hljs-string">&quot;csv&quot;</span>, data_files=data_files, split=<span class="hljs-string">&quot;train&quot;</span>, streaming=<span class="hljs-literal">True</span>)
my_iterable_dataset.num_shards  <span class="hljs-comment"># 1024</span>

<span class="hljs-comment"># From a generator function</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">my_generator</span>(<span class="hljs-params">n, sources</span>):
    <span class="hljs-keyword">for</span> source <span class="hljs-keyword">in</span> sources:
        <span class="hljs-keyword">for</span> example_id_for_current_source <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n):
            <span class="hljs-keyword">yield</span> {<span class="hljs-string">&quot;example_id&quot;</span>: <span class="hljs-string">f&quot;<span class="hljs-subst">{source}</span>_<span class="hljs-subst">{example_id_for_current_source}</span>&quot;</span>}

gen_kwargs = {<span class="hljs-string">&quot;n&quot;</span>: <span class="hljs-number">10</span>, <span class="hljs-string">&quot;sources&quot;</span>: [<span class="hljs-string">f&quot;path/to/data_<span class="hljs-subst">{i}</span>&quot;</span> <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1024</span>)]}
my_iterable_dataset = IterableDataset.from_generator(my_generator, gen_kwargs=gen_kwargs)
my_iterable_dataset.num_shards  <span class="hljs-comment"># 1024</span>`,lang:"python",wrap:!1}}),ae=new b({props:{title:"Speed differences",local:"speed-differences",headingTag:"h2"}}),le=new y({props:{code:"bXlfZGF0YXNldCU1QjAlNUQlMjAlMjAlMjMlMjBmYXN0JTBBbXlfZGF0YXNldCUyMCUzRCUyMG15X2RhdGFzZXQuc2h1ZmZsZShzZWVkJTNENDIpJTBBbXlfZGF0YXNldCU1QjAlNUQlMjAlMjAlMjMlMjB1cCUyMHRvJTIwMTB4JTIwc2xvd2VyJTBBbXlfZGF0YXNldCUyMCUzRCUyMG15X2RhdGFzZXQuZmxhdHRlbl9pbmRpY2VzKCklMjAlMjAlMjMlMjByZXdyaXRlJTIwdGhlJTIwc2h1ZmZsZWQlMjBkYXRhc2V0JTIwb24lMjBkaXNrJTIwYXMlMjBjb250aWd1b3VzJTIwY2h1bmtzJTIwb2YlMjBkYXRhJTBBbXlfZGF0YXNldCU1QjAlNUQlMjAlMjAlMjMlMjBmYXN0JTIwYWdhaW4=",highlighted:`my_dataset[<span class="hljs-number">0</span>]  <span class="hljs-comment"># fast</span>
my_dataset = my_dataset.shuffle(seed=<span class="hljs-number">42</span>)
my_dataset[<span class="hljs-number">0</span>]  <span class="hljs-comment"># up to 10x slower</span>
my_dataset = my_dataset.flatten_indices()  <span class="hljs-comment"># rewrite the shuffled dataset on disk as contiguous chunks of data</span>
my_dataset[<span class="hljs-number">0</span>]  <span class="hljs-comment"># fast again</span>`,lang:"python",wrap:!1}}),ie=new y({props:{code:"Zm9yJTIwZXhhbXBsZSUyMGluJTIwZW51bWVyYXRlKG15X2l0ZXJhYmxlX2RhdGFzZXQpJTNBJTIwJTIwJTIzJTIwZmFzdCUwQSUyMCUyMCUyMCUyMHBhc3MlMEElMEFzaHVmZmxlZF9pdGVyYWJsZV9kYXRhc2V0JTIwJTNEJTIwbXlfaXRlcmFibGVfZGF0YXNldC5zaHVmZmxlKHNlZWQlM0Q0MiUyQyUyMGJ1ZmZlcl9zaXplJTNEMTAwKSUwQSUwQWZvciUyMGV4YW1wbGUlMjBpbiUyMGVudW1lcmF0ZShzaHVmZmxlZF9pdGVyYWJsZV9kYXRhc2V0KSUzQSUyMCUyMCUyMyUyMGFzJTIwZmFzdCUyMGFzJTIwYmVmb3JlJTBBJTIwJTIwJTIwJTIwcGFzcyUwQSUwQXNodWZmbGVkX2l0ZXJhYmxlX2RhdGFzZXQlMjAlM0QlMjBteV9pdGVyYWJsZV9kYXRhc2V0LnNodWZmbGUoc2VlZCUzRDEzMzclMkMlMjBidWZmZXJfc2l6ZSUzRDEwMCklMjAlMjAlMjMlMjByZXNodWZmbGluZyUyMHVzaW5nJTIwYW5vdGhlciUyMHNlZWQlMjBpcyUyMGluc3RhbnRhbmVvdXMlMEElMEFmb3IlMjBleGFtcGxlJTIwaW4lMjBlbnVtZXJhdGUoc2h1ZmZsZWRfaXRlcmFibGVfZGF0YXNldCklM0ElMjAlMjAlMjMlMjBzdGlsbCUyMGFzJTIwZmFzdCUyMGFzJTIwYmVmb3JlJTBBJTIwJTIwJTIwJTIwcGFzcw==",highlighted:`<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(my_iterable_dataset):  <span class="hljs-comment"># fast</span>
    <span class="hljs-keyword">pass</span>

shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=<span class="hljs-number">42</span>, buffer_size=<span class="hljs-number">100</span>)

<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(shuffled_iterable_dataset):  <span class="hljs-comment"># as fast as before</span>
    <span class="hljs-keyword">pass</span>

shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=<span class="hljs-number">1337</span>, buffer_size=<span class="hljs-number">100</span>)  <span class="hljs-comment"># reshuffling using another seed is instantaneous</span>

<span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(shuffled_iterable_dataset):  <span class="hljs-comment"># still as fast as before</span>
    <span class="hljs-keyword">pass</span>`,lang:"python",wrap:!1}}),pe=new y({props:{code:"Zm9yJTIwZXBvY2glMjBpbiUyMHJhbmdlKG5fZXBvY2hzKSUzQSUwQSUyMCUyMCUyMCUyMG15X2l0ZXJhYmxlX2RhdGFzZXQuc2V0X2Vwb2NoKGVwb2NoKSUwQSUyMCUyMCUyMCUyMGZvciUyMGV4YW1wbGUlMjBpbiUyMG15X2l0ZXJhYmxlX2RhdGFzZXQlM0ElMjAlMjAlMjMlMjBmYXN0JTIwJTJCJTIwcmVzaHVmZmxlZCUyMGF0JTIwZWFjaCUyMGVwb2NoJTIwdXNpbmclMjAlNjBlZmZlY3RpdmVfc2VlZCUyMCUzRCUyMHNlZWQlMjAlMkIlMjBlcG9jaCU2MCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhc3M=",highlighted:`<span class="hljs-keyword">for</span> epoch <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_epochs):
    my_iterable_dataset.set_epoch(epoch)
    <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> my_iterable_dataset:  <span class="hljs-comment"># fast + reshuffled at each epoch using \`effective_seed = seed + epoch\`</span>
        <span class="hljs-keyword">pass</span>`,lang:"python",wrap:!1}}),oe=new y({props:{code:"bXlfZGF0YXNldCUyMCUzRCUyMG15X2RhdGFzZXQuc2VsZWN0KHJhbmdlKHN0YXJ0X2luZGV4JTJDJTIwbGVuKGRhdGFzZXQpKSk=",highlighted:'my_dataset = my_dataset.select(<span class="hljs-built_in">range</span>(start_index, <span class="hljs-built_in">len</span>(dataset)))',lang:"python",wrap:!1}}),me=new y({props:{code:"aXRlcmFibGVfZGF0YXNldCUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCU3QiUyMmElMjIlM0ElMjByYW5nZSg2KSU3RCkudG9faXRlcmFibGVfZGF0YXNldChudW1fc2hhcmRzJTNEMyklMEElMjMlMjBzYXZlJTIwaW4lMjB0aGUlMjBtaWRkbGUlMjBvZiUyMHRyYWluaW5nJTBBc3RhdGVfZGljdCUyMCUzRCUyMGl0ZXJhYmxlX2RhdGFzZXQuc3RhdGVfZGljdCgpJTBBJTIzJTIwYW5kJTIwcmVzdW1lJTIwbGF0ZXIlMEFpdGVyYWJsZV9kYXRhc2V0LmxvYWRfc3RhdGVfZGljdChzdGF0ZV9kaWN0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>iterable_dataset = Dataset.from_dict({<span class="hljs-string">&quot;a&quot;</span>: <span class="hljs-built_in">range</span>(<span class="hljs-number">6</span>)}).to_iterable_dataset(num_shards=<span class="hljs-number">3</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># save in the middle of training</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>state_dict = iterable_dataset.state_dict()
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># and resume later</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>iterable_dataset.load_state_dict(state_dict)`,lang:"python",wrap:!1}}),be=new b({props:{title:"Switch from map-style to iterable",local:"switch-from-map-style-to-iterable",headingTag:"h2"}}),we=new y({props:{code:"bXlfaXRlcmFibGVfZGF0YXNldCUyMCUzRCUyMG15X2RhdGFzZXQudG9faXRlcmFibGVfZGF0YXNldCgp",highlighted:"my_iterable_dataset = my_dataset.to_iterable_dataset()",lang:"python",wrap:!1}}),Te=new y({props:{code:"bXlfaXRlcmFibGVfZGF0YXNldCUyMCUzRCUyMG15X2RhdGFzZXQudG9faXRlcmFibGVfZGF0YXNldChudW1fc2hhcmRzJTNEMTAyNCklMEFteV9pdGVyYWJsZV9kYXRhc2V0Lm51bV9zaGFyZHMlMjAlMjAlMjMlMjAxMDI0",highlighted:`my_iterable_dataset = my_dataset.to_iterable_dataset(num_shards=<span class="hljs-number">1024</span>)
my_iterable_dataset.num_shards  <span class="hljs-comment"># 1024</span>`,lang:"python",wrap:!1}}),Je=new gs({props:{source:"https://github.com/huggingface/datasets/blob/main/docs/source/about_mapstyle_vs_iterable.mdx"}}),{c(){M=i("meta"),Ze=l(),ge=i("p"),Xe=l(),d(j.$$.fragment),Ge=l(),d(w.$$.fragment),Ie=l(),_=i("p"),_.innerHTML=Fa,ke=l(),d(T.$$.fragment),$e=l(),J=i("p"),J.innerHTML=xa,Ce=l(),d(g.$$.fragment),Ve=l(),U=i("p"),U.innerHTML=Ya,ve=l(),Z=i("p"),Z.textContent=za,Re=l(),d(X.$$.fragment),Be=l(),G=i("p"),G.innerHTML=Wa,Fe=l(),I=i("p"),I.innerHTML=Na,xe=l(),d(k.$$.fragment),Ye=l(),$=i("p"),$.innerHTML=Qa,ze=l(),d(C.$$.fragment),We=l(),V=i("p"),V.innerHTML=Da,Ne=l(),d(v.$$.fragment),Qe=l(),d(R.$$.fragment),De=l(),B=i("p"),B.innerHTML=Ha,He=l(),d(F.$$.fragment),Ae=l(),x=i("p"),x.textContent=Aa,Ee=l(),Y=i("p"),Y.innerHTML=Ea,Se=l(),d(z.$$.fragment),Le=l(),W=i("p"),W.innerHTML=Sa,qe=l(),d(N.$$.fragment),Pe=l(),Q=i("p"),Q.innerHTML=La,Ke=l(),d(D.$$.fragment),Oe=l(),H=i("p"),H.innerHTML=qa,ea=l(),A=i("p"),A.textContent=Pa,aa=l(),d(E.$$.fragment),sa=l(),d(S.$$.fragment),ta=l(),L=i("p"),L.innerHTML=Ka,la=l(),d(q.$$.fragment),na=l(),P=i("p"),P.innerHTML=Oa,ia=l(),d(K.$$.fragment),ra=l(),O=i("p"),O.innerHTML=es,pa=l(),d(ee.$$.fragment),da=l(),u=i("blockquote"),u.innerHTML=as,oa=l(),d(ae.$$.fragment),ca=l(),se=i("p"),se.innerHTML=ss,ha=l(),te=i("p"),te.innerHTML=ts,ma=l(),d(le.$$.fragment),fa=l(),ne=i("p"),ne.innerHTML=ls,ya=l(),d(ie.$$.fragment),Ma=l(),re=i("p"),re.innerHTML=ns,ua=l(),d(pe.$$.fragment),ba=l(),de=i("p"),de.textContent=is,ja=l(),d(oe.$$.fragment),wa=l(),ce=i("p"),ce.innerHTML=rs,_a=l(),he=i("p"),he.innerHTML=ps,Ta=l(),d(me.$$.fragment),Ja=l(),fe=i("p"),fe.innerHTML=ds,ga=l(),ye=i("p"),ye.textContent=os,Ua=l(),Me=i("p"),Me.textContent=cs,Za=l(),ue=i("p"),ue.innerHTML=hs,Xa=l(),d(be.$$.fragment),Ga=l(),je=i("p"),je.innerHTML=ms,Ia=l(),d(we.$$.fragment),ka=l(),_e=i("p"),_e.innerHTML=fs,$a=l(),d(Te.$$.fragment),Ca=l(),d(Je.$$.fragment),Va=l(),Ue=i("p"),this.h()},l(e){const a=_s("svelte-u9bgzb",document.head);M=r(a,"META",{name:!0,content:!0}),a.forEach(s),Ze=n(e),ge=r(e,"P",{}),ys(ge).forEach(s),Xe=n(e),o(j.$$.fragment,e),Ge=n(e),o(w.$$.fragment,e),Ie=n(e),_=r(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-djjp02"&&(_.innerHTML=Fa),ke=n(e),o(T.$$.fragment,e),$e=n(e),J=r(e,"P",{"data-svelte-h":!0}),p(J)!=="svelte-h2hjbq"&&(J.innerHTML=xa),Ce=n(e),o(g.$$.fragment,e),Ve=n(e),U=r(e,"P",{"data-svelte-h":!0}),p(U)!=="svelte-1osfkbw"&&(U.innerHTML=Ya),ve=n(e),Z=r(e,"P",{"data-svelte-h":!0}),p(Z)!=="svelte-p27tn9"&&(Z.textContent=za),Re=n(e),o(X.$$.fragment,e),Be=n(e),G=r(e,"P",{"data-svelte-h":!0}),p(G)!=="svelte-1m5el5o"&&(G.innerHTML=Wa),Fe=n(e),I=r(e,"P",{"data-svelte-h":!0}),p(I)!=="svelte-1ik5e8b"&&(I.innerHTML=Na),xe=n(e),o(k.$$.fragment,e),Ye=n(e),$=r(e,"P",{"data-svelte-h":!0}),p($)!=="svelte-18x2xbt"&&($.innerHTML=Qa),ze=n(e),o(C.$$.fragment,e),We=n(e),V=r(e,"P",{"data-svelte-h":!0}),p(V)!=="svelte-z2r7k0"&&(V.innerHTML=Da),Ne=n(e),o(v.$$.fragment,e),Qe=n(e),o(R.$$.fragment,e),De=n(e),B=r(e,"P",{"data-svelte-h":!0}),p(B)!=="svelte-16uuuss"&&(B.innerHTML=Ha),He=n(e),o(F.$$.fragment,e),Ae=n(e),x=r(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-gryvti"&&(x.textContent=Aa),Ee=n(e),Y=r(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-gn3sd"&&(Y.innerHTML=Ea),Se=n(e),o(z.$$.fragment,e),Le=n(e),W=r(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-u916f6"&&(W.innerHTML=Sa),qe=n(e),o(N.$$.fragment,e),Pe=n(e),Q=r(e,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-lbtc5o"&&(Q.innerHTML=La),Ke=n(e),o(D.$$.fragment,e),Oe=n(e),H=r(e,"P",{"data-svelte-h":!0}),p(H)!=="svelte-1ruyjc"&&(H.innerHTML=qa),ea=n(e),A=r(e,"P",{"data-svelte-h":!0}),p(A)!=="svelte-vsuw18"&&(A.textContent=Pa),aa=n(e),o(E.$$.fragment,e),sa=n(e),o(S.$$.fragment,e),ta=n(e),L=r(e,"P",{"data-svelte-h":!0}),p(L)!=="svelte-19qqk0o"&&(L.innerHTML=Ka),la=n(e),o(q.$$.fragment,e),na=n(e),P=r(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-1ct1edg"&&(P.innerHTML=Oa),ia=n(e),o(K.$$.fragment,e),ra=n(e),O=r(e,"P",{"data-svelte-h":!0}),p(O)!=="svelte-jkh31x"&&(O.innerHTML=es),pa=n(e),o(ee.$$.fragment,e),da=n(e),u=r(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),p(u)!=="svelte-johy8g"&&(u.innerHTML=as),oa=n(e),o(ae.$$.fragment,e),ca=n(e),se=r(e,"P",{"data-svelte-h":!0}),p(se)!=="svelte-1a2rpb3"&&(se.innerHTML=ss),ha=n(e),te=r(e,"P",{"data-svelte-h":!0}),p(te)!=="svelte-oo5ei7"&&(te.innerHTML=ts),ma=n(e),o(le.$$.fragment,e),fa=n(e),ne=r(e,"P",{"data-svelte-h":!0}),p(ne)!=="svelte-n856o9"&&(ne.innerHTML=ls),ya=n(e),o(ie.$$.fragment,e),Ma=n(e),re=r(e,"P",{"data-svelte-h":!0}),p(re)!=="svelte-1yhobsx"&&(re.innerHTML=ns),ua=n(e),o(pe.$$.fragment,e),ba=n(e),de=r(e,"P",{"data-svelte-h":!0}),p(de)!=="svelte-1vtsyk"&&(de.textContent=is),ja=n(e),o(oe.$$.fragment,e),wa=n(e),ce=r(e,"P",{"data-svelte-h":!0}),p(ce)!=="svelte-1qarbjh"&&(ce.innerHTML=rs),_a=n(e),he=r(e,"P",{"data-svelte-h":!0}),p(he)!=="svelte-1r9oovw"&&(he.innerHTML=ps),Ta=n(e),o(me.$$.fragment,e),Ja=n(e),fe=r(e,"P",{"data-svelte-h":!0}),p(fe)!=="svelte-ul64r3"&&(fe.innerHTML=ds),ga=n(e),ye=r(e,"P",{"data-svelte-h":!0}),p(ye)!=="svelte-fghrva"&&(ye.textContent=os),Ua=n(e),Me=r(e,"P",{"data-svelte-h":!0}),p(Me)!=="svelte-1i92o70"&&(Me.textContent=cs),Za=n(e),ue=r(e,"P",{"data-svelte-h":!0}),p(ue)!=="svelte-1r4w1a2"&&(ue.innerHTML=hs),Xa=n(e),o(be.$$.fragment,e),Ga=n(e),je=r(e,"P",{"data-svelte-h":!0}),p(je)!=="svelte-qjt7f"&&(je.innerHTML=ms),Ia=n(e),o(we.$$.fragment,e),ka=n(e),_e=r(e,"P",{"data-svelte-h":!0}),p(_e)!=="svelte-1rcijfm"&&(_e.innerHTML=fs),$a=n(e),o(Te.$$.fragment,e),Ca=n(e),o(Je.$$.fragment,e),Va=n(e),Ue=r(e,"P",{}),ys(Ue).forEach(s),this.h()},h(){Ra(M,"name","hf:doc:metadata"),Ra(M,"content",Zs),Ra(u,"class","tip")},m(e,a){Ts(document.head,M),t(e,Ze,a),t(e,ge,a),t(e,Xe,a),c(j,e,a),t(e,Ge,a),c(w,e,a),t(e,Ie,a),t(e,_,a),t(e,ke,a),c(T,e,a),t(e,$e,a),t(e,J,a),t(e,Ce,a),c(g,e,a),t(e,Ve,a),t(e,U,a),t(e,ve,a),t(e,Z,a),t(e,Re,a),c(X,e,a),t(e,Be,a),t(e,G,a),t(e,Fe,a),t(e,I,a),t(e,xe,a),c(k,e,a),t(e,Ye,a),t(e,$,a),t(e,ze,a),c(C,e,a),t(e,We,a),t(e,V,a),t(e,Ne,a),c(v,e,a),t(e,Qe,a),c(R,e,a),t(e,De,a),t(e,B,a),t(e,He,a),c(F,e,a),t(e,Ae,a),t(e,x,a),t(e,Ee,a),t(e,Y,a),t(e,Se,a),c(z,e,a),t(e,Le,a),t(e,W,a),t(e,qe,a),c(N,e,a),t(e,Pe,a),t(e,Q,a),t(e,Ke,a),c(D,e,a),t(e,Oe,a),t(e,H,a),t(e,ea,a),t(e,A,a),t(e,aa,a),c(E,e,a),t(e,sa,a),c(S,e,a),t(e,ta,a),t(e,L,a),t(e,la,a),c(q,e,a),t(e,na,a),t(e,P,a),t(e,ia,a),c(K,e,a),t(e,ra,a),t(e,O,a),t(e,pa,a),c(ee,e,a),t(e,da,a),t(e,u,a),t(e,oa,a),c(ae,e,a),t(e,ca,a),t(e,se,a),t(e,ha,a),t(e,te,a),t(e,ma,a),c(le,e,a),t(e,fa,a),t(e,ne,a),t(e,ya,a),c(ie,e,a),t(e,Ma,a),t(e,re,a),t(e,ua,a),c(pe,e,a),t(e,ba,a),t(e,de,a),t(e,ja,a),c(oe,e,a),t(e,wa,a),t(e,ce,a),t(e,_a,a),t(e,he,a),t(e,Ta,a),c(me,e,a),t(e,Ja,a),t(e,fe,a),t(e,ga,a),t(e,ye,a),t(e,Ua,a),t(e,Me,a),t(e,Za,a),t(e,ue,a),t(e,Xa,a),c(be,e,a),t(e,Ga,a),t(e,je,a),t(e,Ia,a),c(we,e,a),t(e,ka,a),t(e,_e,a),t(e,$a,a),c(Te,e,a),t(e,Ca,a),c(Je,e,a),t(e,Va,a),t(e,Ue,a),va=!0},p:us,i(e){va||(h(j.$$.fragment,e),h(w.$$.fragment,e),h(T.$$.fragment,e),h(g.$$.fragment,e),h(X.$$.fragment,e),h(k.$$.fragment,e),h(C.$$.fragment,e),h(v.$$.fragment,e),h(R.$$.fragment,e),h(F.$$.fragment,e),h(z.$$.fragment,e),h(N.$$.fragment,e),h(D.$$.fragment,e),h(E.$$.fragment,e),h(S.$$.fragment,e),h(q.$$.fragment,e),h(K.$$.fragment,e),h(ee.$$.fragment,e),h(ae.$$.fragment,e),h(le.$$.fragment,e),h(ie.$$.fragment,e),h(pe.$$.fragment,e),h(oe.$$.fragment,e),h(me.$$.fragment,e),h(be.$$.fragment,e),h(we.$$.fragment,e),h(Te.$$.fragment,e),h(Je.$$.fragment,e),va=!0)},o(e){m(j.$$.fragment,e),m(w.$$.fragment,e),m(T.$$.fragment,e),m(g.$$.fragment,e),m(X.$$.fragment,e),m(k.$$.fragment,e),m(C.$$.fragment,e),m(v.$$.fragment,e),m(R.$$.fragment,e),m(F.$$.fragment,e),m(z.$$.fragment,e),m(N.$$.fragment,e),m(D.$$.fragment,e),m(E.$$.fragment,e),m(S.$$.fragment,e),m(q.$$.fragment,e),m(K.$$.fragment,e),m(ee.$$.fragment,e),m(ae.$$.fragment,e),m(le.$$.fragment,e),m(ie.$$.fragment,e),m(pe.$$.fragment,e),m(oe.$$.fragment,e),m(me.$$.fragment,e),m(be.$$.fragment,e),m(we.$$.fragment,e),m(Te.$$.fragment,e),m(Je.$$.fragment,e),va=!1},d(e){e&&(s(Ze),s(ge),s(Xe),s(Ge),s(Ie),s(_),s(ke),s($e),s(J),s(Ce),s(Ve),s(U),s(ve),s(Z),s(Re),s(Be),s(G),s(Fe),s(I),s(xe),s(Ye),s($),s(ze),s(We),s(V),s(Ne),s(Qe),s(De),s(B),s(He),s(Ae),s(x),s(Ee),s(Y),s(Se),s(Le),s(W),s(qe),s(Pe),s(Q),s(Ke),s(Oe),s(H),s(ea),s(A),s(aa),s(sa),s(ta),s(L),s(la),s(na),s(P),s(ia),s(ra),s(O),s(pa),s(da),s(u),s(oa),s(ca),s(se),s(ha),s(te),s(ma),s(fa),s(ne),s(ya),s(Ma),s(re),s(ua),s(ba),s(de),s(ja),s(wa),s(ce),s(_a),s(he),s(Ta),s(Ja),s(fe),s(ga),s(ye),s(Ua),s(Me),s(Za),s(ue),s(Xa),s(Ga),s(je),s(Ia),s(ka),s(_e),s($a),s(Ca),s(Va),s(Ue)),s(M),f(j,e),f(w,e),f(T,e),f(g,e),f(X,e),f(k,e),f(C,e),f(v,e),f(R,e),f(F,e),f(z,e),f(N,e),f(D,e),f(E,e),f(S,e),f(q,e),f(K,e),f(ee,e),f(ae,e),f(le,e),f(ie,e),f(pe,e),f(oe,e),f(me,e),f(be,e),f(we,e),f(Te,e),f(Je,e)}}}const Zs='{"title":"Differences between Dataset and IterableDataset","local":"differences-between-dataset-and-iterabledataset","sections":[{"title":"Downloading and streaming","local":"downloading-and-streaming","sections":[],"depth":2},{"title":"Creating map-style datasets and iterable datasets","local":"creating-map-style-datasets-and-iterable-datasets","sections":[],"depth":2},{"title":"Loading local files entirely and progressively","local":"loading-local-files-entirely-and-progressively","sections":[],"depth":2},{"title":"Eager data processing and lazy data processing","local":"eager-data-processing-and-lazy-data-processing","sections":[],"depth":2},{"title":"Exact and fast approximate shuffling","local":"exact-and-fast-approximate-shuffling","sections":[],"depth":2},{"title":"Speed differences","local":"speed-differences","sections":[],"depth":2},{"title":"Switch from map-style to iterable","local":"switch-from-map-style-to-iterable","sections":[],"depth":2}],"depth":1}';function Xs(Ba){return bs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Cs extends js{constructor(M){super(),ws(this,M,Xs,Us,Ms,{})}}export{Cs as component};