robotics-course/pr_27/en/unit1/3.html

<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;LeRobotDataset&quot;,&quot;local&quot;:&quot;lerobotdataset&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;The Dataset Class Design&quot;,&quot;local&quot;:&quot;the-dataset-class-design&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;References&quot;,&quot;local&quot;:&quot;references&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}">
		<link href="/docs/robotics-course/pr_27/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/entry/start.6a087fe8.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/chunks/scheduler.4048030c.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/chunks/singletons.6333d672.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/chunks/index.dd3f5310.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/chunks/paths.cd6cdab2.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/entry/app.f7ddcd05.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/chunks/preload-helper.225f7c4d.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/chunks/index.30ed9803.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/nodes/0.56abb042.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/chunks/each.e59479a4.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/nodes/6.31d0a833.js">
		<link rel="modulepreload" href="/docs/robotics-course/pr_27/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.4d36332b.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;LeRobotDataset&quot;,&quot;local&quot;:&quot;lerobotdataset&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;The Dataset Class Design&quot;,&quot;local&quot;:&quot;the-dataset-class-design&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;References&quot;,&quot;local&quot;:&quot;references&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"><!-- HEAD_svelte-u9bgzb_END -->      <p></p>  <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div>  <h1 class="relative group"><a id="lerobotdataset" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#lerobotdataset"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>LeRobotDataset</span></h1> <p data-svelte-h="svelte-10effi2">LeRobotDataset is a standardized dataset format designed to address the specific needs of robot learning research. In the next few minutes, you’ll see what problems it solves, how it is organized, and where to look first when loading data.</p> <p data-svelte-h="svelte-1q7yjfz">The format provides unified, convenient access to robotics data across modalities, including sensorimotor readings, multiple camera feeds, and teleoperation status. LeRobotDataset also stores general information about the data being collected, including textual task descriptions, the type of robot used, and measurement specifics such as frames per second for both image and robot state streams, together with the types of cameras used, their resolution, and frame-rate.</p> <blockquote class="tip" data-svelte-h="svelte-onsl0e"><p><strong>Why a specialized format?</strong> Traditional ML datasets (like ImageNet) are simple: one image, one label. Robotics data is much more complex:</p> <ul><li><strong>Multi-modal</strong>: Images + sensor readings + actions, all synchronized</li> <li><strong>Temporal</strong>: Both observations and actions are recorded over time, and very much in a sequential manner</li> <li><strong>Episodic</strong>: Data is organized in trajectories/episodes</li> <li><strong>High-dimensional</strong>: Multiple camera views (i.e., multiple images), joint states, forces, etc.</li></ul> <p>LeRobotDataset handles all this complexity seamlessly!</p></blockquote> <p data-svelte-h="svelte-18nm41q">LeRobotDataset provides a unified interface for handling multi‑modal, time‑series data and integrates seamlessly with the PyTorch and Hugging Face ecosystems.</p> <p data-svelte-h="svelte-sknblr">It is extensible and customizable, and already supports openly available data across a variety of embodiments in LeRobot, ranging from manipulator platforms like the SO‑100 and ALOHA‑2 to humanoid arms and hands, simulation‑based datasets, and even autonomous driving.</p> <p data-svelte-h="svelte-12vd5k6">The format is built to be efficient for training and flexible enough to accommodate diverse data types, while promoting reproducibility and ease of use.</p> <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/robotics-course/item-from-dataset.png" alt="Item from dataset">  <h2 class="relative group"><a id="the-dataset-class-design" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#the-dataset-class-design"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>The Dataset Class Design</span></h2> <p data-svelte-h="svelte-14wcggc">You can read more about the design choices behind the design of our dataset class <a href="https://huggingface.co/blog/lerobot-datasets-v3" rel="nofollow">here</a>.
A core design choice behind LeRobotDataset is separating the underlying data storage from the user-facing API. This allows for efficient storage while presenting the data in an intuitive, ready-to-use format.</p> <p data-svelte-h="svelte-g8q04x">Think of it as two layers: a compact on‑disk layout for speed and scale, and a clean Python interface that yields ready‑to‑train tensors.</p> <p data-svelte-h="svelte-njvu1e">Datasets are always organized into three main components:</p> <ul data-svelte-h="svelte-18m61u"><li><strong>Tabular Data</strong>: Low-dimensional, high-frequency data such as joint states, and actions are stored in efficient memory-mapped files, and typically offloaded to the more mature <code>datasets</code> library by Hugging Face, providing fast with limited memory consumption.</li> <li><strong>Visual Data</strong>: To handle large volumes of camera data, frames are concatenated and encoded into MP4 files. Frames from the same episode are always grouped together into the same video, and multiple videos are grouped together by camera. To reduce stress on the file system, groups of videos for the same camera view are also broke into multiple sub-directories.</li> <li><strong>Metadata</strong>: A collection of JSON files which describes the dataset’s structure in terms of its metadata, serving as the relational counterpart to both the tabular and visual dimensions of data. Metadata include the different feature schema, frame rates, normalization statistics, and episode boundaries.</li></ul> <p data-svelte-h="svelte-1ticm9z">As you browse a dataset on disk, keep these three buckets in mind—they explain almost everything you’ll see.</p> <p data-svelte-h="svelte-ksntuc">For scalability, and to support datasets with potentially millions of trajectories (resulting in hundreds of millions or billions of individual camera frames), we merge data from different episodes into the same high-level structure.</p> <p data-svelte-h="svelte-1459i5j">Concretely, a single data file (stored with a parquet file) or recording (stored in MP4 format) often contains multiple episodes. This limits the number of files and speeds up I/O. The trade‑off is that metadata becomes the “map” that tells you where each episode begins and ends. In turn, metadata have a much more “relational” function, similar to how way in a relational database, shared keys allow to retrieve information from multiple tables.</p> <p data-svelte-h="svelte-ll8n4u">An example structure for a given LeRobotDataset would appear as follows:</p> <ul data-svelte-h="svelte-1avvrp5"><li><code>meta/info.json</code>: This metadata is a central metadata file. It contains the complete dataset schema, defining all features (e.g., <code>observation.state</code>, <code>action</code>), their shapes, and data types. It also stores crucial information like the dataset’s frames-per-second (<code>fps</code>), LeRobot’s version at the time of capture, and the path templates used to locate data and video files.</li> <li><code>meta/stats.json</code>: This file stores aggregated statistics (mean, std, min, max) for each feature across the entire dataset, used for data normalization for most policy models and accessible externally via <code>dataset.meta.stats</code>.</li> <li><code>meta/tasks.jsonl</code>: This file contains the mapping from natural language task descriptions to integer task indices, which are useful for task-conditioned policy training.</li> <li><code>meta/episodes/*</code>: This directory contains metadata about each individual episode, such as its length, the corresponding task, and pointers to where its data is stored in the dataset’s files. For scalability, this information is stored in files rather than a single large JSON file.</li> <li><code>data/*</code>: Contains the core frame-by-frame tabular data, using parquet files to allow for fast, memory-mapped access. To improve performance and handle large datasets, data from multiple episodes are concatenated into larger files. These files are organized into chunked subdirectories to keep the size of directories manageable. A single file typically contains data for more than one single episode.</li> <li><code>videos/*</code>: Contains the MP4 video files for all visual observation streams. Similar to the <code>data/</code> directory, the video footage from multiple episodes is concatenated into single MP4 files. This strategy significantly reduces the number of files in the dataset, which is more efficient for modern filesystems.</li></ul> <p data-svelte-h="svelte-1v1unvq">Reading guide: start with <code>meta/info.json</code> to understand the schema and fps; then inspect <code>meta/stats.json</code> for normalization; finally, peek at one file in <code>data/</code> and <code>videos/</code> to connect the dots.</p> <blockquote class="tip" data-svelte-h="svelte-87btsm"><p><strong>Storage Efficiency:</strong> By concatenating episodes into larger files, LeRobotDataset avoids the “small files problem” that can slow down filesystems. A dataset with 1M episodes might have only hundreds of actual files on disk!</p> <p><strong>Pro Tip:</strong> The metadata files act like a database index, allowing fast access to specific episodes without loading entire video files.</p></blockquote>  <h2 class="relative group"><a id="references" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#references"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>References</span></h2> <p data-svelte-h="svelte-12jfyok">For a full list of references, check out the <a href="https://huggingface.co/spaces/lerobot/robot-learning-tutorial" rel="nofollow">tutorial</a>.</p> <ul data-svelte-h="svelte-17oiucg"><li><p><strong>Implicit Behavioral Cloning</strong> (2022)<br>
Pete Florence et al.<br>
This paper introduces energy-based models for behavioral cloning, demonstrating how implicit models can handle multi-modal action distributions more effectively than explicit models—a key consideration when designing dataset formats for robot learning.<br> <a href="https://proceedings.mlr.press/v164/florence22a.html" rel="nofollow">Paper (CoRL 2022)</a></p></li> <li><p><strong>A Dataset for Interactive Vision-Language Navigation with Unknown Command Feasibility</strong> (2022)<br>
Various Authors<br>
An example of how specialized dataset formats enable new capabilities in robot learning, particularly for handling multi-modal sensory data and episodic structure.<br> <a href="https://huggingface.co/papers/2202.02312" rel="nofollow">arXiv:2202.02312</a></p></li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/robotics-course/blob/main/units/en/unit1/3.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a>  <p></p> 
			
			<script>
				{
					__sveltekit_157nhjv = {
						assets: "/docs/robotics-course/pr_27/en",
						base: "/docs/robotics-course/pr_27/en",
						env: {}
					};

					const element = document.currentScript.parentElement;

					const data = [null,null];

					Promise.all([
						import("/docs/robotics-course/pr_27/en/_app/immutable/entry/start.6a087fe8.js"),
						import("/docs/robotics-course/pr_27/en/_app/immutable/entry/app.f7ddcd05.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 6],
							data,
							form: null,
							error: null
						});
					});
				}
			</script>