Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Introduction to Processors","local":"introduction-to-processors","sections":[{"title":"What are Processors?","local":"what-are-processors","sections":[],"depth":2},{"title":"Core Concepts","local":"core-concepts","sections":[{"title":"EnvTransition: The Universal Data Container","local":"envtransition-the-universal-data-container","sections":[],"depth":3},{"title":"ProcessorStep: The Building Block","local":"processorstep-the-building-block","sections":[],"depth":3},{"title":"DataProcessorPipeline: The Generic Orchestrator","local":"dataprocessorpipeline-the-generic-orchestrator","sections":[],"depth":3}],"depth":2},{"title":"RobotProcessorPipeline vs PolicyProcessorPipeline","local":"robotprocessorpipeline-vs-policyprocessorpipeline","sections":[],"depth":2},{"title":"Converter Functions","local":"converter-functions","sections":[],"depth":2},{"title":"Processor Examples","local":"processor-examples","sections":[{"title":"An interaction between a robot and a policy with processors","local":"an-interaction-between-a-robot-and-a-policy-with-processors","sections":[],"depth":3}],"depth":2},{"title":"Feature Contracts: Shape and Type Transformation","local":"feature-contracts-shape-and-type-transformation","sections":[{"title":"Why Feature Contracts Matter","local":"why-feature-contracts-matter","sections":[],"depth":3},{"title":"Feature Specification Functions","local":"feature-specification-functions","sections":[],"depth":3}],"depth":2},{"title":"Common Processor Steps","local":"common-processor-steps","sections":[{"title":"Essential Processors","local":"essential-processors","sections":[],"depth":3},{"title":"Next Steps","local":"next-steps","sections":[],"depth":3}],"depth":2},{"title":"Summary","local":"summary","sections":[],"depth":2}],"depth":1}"> | |
| <link href="/docs/lerobot/pr_3313/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/entry/start.d3f1c0f3.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/scheduler.eb244325.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/singletons.1f33814c.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/index.3c23fb4b.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/paths.17f05d75.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/entry/app.04bb7687.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/preload-helper.b00aacbc.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/index.3fe63ad3.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/nodes/0.07fbe93e.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/nodes/29.12c3c8ac.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/CopyLLMTxtMenu.d0c64540.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.6453902c.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/CodeBlock.48dd2cc2.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Introduction to Processors","local":"introduction-to-processors","sections":[{"title":"What are Processors?","local":"what-are-processors","sections":[],"depth":2},{"title":"Core Concepts","local":"core-concepts","sections":[{"title":"EnvTransition: The Universal Data Container","local":"envtransition-the-universal-data-container","sections":[],"depth":3},{"title":"ProcessorStep: The Building Block","local":"processorstep-the-building-block","sections":[],"depth":3},{"title":"DataProcessorPipeline: The Generic Orchestrator","local":"dataprocessorpipeline-the-generic-orchestrator","sections":[],"depth":3}],"depth":2},{"title":"RobotProcessorPipeline vs PolicyProcessorPipeline","local":"robotprocessorpipeline-vs-policyprocessorpipeline","sections":[],"depth":2},{"title":"Converter Functions","local":"converter-functions","sections":[],"depth":2},{"title":"Processor Examples","local":"processor-examples","sections":[{"title":"An interaction between a robot and a policy with processors","local":"an-interaction-between-a-robot-and-a-policy-with-processors","sections":[],"depth":3}],"depth":2},{"title":"Feature Contracts: Shape and Type Transformation","local":"feature-contracts-shape-and-type-transformation","sections":[{"title":"Why Feature Contracts Matter","local":"why-feature-contracts-matter","sections":[],"depth":3},{"title":"Feature Specification Functions","local":"feature-specification-functions","sections":[],"depth":3}],"depth":2},{"title":"Common Processor Steps","local":"common-processor-steps","sections":[{"title":"Essential Processors","local":"essential-processors","sections":[],"depth":3},{"title":"Next Steps","local":"next-steps","sections":[],"depth":3}],"depth":2},{"title":"Summary","local":"summary","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="introduction-to-processors" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#introduction-to-processors"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Introduction to Processors</span></h1> <p data-svelte-h="svelte-1p4wxkn">In robotics, there’s a fundamental mismatch between the data that robots and humans produce and what machine learning models expect. | |
| Robots output raw sensor data like camera images and joint positions that need normalization, batching, and device placement before models can process them. | |
| Language instructions from humans must be tokenized into numerical representations, and different robots use different coordinate systems that need standardization.</p> <p data-svelte-h="svelte-14636jx">The challenge extends to model outputs as well. | |
| Models might output end-effector positions while robots need joint-space commands, or teleoperators produce relative movements while robots expect absolute commands. | |
| Model predictions are often normalized and need conversion back to real-world scales.</p> <p data-svelte-h="svelte-1oobog8">Cross-domain translation adds another layer of complexity. | |
| Training data from one robot setup needs adaptation for deployment on different hardware, models trained with specific camera configurations must work with new arrangements, and datasets with different naming conventions need harmonization.</p> <p data-svelte-h="svelte-1ur6am2"><strong>That’s where processors come in.</strong> They serve as universal translators that bridge these gaps, ensuring seamless data flow from sensors to models to actuators. | |
| Processors handle all the preprocessing and postprocessing steps needed to convert raw environment data into model-ready inputs and vice versa.</p> <p data-svelte-h="svelte-1ka3nux">This means that your favorite policy can be used like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> lerobot.datasets <span class="hljs-keyword">import</span> LeRobotDataset | |
| <span class="hljs-keyword">from</span> lerobot.policies <span class="hljs-keyword">import</span> make_pre_post_processors | |
| <span class="hljs-keyword">from</span> lerobot.policies.your_policy <span class="hljs-keyword">import</span> YourPolicy | |
| <span class="hljs-keyword">from</span> lerobot.processor <span class="hljs-keyword">import</span> RobotProcessorPipeline, PolicyProcessorPipeline | |
| dataset = LeRobotDataset(<span class="hljs-string">"hf_user/dataset"</span>, episodes=[<span class="hljs-number">0</span>]) | |
| sample = dataset[<span class="hljs-number">10</span>] | |
| model = YourPolicy.from_pretrained( | |
| <span class="hljs-string">"hf_user/model"</span>, | |
| ) | |
| model.<span class="hljs-built_in">eval</span>() | |
| model.to(<span class="hljs-string">"cuda"</span>) | |
| preprocessor, postprocessor = make_pre_post_processors(model.config, pretrained_path=<span class="hljs-string">"hf_user/model"</span>, dataset_stats=dataset.meta.stats) | |
| preprocessed_sample = preprocessor(sample) | |
| action = model.select_action(preprocessed_sample) | |
| postprocessed_action = postprocessor(action)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="what-are-processors" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#what-are-processors"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>What are Processors?</span></h2> <p data-svelte-h="svelte-1begaqk">In robotics, data comes in many forms: images from cameras, joint positions from sensors, text instructions from users, and more. Each type of data requires specific transformations before a model can use it effectively. Models need this data to be:</p> <ul data-svelte-h="svelte-1ladsw4"><li><strong>Normalized</strong>: Scaled to appropriate ranges for neural network processing</li> <li><strong>Batched</strong>: Organized with proper dimensions for batch processing</li> <li><strong>Tokenized</strong>: Text converted to numerical representations</li> <li><strong>Device-placed</strong>: Moved to the right hardware (CPU/GPU)</li> <li><strong>Type-converted</strong>: Cast to appropriate data types</li></ul> <p data-svelte-h="svelte-u7yi05">Processors handle these transformations through composable, reusable steps that can be chained together into pipelines. Think of them as a modular assembly line where each station performs a specific transformation on your data.</p> <h2 class="relative group"><a id="core-concepts" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#core-concepts"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Core Concepts</span></h2> <h3 class="relative group"><a id="envtransition-the-universal-data-container" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#envtransition-the-universal-data-container"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>EnvTransition: The Universal Data Container</span></h3> <p data-svelte-h="svelte-1ppq2fo">The <code>EnvTransition</code> is the fundamental data structure that flows through all processors. | |
| It’s a typed dictionary that represents a complete robot-environment interaction:</p> <ul data-svelte-h="svelte-106dygg"><li><strong>OBSERVATION</strong>: All sensor data (images, states, proprioception)</li> <li><strong>ACTION</strong>: The action to execute or that was executed</li> <li><strong>REWARD</strong>: Reinforcement learning signal</li> <li><strong>DONE/TRUNCATED</strong>: Episode boundary indicators</li> <li><strong>INFO</strong>: Arbitrary metadata</li> <li><strong>COMPLEMENTARY_DATA</strong>: Task descriptions, indices, padding flags, inter-step data</li></ul> <h3 class="relative group"><a id="processorstep-the-building-block" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#processorstep-the-building-block"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>ProcessorStep: The Building Block</span></h3> <p data-svelte-h="svelte-8rfarm">A <code>ProcessorStep</code> is a single transformation unit that processes transitions. It’s an abstract base class with two required methods:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> lerobot.processor <span class="hljs-keyword">import</span> ProcessorStep, EnvTransition | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyProcessorStep</span>(<span class="hljs-title class_ inherited__">ProcessorStep</span>): | |
| <span class="hljs-string">"""Example processor step - inherit and implement abstract methods."""</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, transition: EnvTransition</span>) -> EnvTransition: | |
| <span class="hljs-string">"""Transform the transition - REQUIRED abstract method."""</span> | |
| <span class="hljs-comment"># Your processing logic here</span> | |
| <span class="hljs-keyword">return</span> transition | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">transform_features</span>(<span class="hljs-params">self, features</span>): | |
| <span class="hljs-string">"""Declare how this step transforms feature shapes/types - REQUIRED abstract method."""</span> | |
| <span class="hljs-keyword">return</span> features <span class="hljs-comment"># Most processors return features unchanged</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13brd71"><code>__call__</code> is the core of your processor step. It takes an <code>EnvTransition</code> and returns a modified <code>EnvTransition</code>.</p> <p data-svelte-h="svelte-1mifbwe"><code>transform_features</code> is used to declare how this step transforms feature shapes/types.</p> <h3 class="relative group"><a id="dataprocessorpipeline-the-generic-orchestrator" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#dataprocessorpipeline-the-generic-orchestrator"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>DataProcessorPipeline: The Generic Orchestrator</span></h3> <p data-svelte-h="svelte-8mdwcy">The <code>DataProcessorPipeline[TInput, TOutput]</code> chains multiple <code>ProcessorStep</code> instances together:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> lerobot.processor <span class="hljs-keyword">import</span> RobotProcessorPipeline, PolicyProcessorPipeline | |
| <span class="hljs-comment"># For robot hardware (unbatched data)</span> | |
| robot_processor = RobotProcessorPipeline[RobotAction, RobotAction]( | |
| steps=[step1, step2, step3], | |
| name=<span class="hljs-string">"robot_pipeline"</span> | |
| ) | |
| <span class="hljs-comment"># For model training/inference (batched data)</span> | |
| policy_processor = PolicyProcessorPipeline[<span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-type">Any</span>], <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-type">Any</span>]]( | |
| steps=[step1, step2, step3], | |
| name=<span class="hljs-string">"policy_pipeline"</span> | |
| )<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="robotprocessorpipeline-vs-policyprocessorpipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#robotprocessorpipeline-vs-policyprocessorpipeline"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>RobotProcessorPipeline vs PolicyProcessorPipeline</span></h2> <p data-svelte-h="svelte-wjgvgq">The key distinction is in the data structures they handle:</p> <table data-svelte-h="svelte-1gwm6ye"><thead><tr><th>Aspect</th> <th>RobotProcessorPipeline</th> <th>PolicyProcessorPipeline</th></tr></thead> <tbody><tr><td><strong>Input</strong></td> <td><code>dict[str, Any]</code> - Individual robot values</td> <td><code>dict[str, Any]</code> - Batched tensors</td></tr> <tr><td><strong>Output</strong></td> <td><code>dict[str, Any]</code> - Individual robot commands</td> <td><code>torch.Tensor</code> - Policy predictions</td></tr> <tr><td><strong>Use Case</strong></td> <td>Real-time robot control</td> <td>Model training/inference</td></tr> <tr><td><strong>Data Format</strong></td> <td>Unbatched, heterogeneous</td> <td>Batched, homogeneous</td></tr> <tr><td><strong>Examples</strong></td> <td><code>{"joint_1": 0.5}</code></td> <td><code>{"observation.state": tensor([[0.5]])}</code></td></tr></tbody></table> <p data-svelte-h="svelte-1tvtb9y"><strong>Use <code>RobotProcessorPipeline</code></strong> for robot hardware interfaces:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Robot data structures: dict[str, Any] for observations and actions</span> | |
| robot_obs: <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-type">Any</span>] = { | |
| <span class="hljs-string">"joint_1"</span>: <span class="hljs-number">0.5</span>, <span class="hljs-comment"># Individual joint values</span> | |
| <span class="hljs-string">"joint_2"</span>: -<span class="hljs-number">0.3</span>, | |
| <span class="hljs-string">"camera_0"</span>: image_array <span class="hljs-comment"># Raw camera data</span> | |
| } | |
| robot_action: <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-type">Any</span>] = { | |
| <span class="hljs-string">"joint_1"</span>: <span class="hljs-number">0.2</span>, <span class="hljs-comment"># Target joint positions</span> | |
| <span class="hljs-string">"joint_2"</span>: <span class="hljs-number">0.1</span>, | |
| <span class="hljs-string">"gripper"</span>: <span class="hljs-number">0.8</span> | |
| }<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-u5i30h"><strong>Use <code>PolicyProcessorPipeline</code></strong> for model training and batch processing:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Policy data structures: batch dicts and tensors</span> | |
| policy_batch: <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-type">Any</span>] = { | |
| <span class="hljs-string">"observation.state"</span>: torch.tensor([[<span class="hljs-number">0.5</span>, -<span class="hljs-number">0.3</span>]]), <span class="hljs-comment"># Batched states</span> | |
| <span class="hljs-string">"observation.images.camera0"</span>: torch.tensor(...), <span class="hljs-comment"># Batched images</span> | |
| <span class="hljs-string">"action"</span>: torch.tensor([[<span class="hljs-number">0.2</span>, <span class="hljs-number">0.1</span>, <span class="hljs-number">0.8</span>]]) <span class="hljs-comment"># Batched actions</span> | |
| } | |
| policy_action: torch.Tensor = torch.tensor([[<span class="hljs-number">0.2</span>, <span class="hljs-number">0.1</span>, <span class="hljs-number">0.8</span>]]) <span class="hljs-comment"># Model output tensor</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="converter-functions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#converter-functions"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Converter Functions</span></h2> <p data-svelte-h="svelte-1svxhl3">LeRobot provides converter functions to bridge different data formats in <code>lerobot.processor.converters</code>. These functions handle the crucial translations between robot hardware data structures, policy model formats, and the internal <code>EnvTransition</code> representation that flows through processor pipelines.</p> <table data-svelte-h="svelte-5rwep8"><thead><tr><th>Category</th> <th>Function</th> <th>Description</th></tr></thead> <tbody><tr><td><strong>Robot Hardware Converters</strong></td> <td><code>robot_action_to_transition</code></td> <td>Robot dict → EnvTransition</td></tr> <tr><td></td> <td><code>observation_to_transition</code></td> <td>Robot obs → EnvTransition</td></tr> <tr><td></td> <td><code>transition_to_robot_action</code></td> <td>EnvTransition → Robot dict</td></tr> <tr><td><strong>Policy/Training Converters</strong></td> <td><code>batch_to_transition</code></td> <td>Batch dict → EnvTransition</td></tr> <tr><td></td> <td><code>transition_to_batch</code></td> <td>EnvTransition → Batch dict</td></tr> <tr><td></td> <td><code>policy_action_to_transition</code></td> <td>Policy tensor → EnvTransition</td></tr> <tr><td></td> <td><code>transition_to_policy_action</code></td> <td>EnvTransition → Policy tensor</td></tr> <tr><td><strong>Utilities</strong></td> <td><code>create_transition</code></td> <td>Build transitions with defaults</td></tr> <tr><td></td> <td><code>identity_transition</code></td> <td>Pass-through converter</td></tr></tbody></table> <p data-svelte-h="svelte-fqoi61">The key insight is that <strong>robot hardware converters</strong> work with individual values and dictionaries, while <strong>policy/training converters</strong> work with batched tensors and model outputs. The converter functions automatically handle the structural differences, so your processor steps can focus on the core transformations without worrying about data format compatibility.</p> <h2 class="relative group"><a id="processor-examples" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#processor-examples"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Processor Examples</span></h2> <p data-svelte-h="svelte-zzgjs2">The following examples demonstrate real-world processor configurations for policy training and inference.</p> <p data-svelte-h="svelte-1bcvc84">Here is an example processor for policy training and inference:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Training data preprocessing (optimized order for GPU performance)</span> | |
| training_preprocessor = PolicyProcessorPipeline[<span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-type">Any</span>], <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-type">Any</span>]]( | |
| steps=[ | |
| RenameObservationsProcessorStep(rename_map={}), <span class="hljs-comment"># Standardize keys</span> | |
| AddBatchDimensionProcessorStep(), <span class="hljs-comment"># Add batch dims</span> | |
| TokenizerProcessorStep(tokenizer_name=<span class="hljs-string">"..."</span>, ...), <span class="hljs-comment"># Tokenize language</span> | |
| DeviceProcessorStep(device=<span class="hljs-string">"cuda"</span>), <span class="hljs-comment"># Move to GPU first</span> | |
| NormalizerProcessorStep(features=..., stats=...), <span class="hljs-comment"># Normalize on GPU</span> | |
| ] | |
| ) | |
| <span class="hljs-comment"># Model output postprocessing</span> | |
| training_postprocessor = PolicyProcessorPipeline[torch.Tensor, torch.Tensor]( | |
| steps=[ | |
| DeviceProcessorStep(device=<span class="hljs-string">"cpu"</span>), <span class="hljs-comment"># Move to CPU</span> | |
| UnnormalizerProcessorStep(features=..., stats=...), <span class="hljs-comment"># Denormalize</span> | |
| ] | |
| to_transition=policy_action_to_transition, | |
| to_output=transition_to_policy_action, | |
| )<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="an-interaction-between-a-robot-and-a-policy-with-processors" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#an-interaction-between-a-robot-and-a-policy-with-processors"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>An interaction between a robot and a policy with processors</span></h3> <p data-svelte-h="svelte-1is1ww4">The most common real-world scenario combines both pipeline types robot hardware generates observations that need policy processing, and policy outputs need robot-compatible postprocessing:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Real deployment: Robot sensors → Model → Robot commands</span> | |
| <span class="hljs-keyword">with</span> torch.no_grad(): | |
| <span class="hljs-keyword">while</span> <span class="hljs-keyword">not</span> done: | |
| raw_obs = robot.get_observation() <span class="hljs-comment"># dict[str, Any]</span> | |
| <span class="hljs-comment"># Add your robot observation to policy observation processor</span> | |
| policy_input = policy_preprocessor(raw_obs) <span class="hljs-comment"># Batched dict</span> | |
| policy_output = policy.select_action(policy_input) <span class="hljs-comment"># Policy tensor</span> | |
| policy_action = policy_postprocessor(policy_output) | |
| <span class="hljs-comment"># Add your robot action to policy action processor</span> | |
| robot.send_action(policy_action)<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="feature-contracts-shape-and-type-transformation" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#feature-contracts-shape-and-type-transformation"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Feature Contracts: Shape and Type Transformation</span></h2> <p data-svelte-h="svelte-1xj3b0m">Processors don’t just transform data - they can also <strong>change the data structure itself</strong>. The <code>transform_features()</code> method declares these changes, which is crucial for dataset recording and policy creation.</p> <h3 class="relative group"><a id="why-feature-contracts-matter" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#why-feature-contracts-matter"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Why Feature Contracts Matter</span></h3> <p data-svelte-h="svelte-np5mv2">When building datasets or policies, LeRobot needs to know:</p> <ul data-svelte-h="svelte-1yhmz2o"><li><strong>What data fields will exist</strong> after processing</li> <li><strong>What shapes and types</strong> each field will have</li> <li><strong>How to configure models</strong> for the expected data structure</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment"># Example: A processor that adds velocity to observations</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">VelocityProcessor</span>(<span class="hljs-title class_ inherited__">ObservationProcessorStep</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">observation</span>(<span class="hljs-params">self, obs</span>): | |
| new_obs = obs.copy() | |
| <span class="hljs-keyword">if</span> <span class="hljs-string">"observation.state"</span> <span class="hljs-keyword">in</span> obs: | |
| <span class="hljs-comment"># concatenate computed velocity field to the state</span> | |
| new_obs[<span class="hljs-string">"observation.state"</span>] = self._compute_velocity(obs[<span class="hljs-string">"observation.state"</span>]) | |
| <span class="hljs-keyword">return</span> new_obs | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">transform_features</span>(<span class="hljs-params">self, features</span>): | |
| <span class="hljs-string">"""Declare the new velocity field we're adding."""</span> | |
| state_feature = features[PipelineFeatureType.OBSERVATION].get(<span class="hljs-string">"observation.state"</span>) | |
| <span class="hljs-keyword">if</span> state_feature: | |
| double_shape = (state_feature.shape[<span class="hljs-number">0</span>] * <span class="hljs-number">2</span>,) <span class="hljs-keyword">if</span> state_feature.shape <span class="hljs-keyword">else</span> (<span class="hljs-number">2</span>,) | |
| features[PipelineFeatureType.OBSERVATION][<span class="hljs-string">"observation.state"</span>] = PolicyFeature( | |
| <span class="hljs-built_in">type</span>=FeatureType.STATE, shape=double_shape | |
| ) | |
| <span class="hljs-keyword">return</span> features<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="feature-specification-functions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#feature-specification-functions"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Feature Specification Functions</span></h3> <p data-svelte-h="svelte-6gv7sj"><code>create_initial_features()</code> and <code>aggregate_pipeline_dataset_features()</code> solve a critical dataset creation problem: determining the exact final data structure before any data is processed. | |
| Since processor pipelines can add new features (like velocity fields), change tensor shapes (like cropping images), or rename keys, datasets need to know the complete output specification upfront to allocate proper storage and define schemas. | |
| These functions work together by starting with robot hardware specifications (<code>create_initial_features()</code>) then simulating the entire pipeline transformation (<code>aggregate_pipeline_dataset_features()</code>) to compute the final feature dictionary that gets passed to <code>LeRobotDataset.create()</code>, ensuring perfect alignment between what processors output and what datasets expect to store.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> lerobot.datasets <span class="hljs-keyword">import</span> aggregate_pipeline_dataset_features | |
| <span class="hljs-comment"># Start with robot's raw features</span> | |
| initial_features = create_initial_features( | |
| observation=robot.observation_features, <span class="hljs-comment"># {"joint_1.pos": float, "camera_0": (480,640,3)}</span> | |
| action=robot.action_features <span class="hljs-comment"># {"joint_1.pos": float, "gripper.pos": float}</span> | |
| ) | |
| <span class="hljs-comment"># Apply processor pipeline to compute final features</span> | |
| final_features = aggregate_pipeline_dataset_features( | |
| pipeline=my_processor_pipeline, | |
| initial_features=initial_features, | |
| use_videos=<span class="hljs-literal">True</span> | |
| ) | |
| <span class="hljs-comment"># Use for dataset creation</span> | |
| dataset = LeRobotDataset.create( | |
| repo_id=<span class="hljs-string">"my_dataset"</span>, | |
| features=final_features, <span class="hljs-comment"># Knows exactly what data to expect</span> | |
| ... | |
| )<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="common-processor-steps" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#common-processor-steps"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Common Processor Steps</span></h2> <p data-svelte-h="svelte-kvop2o">LeRobot provides many registered processor steps. Here are the most commonly used core processors:</p> <h3 class="relative group"><a id="essential-processors" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#essential-processors"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Essential Processors</span></h3> <ul data-svelte-h="svelte-cbajy3"><li><strong><code>normalizer_processor</code></strong>: Normalize observations/actions using dataset statistics (mean/std or min/max)</li> <li><strong><code>device_processor</code></strong>: Move tensors to CPU/GPU with optional dtype conversion</li> <li><strong><code>to_batch_processor</code></strong>: Add batch dimensions to transitions for model compatibility</li> <li><strong><code>rename_observations_processor</code></strong>: Rename observation keys using mapping dictionaries</li> <li><strong><code>tokenizer_processor</code></strong>: Tokenize natural language task descriptions into tokens and attention masks</li></ul> <h3 class="relative group"><a id="next-steps" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#next-steps"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Next Steps</span></h3> <ul data-svelte-h="svelte-z1kkrk"><li><strong><a href="./implement_your_own_processor">Implement Your Own Processor</a></strong> - Create custom processor steps</li> <li><strong><a href="./debug_processor_pipeline">Debug Your Pipeline</a></strong> - Troubleshoot and optimize pipelines</li> <li><strong><a href="./processors_robots_teleop">Processors for Robots and Teleoperators</a></strong> - Real-world integration patterns</li></ul> <h2 class="relative group"><a id="summary" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#summary"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Summary</span></h2> <p data-svelte-h="svelte-kqr9xc">Processors solve the data translation problem in robotics by providing:</p> <ul data-svelte-h="svelte-tctthc"><li><strong>Modular transformations</strong>: Composable, reusable processing steps</li> <li><strong>Type safety</strong>: Generic pipelines with compile-time checking</li> <li><strong>Performance optimization</strong>: GPU-accelerated operations</li> <li><strong>Robot/Policy distinction</strong>: Separate pipelines for different data structures</li> <li><strong>Comprehensive ecosystem</strong>: 30+ registered processors for common tasks</li></ul> <p data-svelte-h="svelte-1eyaijl">The key insight: <code>RobotProcessorPipeline</code> handles unbatched robot hardware data, while <code>PolicyProcessorPipeline</code> handles batched model data. Choose the right tool for your data structure!</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/lerobot/blob/main/docs/source/introduction_processors.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_9kza6s = { | |
| assets: "/docs/lerobot/pr_3313/en", | |
| base: "/docs/lerobot/pr_3313/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/lerobot/pr_3313/en/_app/immutable/entry/start.d3f1c0f3.js"), | |
| import("/docs/lerobot/pr_3313/en/_app/immutable/entry/app.04bb7687.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 29], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 62.7 kB
- Xet hash:
- 51ac65976f11fe3e9a54cc532f8ac61982d722919ae08244cdee4d7030a63091
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.