Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Adding a New Benchmark","local":"adding-a-new-benchmark","sections":[{"title":"Existing benchmarks at a glance","local":"existing-benchmarks-at-a-glance","sections":[],"depth":2},{"title":"How it all fits together","local":"how-it-all-fits-together","sections":[{"title":"Data flow","local":"data-flow","sections":[],"depth":3},{"title":"Environment structure","local":"environment-structure","sections":[],"depth":3},{"title":"How evaluation runs","local":"how-evaluation-runs","sections":[],"depth":3}],"depth":2},{"title":"What your environment must provide","local":"what-your-environment-must-provide","sections":[{"title":"Env attributes","local":"env-attributes","sections":[],"depth":3},{"title":"Success reporting","local":"success-reporting","sections":[],"depth":3},{"title":"Observations","local":"observations","sections":[],"depth":3},{"title":"Actions","local":"actions","sections":[],"depth":3},{"title":"Feature declaration","local":"feature-declaration","sections":[],"depth":3}],"depth":2},{"title":"Step by step","local":"step-by-step","sections":[{"title":"Checklist","local":"checklist","sections":[],"depth":3},{"title":"1. The gym.Env wrapper ( src/lerobot/envs/&lt;benchmark&gt;.py )","local":"1-the-gymenv-wrapper--srclerobotenvsltbenchmarkgtpy-","sections":[],"depth":3},{"title":"2. The config ( src/lerobot/envs/configs.py )","local":"2-the-config--srclerobotenvsconfigspy-","sections":[],"depth":3},{"title":"3. Env processor (optional — src/lerobot/processor/env_processor.py )","local":"3-env-processor-optional--srclerobotprocessorenvprocessorpy-","sections":[],"depth":3},{"title":"4. Dependencies ( pyproject.toml )","local":"4-dependencies--pyprojecttoml-","sections":[],"depth":3},{"title":"5. Documentation ( docs/source/&lt;benchmark&gt;.mdx )","local":"5-documentation--docssourceltbenchmarkgtmdx-","sections":[],"depth":3},{"title":"6. Table of contents ( docs/source/_toctree.yml )","local":"6-table-of-contents--docssourcetoctreeyml-","sections":[],"depth":3}],"depth":2},{"title":"Verifying your integration","local":"verifying-your-integration","sections":[],"depth":2},{"title":"Writing a benchmark doc page","local":"writing-a-benchmark-doc-page","sections":[],"depth":2}],"depth":1}"> | |
| <link href="/docs/lerobot/pr_3313/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/entry/start.d3f1c0f3.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/scheduler.eb244325.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/singletons.1f33814c.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/index.3c23fb4b.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/paths.17f05d75.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/entry/app.04bb7687.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/preload-helper.b00aacbc.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/index.3fe63ad3.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/nodes/0.07fbe93e.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/nodes/4.2b05f202.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/Tip.8a9a4ce7.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/CopyLLMTxtMenu.d0c64540.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.6453902c.js"> | |
| <link rel="modulepreload" href="/docs/lerobot/pr_3313/en/_app/immutable/chunks/CodeBlock.48dd2cc2.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Adding a New Benchmark","local":"adding-a-new-benchmark","sections":[{"title":"Existing benchmarks at a glance","local":"existing-benchmarks-at-a-glance","sections":[],"depth":2},{"title":"How it all fits together","local":"how-it-all-fits-together","sections":[{"title":"Data flow","local":"data-flow","sections":[],"depth":3},{"title":"Environment structure","local":"environment-structure","sections":[],"depth":3},{"title":"How evaluation runs","local":"how-evaluation-runs","sections":[],"depth":3}],"depth":2},{"title":"What your environment must provide","local":"what-your-environment-must-provide","sections":[{"title":"Env attributes","local":"env-attributes","sections":[],"depth":3},{"title":"Success reporting","local":"success-reporting","sections":[],"depth":3},{"title":"Observations","local":"observations","sections":[],"depth":3},{"title":"Actions","local":"actions","sections":[],"depth":3},{"title":"Feature declaration","local":"feature-declaration","sections":[],"depth":3}],"depth":2},{"title":"Step by step","local":"step-by-step","sections":[{"title":"Checklist","local":"checklist","sections":[],"depth":3},{"title":"1. The gym.Env wrapper ( src/lerobot/envs/&lt;benchmark&gt;.py )","local":"1-the-gymenv-wrapper--srclerobotenvsltbenchmarkgtpy-","sections":[],"depth":3},{"title":"2. The config ( src/lerobot/envs/configs.py )","local":"2-the-config--srclerobotenvsconfigspy-","sections":[],"depth":3},{"title":"3. Env processor (optional — src/lerobot/processor/env_processor.py )","local":"3-env-processor-optional--srclerobotprocessorenvprocessorpy-","sections":[],"depth":3},{"title":"4. Dependencies ( pyproject.toml )","local":"4-dependencies--pyprojecttoml-","sections":[],"depth":3},{"title":"5. Documentation ( docs/source/&lt;benchmark&gt;.mdx )","local":"5-documentation--docssourceltbenchmarkgtmdx-","sections":[],"depth":3},{"title":"6. Table of contents ( docs/source/_toctree.yml )","local":"6-table-of-contents--docssourcetoctreeyml-","sections":[],"depth":3}],"depth":2},{"title":"Verifying your integration","local":"verifying-your-integration","sections":[],"depth":2},{"title":"Writing a benchmark doc page","local":"writing-a-benchmark-doc-page","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="adding-a-new-benchmark" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#adding-a-new-benchmark"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Adding a New Benchmark</span></h1> <p data-svelte-h="svelte-1o4zmn9">This guide walks you through adding a new simulation benchmark to LeRobot. Follow the steps in order and use the existing benchmarks as templates.</p> <p data-svelte-h="svelte-18ewlzf">A benchmark in LeRobot is a set of <a href="https://gymnasium.farama.org/" rel="nofollow">Gymnasium</a> environments that wrap a third-party simulator (like LIBERO or Meta-World) behind a standard <code>gym.Env</code> interface. The <code>lerobot-eval</code> CLI then runs evaluation uniformly across all benchmarks.</p> <h2 class="relative group"><a id="existing-benchmarks-at-a-glance" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#existing-benchmarks-at-a-glance"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Existing benchmarks at a glance</span></h2> <p data-svelte-h="svelte-94phxg">Before diving in, here is what is already integrated:</p> <table data-svelte-h="svelte-k02edi"><thead><tr><th>Benchmark</th> <th>Env file</th> <th>Config class</th> <th>Tasks</th> <th>Action dim</th> <th>Processor</th></tr></thead> <tbody><tr><td>LIBERO</td> <td><code>envs/libero.py</code></td> <td><code>LiberoEnv</code></td> <td>130 across 5 suites</td> <td>7</td> <td><code>LiberoProcessorStep</code></td></tr> <tr><td>Meta-World</td> <td><code>envs/metaworld.py</code></td> <td><code>MetaworldEnv</code></td> <td>50 (MT50)</td> <td>4</td> <td>None</td></tr> <tr><td>IsaacLab Arena</td> <td>Hub-hosted</td> <td><code>IsaaclabArenaEnv</code></td> <td>Configurable</td> <td>Configurable</td> <td><code>IsaaclabArenaProcessorStep</code></td></tr></tbody></table> <p data-svelte-h="svelte-7gv1gw">Use <code>src/lerobot/envs/libero.py</code> and <code>src/lerobot/envs/metaworld.py</code> as reference implementations.</p> <h2 class="relative group"><a id="how-it-all-fits-together" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-it-all-fits-together"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>How it all fits together</span></h2> <h3 class="relative group"><a id="data-flow" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#data-flow"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Data flow</span></h3> <p data-svelte-h="svelte-l6xion">During evaluation, data moves through four stages:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-number">1</span>. gym.Env ──→ raw observations (numpy dicts) | |
| <span class="hljs-number">2</span>. Preprocessing ──→ standard LeRobot keys + <span class="hljs-keyword">task</span> <span class="hljs-keyword">description</span> | |
| (preprocess_observation in envs/utils.py, env.<span class="hljs-keyword">call</span>(<span class="hljs-string">"task_description"</span>)) | |
| <span class="hljs-number">3</span>. Processors ──→ env-specific then policy-specific transforms | |
| (env_preprocessor, policy_preprocessor) | |
| <span class="hljs-number">4</span>. Policy ──→ select_action() ──→ action tensor | |
| then <span class="hljs-keyword">reverse</span>: policy_postprocessor → env_postprocessor → numpy action → env.<span class="hljs-keyword">step</span>()<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1fcdi4o">Most benchmarks only need to care about stage 1 (producing observations in the right format) and optionally stage 3 (if env-specific transforms are needed).</p> <h3 class="relative group"><a id="environment-structure" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#environment-structure"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Environment structure</span></h3> <p data-svelte-h="svelte-10d5yv9"><code>make_env()</code> returns a nested dict of vectorized environments:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">int</span>, gym.vector.VectorEnv]] | |
| <span class="hljs-comment"># ^suite ^task_id</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1sxmkwe">A single-task env (e.g. PushT) looks like <code>{"pusht": {0: vec_env}}</code>. | |
| A multi-task benchmark (e.g. LIBERO) looks like <code>{"libero_spatial": {0: vec0, 1: vec1, ...}, ...}</code>.</p> <h3 class="relative group"><a id="how-evaluation-runs" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#how-evaluation-runs"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>How evaluation runs</span></h3> <p data-svelte-h="svelte-1ccnbr6">All benchmarks are evaluated the same way by <code>lerobot-eval</code>:</p> <ol data-svelte-h="svelte-1wq293v"><li><code>make_env()</code> builds the nested <code>{suite: {task_id: VectorEnv}}</code> dict.</li> <li><code>eval_policy_all()</code> iterates over every suite and task.</li> <li>For each task, it runs <code>n_episodes</code> rollouts via <code>rollout()</code>.</li> <li>Results are aggregated hierarchically: episode, task, suite, overall.</li> <li>Metrics include <code>pc_success</code> (success rate), <code>avg_sum_reward</code>, and <code>avg_max_reward</code>.</li></ol> <p data-svelte-h="svelte-axgx7g">The critical piece: your env must return <code>info["is_success"]</code> on every <code>step()</code> call. This is how the eval loop knows whether a task was completed.</p> <h2 class="relative group"><a id="what-your-environment-must-provide" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#what-your-environment-must-provide"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>What your environment must provide</span></h2> <p data-svelte-h="svelte-1p87vsp">LeRobot does not enforce a strict observation schema. Instead it relies on a set of conventions that all benchmarks follow.</p> <h3 class="relative group"><a id="env-attributes" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#env-attributes"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Env attributes</span></h3> <p data-svelte-h="svelte-1pb86vh">Your <code>gym.Env</code> must set these attributes:</p> <table data-svelte-h="svelte-a6acn9"><thead><tr><th>Attribute</th> <th>Type</th> <th>Why</th></tr></thead> <tbody><tr><td><code>_max_episode_steps</code></td> <td><code>int</code></td> <td><code>rollout()</code> uses this to cap episode length</td></tr> <tr><td><code>task_description</code></td> <td><code>str</code></td> <td>Passed to VLA policies as a language instruction</td></tr> <tr><td><code>task</code></td> <td><code>str</code></td> <td>Fallback identifier if <code>task_description</code> is not set</td></tr></tbody></table> <h3 class="relative group"><a id="success-reporting" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#success-reporting"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Success reporting</span></h3> <p data-svelte-h="svelte-hl018o">Your <code>step()</code> and <code>reset()</code> must include <code>"is_success"</code> in the <code>info</code> dict:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->info = {<span class="hljs-string">"is_success"</span>: <span class="hljs-literal">True</span>} <span class="hljs-comment"># or False</span> | |
| <span class="hljs-keyword">return</span> observation, reward, terminated, truncated, info<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="observations" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#observations"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Observations</span></h3> <p data-svelte-h="svelte-ae1bwm">The simplest approach is to map your simulator’s outputs to the standard keys that <code>preprocess_observation()</code> already understands. Do this inside your <code>gym.Env</code> (e.g. in a <code>_format_raw_obs()</code> helper):</p> <table data-svelte-h="svelte-tf1ma0"><thead><tr><th>Your env should output</th> <th>LeRobot maps it to</th> <th>What it is</th></tr></thead> <tbody><tr><td><code>"pixels"</code> (single array)</td> <td><code>observation.image</code></td> <td>Single camera image, HWC uint8</td></tr> <tr><td><code>"pixels"</code> (dict)</td> <td><code>observation.images.<cam></code></td> <td>Multiple cameras, each HWC uint8</td></tr> <tr><td><code>"agent_pos"</code></td> <td><code>observation.state</code></td> <td>Proprioceptive state vector</td></tr> <tr><td><code>"environment_state"</code></td> <td><code>observation.env_state</code></td> <td>Full environment state (e.g. PushT)</td></tr> <tr><td><code>"robot_state"</code></td> <td><code>observation.robot_state</code></td> <td>Nested robot state dict (e.g. LIBERO)</td></tr></tbody></table> <p data-svelte-h="svelte-10mgo0x">If your simulator uses different key names, you have two options:</p> <ol data-svelte-h="svelte-15iitgx"><li><strong>Recommended:</strong> Rename them to the standard keys inside your <code>gym.Env</code> wrapper.</li> <li><strong>Alternative:</strong> Write an env processor to transform observations after <code>preprocess_observation()</code> runs (see step 4 below).</li></ol> <h3 class="relative group"><a id="actions" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#actions"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Actions</span></h3> <p data-svelte-h="svelte-yok5ly">Actions are continuous numpy arrays in a <code>gym.spaces.Box</code>. The dimensionality depends on your benchmark (7 for LIBERO, 4 for Meta-World, etc.). Policies adapt to different action dimensions through their <code>input_features</code> / <code>output_features</code> config.</p> <h3 class="relative group"><a id="feature-declaration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#feature-declaration"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Feature declaration</span></h3> <p data-svelte-h="svelte-dtkbm">Each <code>EnvConfig</code> subclass declares two dicts that tell the policy what to expect:</p> <ul data-svelte-h="svelte-e1kosm"><li><code>features</code> — maps feature names to <code>PolicyFeature(type, shape)</code> (e.g. action dim, image shape).</li> <li><code>features_map</code> — maps raw observation keys to LeRobot convention keys (e.g. <code>"agent_pos"</code> to <code>"observation.state"</code>).</li></ul> <h2 class="relative group"><a id="step-by-step" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#step-by-step"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Step by step</span></h2> <blockquote class="tip">At minimum, you need two files: a **gym.Env wrapper** and an **EnvConfig | |
| subclass** with a `create_envs()` override. Everything else is optional or | |
| documentation. No changes to `factory.py` are needed.</blockquote> <h3 class="relative group"><a id="checklist" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#checklist"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Checklist</span></h3> <table data-svelte-h="svelte-1kftonw"><thead><tr><th>File</th> <th>Required</th> <th>Why</th></tr></thead> <tbody><tr><td><code>src/lerobot/envs/<benchmark>.py</code></td> <td>Yes</td> <td>Wraps the simulator as a standard gym.Env</td></tr> <tr><td><code>src/lerobot/envs/configs.py</code></td> <td>Yes</td> <td>Registers your benchmark and its <code>create_envs()</code> for the CLI</td></tr> <tr><td><code>src/lerobot/processor/env_processor.py</code></td> <td>Optional</td> <td>Custom observation/action transforms</td></tr> <tr><td><code>src/lerobot/envs/utils.py</code></td> <td>Optional</td> <td>Only if you need new raw observation keys</td></tr> <tr><td><code>pyproject.toml</code></td> <td>Yes</td> <td>Declares benchmark-specific dependencies</td></tr> <tr><td><code>docs/source/<benchmark>.mdx</code></td> <td>Yes</td> <td>User-facing documentation page</td></tr> <tr><td><code>docs/source/_toctree.yml</code></td> <td>Yes</td> <td>Adds your page to the docs sidebar</td></tr></tbody></table> <h3 class="relative group"><a id="1-the-gymenv-wrapper--srclerobotenvsltbenchmarkgtpy-" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#1-the-gymenv-wrapper--srclerobotenvsltbenchmarkgtpy-"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>1. The gym.Env wrapper ( src/lerobot/envs/<benchmark>.py )</span></h3> <p data-svelte-h="svelte-322ao1">Create a <code>gym.Env</code> subclass that wraps the third-party simulator:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">class</span> <span class="hljs-title class_">MyBenchmarkEnv</span>(gym.Env): | |
| metadata = {<span class="hljs-string">"render_modes"</span>: [<span class="hljs-string">"rgb_array"</span>], <span class="hljs-string">"render_fps"</span>: <fps>} | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, task_suite, task_id, ...</span>): | |
| <span class="hljs-built_in">super</span>().__init__() | |
| self.task = <task_name_string> | |
| self.task_description = <natural_language_instruction> | |
| self._max_episode_steps = <max_steps> | |
| self.observation_space = spaces.<span class="hljs-type">Dict</span>({...}) | |
| self.action_space = spaces.Box(low=..., high=..., shape=(...,), dtype=np.float32) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">reset</span>(<span class="hljs-params">self, seed=<span class="hljs-literal">None</span>, **kwargs</span>): | |
| ... <span class="hljs-comment"># return (observation, info) — info must contain {"is_success": False}</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">step</span>(<span class="hljs-params">self, action: np.ndarray</span>): | |
| ... <span class="hljs-comment"># return (obs, reward, terminated, truncated, info) — info must contain {"is_success": <bool>}</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">render</span>(<span class="hljs-params">self</span>): | |
| ... <span class="hljs-comment"># return RGB image as numpy array</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">close</span>(<span class="hljs-params">self</span>): | |
| ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-tgohww"><strong>GPU-based simulators (e.g. MuJoCo with EGL rendering):</strong> If your simulator allocates GPU/EGL contexts during <code>__init__</code>, defer that allocation to a <code>_ensure_env()</code> helper called on first <code>reset()</code>/<code>step()</code>. This avoids inheriting stale GPU handles when <code>AsyncVectorEnv</code> spawns worker processes. See <code>LiberoEnv._ensure_env()</code> for the pattern.</p> <p data-svelte-h="svelte-agzf2c">Also provide a factory function that returns the nested dict structure:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">def</span> <span class="hljs-title function_">create_mybenchmark_envs</span>(<span class="hljs-params"> | |
| task: <span class="hljs-built_in">str</span>, | |
| n_envs: <span class="hljs-built_in">int</span>, | |
| gym_kwargs: <span class="hljs-built_in">dict</span> | <span class="hljs-literal">None</span> = <span class="hljs-literal">None</span>, | |
| env_cls: <span class="hljs-built_in">type</span> | <span class="hljs-literal">None</span> = <span class="hljs-literal">None</span>, | |
| </span>) -> <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">int</span>, <span class="hljs-type">Any</span>]]: | |
| <span class="hljs-string">"""Create {suite_name: {task_id: VectorEnv}} for MyBenchmark."""</span> | |
| ...<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-7ilav1">See <code>create_libero_envs()</code> (multi-suite, multi-task) and <code>create_metaworld_envs()</code> (difficulty-grouped tasks) for reference.</p> <h3 class="relative group"><a id="2-the-config--srclerobotenvsconfigspy-" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#2-the-config--srclerobotenvsconfigspy-"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>2. The config ( src/lerobot/envs/configs.py )</span></h3> <p data-svelte-h="svelte-1btlddm">Register a config dataclass so users can select your benchmark with <code>--env.type=<name></code>. Each config owns its environment creation and processor logic via two methods:</p> <ul data-svelte-h="svelte-w5gycz"><li><strong><code>create_envs(n_envs, use_async_envs)</code></strong> — Returns <code>{suite: {task_id: VectorEnv}}</code>. The base class default uses <code>gym.make()</code> for single-task envs. Multi-task benchmarks override this.</li> <li><strong><code>get_env_processors()</code></strong> — Returns <code>(preprocessor, postprocessor)</code>. The base class default returns identity (no-op) pipelines. Override if your benchmark needs observation/action transforms.</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">@EnvConfig.register_subclass(<span class="hljs-params"><span class="hljs-string">"<benchmark_name>"</span></span>)</span> | |
| <span class="hljs-meta">@dataclass</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyBenchmarkEnvConfig</span>(<span class="hljs-title class_ inherited__">EnvConfig</span>): | |
| task: <span class="hljs-built_in">str</span> = <span class="hljs-string">"<default_task>"</span> | |
| fps: <span class="hljs-built_in">int</span> = <fps> | |
| obs_type: <span class="hljs-built_in">str</span> = <span class="hljs-string">"pixels_agent_pos"</span> | |
| features: <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, PolicyFeature] = field(default_factory=<span class="hljs-keyword">lambda</span>: { | |
| ACTION: PolicyFeature(<span class="hljs-built_in">type</span>=FeatureType.ACTION, shape=(<action_dim>,)), | |
| }) | |
| features_map: <span class="hljs-built_in">dict</span>[<span class="hljs-built_in">str</span>, <span class="hljs-built_in">str</span>] = field(default_factory=<span class="hljs-keyword">lambda</span>: { | |
| ACTION: ACTION, | |
| <span class="hljs-string">"agent_pos"</span>: OBS_STATE, | |
| <span class="hljs-string">"pixels"</span>: OBS_IMAGE, | |
| }) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">__post_init__</span>(<span class="hljs-params">self</span>): | |
| ... <span class="hljs-comment"># populate features based on obs_type</span> | |
| <span class="hljs-meta"> @property</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">gym_kwargs</span>(<span class="hljs-params">self</span>) -> <span class="hljs-built_in">dict</span>: | |
| <span class="hljs-keyword">return</span> {<span class="hljs-string">"obs_type"</span>: self.obs_type, <span class="hljs-string">"render_mode"</span>: self.render_mode} | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">create_envs</span>(<span class="hljs-params">self, n_envs: <span class="hljs-built_in">int</span>, use_async_envs: <span class="hljs-built_in">bool</span> = <span class="hljs-literal">True</span></span>): | |
| <span class="hljs-string">"""Override for multi-task benchmarks or custom env creation."""</span> | |
| <span class="hljs-keyword">from</span> lerobot.envs.<benchmark> <span class="hljs-keyword">import</span> create_<benchmark>_envs | |
| <span class="hljs-keyword">return</span> create_<benchmark>_envs(task=self.task, n_envs=n_envs, ...) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_env_processors</span>(<span class="hljs-params">self</span>): | |
| <span class="hljs-string">"""Override if your benchmark needs observation/action transforms."""</span> | |
| <span class="hljs-keyword">from</span> lerobot.processor <span class="hljs-keyword">import</span> PolicyProcessorPipeline | |
| <span class="hljs-keyword">from</span> lerobot.processor.env_processor <span class="hljs-keyword">import</span> MyBenchmarkProcessorStep | |
| <span class="hljs-keyword">return</span> ( | |
| PolicyProcessorPipeline(steps=[MyBenchmarkProcessorStep()]), | |
| PolicyProcessorPipeline(steps=[]), | |
| )<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-13e4au4">Key points:</p> <ul data-svelte-h="svelte-1d43gz7"><li>The <code>register_subclass</code> name is what users pass on the CLI (<code>--env.type=<name></code>).</li> <li><code>features</code> tells the policy what the environment produces.</li> <li><code>features_map</code> maps raw observation keys to LeRobot convention keys.</li> <li><strong>No changes to <code>factory.py</code> needed</strong> — the factory delegates to <code>cfg.create_envs()</code> and <code>cfg.get_env_processors()</code> automatically.</li></ul> <h3 class="relative group"><a id="3-env-processor-optional--srclerobotprocessorenvprocessorpy-" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#3-env-processor-optional--srclerobotprocessorenvprocessorpy-"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>3. Env processor (optional — src/lerobot/processor/env_processor.py )</span></h3> <p data-svelte-h="svelte-7rnl77">Only needed if your benchmark requires observation transforms beyond what <code>preprocess_observation()</code> handles (e.g. image flipping, coordinate conversion). Define the processor step here and return it from <code>get_env_processors()</code> in your config (see step 2):</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">@dataclass</span> | |
| <span class="hljs-meta">@ProcessorStepRegistry.register(<span class="hljs-params">name=<span class="hljs-string">"<benchmark>_processor"</span></span>)</span> | |
| <span class="hljs-keyword">class</span> <span class="hljs-title class_">MyBenchmarkProcessorStep</span>(<span class="hljs-title class_ inherited__">ObservationProcessorStep</span>): | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">_process_observation</span>(<span class="hljs-params">self, observation</span>): | |
| processed = observation.copy() | |
| <span class="hljs-comment"># your transforms here</span> | |
| <span class="hljs-keyword">return</span> processed | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">transform_features</span>(<span class="hljs-params">self, features</span>): | |
| <span class="hljs-keyword">return</span> features <span class="hljs-comment"># update if shapes change</span> | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">observation</span>(<span class="hljs-params">self, observation</span>): | |
| <span class="hljs-keyword">return</span> self._process_observation(observation)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-u0xlke">See <code>LiberoProcessorStep</code> for a full example (image rotation, quaternion-to-axis-angle conversion).</p> <h3 class="relative group"><a id="4-dependencies--pyprojecttoml-" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#4-dependencies--pyprojecttoml-"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>4. Dependencies ( pyproject.toml )</span></h3> <p data-svelte-h="svelte-1g36wqt">Add a new optional-dependency group:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-attr">mybenchmark</span> = [<span class="hljs-string">"my-benchmark-pkg==1.2.3"</span>, <span class="hljs-string">"lerobot[scipy-dep]"</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-n1zy1s">Pinning rules:</p> <ul data-svelte-h="svelte-hodlpy"><li><strong>Always pin</strong> benchmark packages to exact versions for reproducibility (e.g. <code>metaworld==3.0.0</code>).</li> <li><strong>Add platform markers</strong> when needed (e.g. <code>; sys_platform == 'linux'</code>).</li> <li><strong>Pin fragile transitive deps</strong> if known (e.g. <code>gymnasium==1.1.0</code> for Meta-World).</li> <li><strong>Document constraints</strong> in your benchmark doc page.</li></ul> <p data-svelte-h="svelte-1gagn9t">Users install with:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install -e <span class="hljs-string">".[mybenchmark]"</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="5-documentation--docssourceltbenchmarkgtmdx-" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#5-documentation--docssourceltbenchmarkgtmdx-"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>5. Documentation ( docs/source/<benchmark>.mdx )</span></h3> <p data-svelte-h="svelte-5wz1ge">Write a user-facing page following the template in the next section. See <code>docs/source/libero.mdx</code> and <code>docs/source/metaworld.mdx</code> for full examples.</p> <h3 class="relative group"><a id="6-table-of-contents--docssourcetoctreeyml-" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#6-table-of-contents--docssourcetoctreeyml-"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>6. Table of contents ( docs/source/_toctree.yml )</span></h3> <p data-svelte-h="svelte-7u1kfj">Add your benchmark to the “Benchmarks” section:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-bullet">-</span> <span class="hljs-attr">sections:</span> | |
| <span class="hljs-bullet">-</span> <span class="hljs-attr">local:</span> <span class="hljs-string">libero</span> | |
| <span class="hljs-attr">title:</span> <span class="hljs-string">LIBERO</span> | |
| <span class="hljs-bullet">-</span> <span class="hljs-attr">local:</span> <span class="hljs-string">metaworld</span> | |
| <span class="hljs-attr">title:</span> <span class="hljs-string">Meta-World</span> | |
| <span class="hljs-bullet">-</span> <span class="hljs-attr">local:</span> <span class="hljs-string">envhub_isaaclab_arena</span> | |
| <span class="hljs-attr">title:</span> <span class="hljs-string">NVIDIA</span> <span class="hljs-string">IsaacLab</span> <span class="hljs-string">Arena</span> <span class="hljs-string">Environments</span> | |
| <span class="hljs-bullet">-</span> <span class="hljs-attr">local:</span> <span class="hljs-string"><your_benchmark></span> | |
| <span class="hljs-attr">title:</span> <span class="hljs-string"><Your</span> <span class="hljs-string">Benchmark</span> <span class="hljs-string">Name></span> | |
| <span class="hljs-attr">title:</span> <span class="hljs-string">"Benchmarks"</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="verifying-your-integration" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#verifying-your-integration"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Verifying your integration</span></h2> <p data-svelte-h="svelte-habs3">After completing the steps above, confirm that everything works:</p> <ol data-svelte-h="svelte-1sukbvo"><li><strong>Install</strong> — <code>pip install -e ".[mybenchmark]"</code> and verify the dependency group installs cleanly.</li> <li><strong>Smoke test env creation</strong> — call <code>make_env()</code> with your config in Python, check that the returned dict has the expected <code>{suite: {task_id: VectorEnv}}</code> shape, and that <code>reset()</code> returns observations with the right keys.</li> <li><strong>Run a full eval</strong> — <code>lerobot-eval --env.type=<name> --env.task=<task> --eval.n_episodes=1 --policy.path=<any_compatible_policy></code> to exercise the full pipeline end-to-end. (<code>batch_size</code> defaults to auto-tuning based on CPU cores; pass <code>--eval.batch_size=1</code> to force a single environment.)</li> <li><strong>Check success detection</strong> — verify that <code>info["is_success"]</code> flips to <code>True</code> when the task is actually completed. This is what the eval loop uses to compute success rates.</li></ol> <h2 class="relative group"><a id="writing-a-benchmark-doc-page" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#writing-a-benchmark-doc-page"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Writing a benchmark doc page</span></h2> <p data-svelte-h="svelte-eim8ek">Each benchmark <code>.mdx</code> page should include:</p> <ul data-svelte-h="svelte-12k4qr8"><li><strong>Title and description</strong> — 1-2 paragraphs on what the benchmark tests and why it matters.</li> <li><strong>Links</strong> — paper, GitHub repo, project website (if available).</li> <li><strong>Overview image or GIF.</strong></li> <li><strong>Available tasks</strong> — table of task suites with counts and brief descriptions.</li> <li><strong>Installation</strong> — <code>pip install -e ".[<benchmark>]"</code> plus any extra steps (env vars, system packages).</li> <li><strong>Evaluation</strong> — recommended <code>lerobot-eval</code> command with <code>n_episodes</code> for reproducible results. <code>batch_size</code> defaults to auto; only specify it if needed. Include single-task and multi-task examples if applicable.</li> <li><strong>Policy inputs and outputs</strong> — observation keys with shapes, action space description.</li> <li><strong>Recommended evaluation episodes</strong> — how many episodes per task is standard.</li> <li><strong>Training</strong> — example <code>lerobot-train</code> command.</li> <li><strong>Reproducing published results</strong> — link to pretrained model, eval command, results table (if available).</li></ul> <p data-svelte-h="svelte-1qdl3hx">See <code>docs/source/libero.mdx</code> and <code>docs/source/metaworld.mdx</code> for complete examples.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/lerobot/blob/main/docs/source/adding_benchmarks.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p> | |
| <script> | |
| { | |
| __sveltekit_9kza6s = { | |
| assets: "/docs/lerobot/pr_3313/en", | |
| base: "/docs/lerobot/pr_3313/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/lerobot/pr_3313/en/_app/immutable/entry/start.d3f1c0f3.js"), | |
| import("/docs/lerobot/pr_3313/en/_app/immutable/entry/app.04bb7687.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 4], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 72.5 kB
- Xet hash:
- 54384a1809320201b39b4eba676ba90cf809e2afc8063eb2bf014e42038ad208
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.