lerobot/pr_1713/en/_app/immutable/nodes/37.fmOWOd-R.js

import"../chunks/DsnmJJEf.js";import{i as v,h as f,C as I,H as a,a as s,E as A,s as _}from"../chunks/C8NOCHVR.js";import{p as C,o as Z,s as e,f as Q,a as w,b as S,c as T,n as z}from"../chunks/CTvsL7Wh.js";const B='{"title":"Language columns and recipes","local":"language-columns-and-recipes","sections":[{"title":"Layer 1 — language columns in the dataset","local":"layer-1--language-columns-in-the-dataset","sections":[{"title":"Architecture","local":"architecture","sections":[],"depth":3}],"depth":2},{"title":"Layer 2 — recipe anatomy","local":"layer-2--recipe-anatomy","sections":[{"title":"Temporal semantics","local":"temporal-semantics","sections":[],"depth":3},{"title":"View-dependent resolution","local":"view-dependent-resolution","sections":[],"depth":3}],"depth":2},{"title":"Layer 3 — training format","local":"layer-3--training-format","sections":[],"depth":2},{"title":"Graceful absence","local":"graceful-absence","sections":[],"depth":2}],"depth":1}';var W=T('<meta name="hf:doc:metadata"/>'),E=T(`<p></p> <!> <!> <p>Most LeRobot datasets ship with a single <code>task</code> string per episode — fine for
short, single-instruction skills, but not enough for the longer-horizon,
multi-modal robot policies the field is moving toward (high-level planning,
memory, interjections, VQA, tool use). To support those policies without
forking the dataset format, LeRobot extends <code>LeRobotDataset</code> with two optional
language columns and a small recipe layer that turns those rows into
chat-style training samples on the fly.</p> <p>The design splits cleanly into three layers:</p> <ol><li><strong>Data in the dataset</strong> — language annotations stored next to frames in <code>data/chunk-*/file-*.parquet</code> as two optional columns (<code>language_persistent</code> and <code>language_events</code>). Datasets without these columns keep their existing
behavior.</li> <li><strong>Recipe</strong> — a YAML file that declares which annotation rows to bind and
how to lay them out as chat turns (<code>role</code>, <code>content</code>, optional images,
optional tool calls). Recipes are pure config; no Python required to add a
new one.</li> <li><strong>Training format</strong> — at sample time, <code>RenderMessagesStep</code> resolves the
recipe against the per-frame annotations and emits HF-style <code>messages</code> plus
LeRobot-specific sidecars (<code>message_streams</code>, <code>target_message_indices</code>)
that policy processors consume.</li></ol> <p>This page describes each layer in turn.</p> <!> <p>The two optional columns live next to frame data in <code>data/chunk-*/file-*.parquet</code>:</p> <ul><li><code>language_persistent</code>: a list of rows broadcast across every frame in an episode for state that remains active, such as <code>subtask</code>, <code>plan</code>, and <code>memory</code>.</li> <li><code>language_events</code>: a list of rows only on the exact frame where an event was emitted, such as <code>interjection</code>, <code>vqa</code>, and speech tool calls.</li></ul> <p>Both columns share the same row shape (event rows omit <code>timestamp</code> because the
frame the row sits on already provides it):</p> <!> <p>The <code>camera</code> field tags rows whose <code>content</code> is grounded in a specific camera
view. Rows of view-dependent styles (<code>vqa</code> and <code>trace</code>) MUST set <code>camera</code> to
the matching <code>observation.images.*</code> feature key. Rows of every other style —
including <code>motion</code>, which describes robot-frame primitives in joint / Cartesian
terms — MUST leave <code>camera</code> as <code>null</code>. Pipeline writers and the validator
enforce this via <code>validate_camera_field(style, camera)</code>.</p> <p><code>meta/tasks.parquet</code> remains the canonical source for the task. The special <code>$&#123;task&#125;</code> recipe binding always reads that task string and does not depend on language annotations.</p> <!> <p>The language stack itself has three internal modules backing layer 1:</p> <ol><li><code>lerobot.datasets.language</code> defines the schema, style registry, and <code>column_for_style</code>.</li> <li><code>lerobot.datasets.language_render</code> resolves rows and renders messages.</li> <li><code>RenderMessagesStep</code> turns dataset samples into <code>messages</code>, <code>message_streams</code>, and <code>target_message_indices</code>.</li></ol> <p><code>LeRobotDataset</code> stays recipe-agnostic. It passes <code>language_persistent</code> and <code>language_events</code> through when present, and unannotated datasets keep their existing behavior.</p> <!> <p>Recipes are YAML files backed by <code>TrainingRecipe</code> and <code>MessageTurn</code>. They
declare which annotation rows to pull (via <code>bindings</code>) and how to compose them
into chat turns (<code>messages</code>).</p> <!> <p>A recipe can also branch into a weighted <strong>blend</strong> of sub-recipes. At sample
time, exactly one branch is selected deterministically from the sample index,
so different frames train different objectives (e.g. memory updates vs.
low-level execution vs. VQA) without any Python wiring.</p> <!> <p>Persistent styles are active after emission until replaced:</p> <ul><li><code>active_at(t, style=subtask)</code></li> <li><code>nth_prev(style=memory, offset=1)</code></li> <li><code>nth_next(style=subtask, offset=1)</code></li></ul> <p>Event styles only exist on their exact timestamp:</p> <ul><li><code>emitted_at(t, style=interjection)</code></li> <li><code>emitted_at(t, style=vqa, role=user, camera=observation.images.top)</code></li> <li><code>emitted_at(t, role=assistant, tool_name=say)</code></li></ul> <p>Exact event matching has no tolerance window, so writers must stamp event rows with frame timestamps from the parquet data.</p> <!> <p>For view-dependent styles (<code>vqa</code> and <code>trace</code>), the resolver gains a <code>camera=</code> filter parallel to <code>role=</code> and <code>tool_name=</code>. Datasets with multiple
cameras typically emit one (<code>vqa</code>, <code>user</code>) + (<code>vqa</code>, <code>assistant</code>) pair per
camera at the same timestamp; without <code>camera=</code>, those resolvers see two
matches and raise an ambiguity error. Recipes consume each camera through its
own binding plus a matching image block, e.g.</p> <!> <p>Add one such sub-recipe per camera the dataset records.</p> <!> <p>Rendered samples use HF-style chat messages plus LeRobot sidecars:</p> <!> <p>The renderer does not apply a tokenizer chat template. Policy processors decide how to serialize the messages for their backbone, which keeps the same dataset usable across SmolVLA, Pi0.5, and any future VLM that expects OpenAI-style chat messages.</p> <!> <p>If both language columns are missing, <code>None</code>, or empty, <code>RenderMessagesStep</code> is a no-op.
If an event-scoped branch is selected on a frame without the required event row, rendering returns <code>None</code>, allowing a loader to retry another sample.</p> <!> <p></p>`,1);function x(J,U){C(U,!1),Z(()=>{new URLSearchParams(window.location.search).get("fw")}),v();var t=E();f("1yjjoj5",u=>{var j=W();_(j,"content",B),w(u,j)});var l=e(Q(t),2);I(l,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var n=e(l,2);a(n,{title:"Language columns and recipes",local:"language-columns-and-recipes",headingTag:"h1"});var o=e(n,10);a(o,{title:"Layer 1 — language columns in the dataset",local:"layer-1--language-columns-in-the-dataset",headingTag:"h2"});var c=e(o,8);s(c,{code:"cm9sZSUzQSUyMHN0cmluZyUwQWNvbnRlbnQlM0ElMjBzdHJpbmclMjAlN0MlMjBudWxsJTBBc3R5bGUlM0ElMjBzdHJpbmclMjAlN0MlMjBudWxsJTBBdGltZXN0YW1wJTNBJTIwZmxvYXQzMiUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMHBlcnNpc3RlbnQlMjByb3dzJTIwb25seSUwQWNhbWVyYSUzQSUyMHN0cmluZyUyMCU3QyUyMG51bGwlMjAlMjAlMjAlMjAlMjAlMjMlMjBvYnNlcnZhdGlvbi5pbWFnZXMuKiUyMGZlYXR1cmUlMjBrZXklMkMlMjB2aWV3LWRlcGVuZGVudCUyMHJvd3MlMjBvbmx5JTBBdG9vbF9jYWxscyUzQSUyMGxpc3QlNUJKc29uJTVEJTIwJTdDJTIwbnVsbA==",highlighted:`role: string
content: string | null
style: string | null
timestamp: float32        # persistent rows only
camera: string | null     # observation.images.* feature key, view-dependent rows only
tool_calls: list[Json] | null`,lang:"text",wrap:!1});var i=e(c,6);a(i,{title:"Architecture",local:"architecture",headingTag:"h3"});var r=e(i,8);a(r,{title:"Layer 2 — recipe anatomy",local:"layer-2--recipe-anatomy",headingTag:"h2"});var d=e(r,4);s(d,{code:"bWVzc2FnZXMlM0ElMEElMjAlMjAtJTIwJTdCJTIwcm9sZSUzQSUyMHVzZXIlMkMlMjBjb250ZW50JTNBJTIwJTIyJTI0JTdCdGFzayU3RCUyMiUyQyUyMHN0cmVhbSUzQSUyMGhpZ2hfbGV2ZWwlMjAlN0QlMEElMjAlMjAtJTIwJTdCJTIwcm9sZSUzQSUyMGFzc2lzdGFudCUyQyUyMGNvbnRlbnQlM0ElMjAlMjIlMjQlN0JzdWJ0YXNrJTdEJTIyJTJDJTIwc3RyZWFtJTNBJTIwbG93X2xldmVsJTJDJTIwdGFyZ2V0JTNBJTIwdHJ1ZSUyMCU3RA==",highlighted:'<span class="hljs-attr">messages:</span>\n  <span class="hljs-bullet">-</span> { <span class="hljs-attr">role:</span> <span class="hljs-string">user</span>, <span class="hljs-attr">content:</span> <span class="hljs-string">&quot;${task}&quot;</span>, <span class="hljs-attr">stream:</span> <span class="hljs-string">high_level</span> }\n  <span class="hljs-bullet">-</span> { <span class="hljs-attr">role:</span> <span class="hljs-string">assistant</span>, <span class="hljs-attr">content:</span> <span class="hljs-string">&quot;${subtask}&quot;</span>, <span class="hljs-attr">stream:</span> <span class="hljs-string">low_level</span>, <span class="hljs-attr">target:</span> <span class="hljs-literal">true</span> }',lang:"yaml",wrap:!1});var p=e(d,4);a(p,{title:"Temporal semantics",local:"temporal-semantics",headingTag:"h3"});var h=e(p,12);a(h,{title:"View-dependent resolution",local:"view-dependent-resolution",headingTag:"h3"});var m=e(h,4);s(m,{code:"YXNrX3ZxYV90b3AlM0ElMEElMjAlMjBiaW5kaW5ncyUzQSUwQSUyMCUyMCUyMCUyMHZxYV9xdWVyeSUzQSUyMCUyMmVtaXR0ZWRfYXQodCUyQyUyMHN0eWxlJTNEdnFhJTJDJTIwcm9sZSUzRHVzZXIlMkMlMjBjYW1lcmElM0RvYnNlcnZhdGlvbi5pbWFnZXMudG9wKSUyMiUwQSUyMCUyMCUyMCUyMHZxYSUzQSUyMCUyMmVtaXR0ZWRfYXQodCUyQyUyMHN0eWxlJTNEdnFhJTJDJTIwcm9sZSUzRGFzc2lzdGFudCUyQyUyMGNhbWVyYSUzRG9ic2VydmF0aW9uLmltYWdlcy50b3ApJTIyJTBBJTIwJTIwbWVzc2FnZXMlM0ElMEElMjAlMjAlMjAlMjAtJTIwcm9sZSUzQSUyMHVzZXIlMEElMjAlMjAlMjAlMjAlMjAlMjBzdHJlYW0lM0ElMjBoaWdoX2xldmVsJTBBJTIwJTIwJTIwJTIwJTIwJTIwaWZfcHJlc2VudCUzQSUyMHZxYV9xdWVyeSUwQSUyMCUyMCUyMCUyMCUyMCUyMGNvbnRlbnQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAtJTIwJTdCJTIwdHlwZSUzQSUyMGltYWdlJTJDJTIwZmVhdHVyZSUzQSUyMG9ic2VydmF0aW9uLmltYWdlcy50b3AlMjAlN0QlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAtJTIwJTdCJTIwdHlwZSUzQSUyMHRleHQlMkMlMjB0ZXh0JTNBJTIwJTIyJTI0JTdCdnFhX3F1ZXJ5JTdEJTIyJTIwJTdEJTBBJTIwJTIwJTIwJTIwLSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJvbGUlM0ElMjBhc3Npc3RhbnQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjb250ZW50JTNBJTIwJTIyJTI0JTdCdnFhJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RyZWFtJTNBJTIwaGlnaF9sZXZlbCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRhcmdldCUzQSUyMHRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZl9wcmVzZW50JTNBJTIwdnFhJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTdE",highlighted:`<span class="hljs-attr">ask_vqa_top:</span>
  <span class="hljs-attr">bindings:</span>
    <span class="hljs-attr">vqa_query:</span> <span class="hljs-string">&quot;emitted_at(t, style=vqa, role=user, camera=observation.images.top)&quot;</span>
    <span class="hljs-attr">vqa:</span> <span class="hljs-string">&quot;emitted_at(t, style=vqa, role=assistant, camera=observation.images.top)&quot;</span>
  <span class="hljs-attr">messages:</span>
    <span class="hljs-bullet">-</span> <span class="hljs-attr">role:</span> <span class="hljs-string">user</span>
      <span class="hljs-attr">stream:</span> <span class="hljs-string">high_level</span>
      <span class="hljs-attr">if_present:</span> <span class="hljs-string">vqa_query</span>
      <span class="hljs-attr">content:</span>
        <span class="hljs-bullet">-</span> { <span class="hljs-attr">type:</span> <span class="hljs-string">image</span>, <span class="hljs-attr">feature:</span> <span class="hljs-string">observation.images.top</span> }
        <span class="hljs-bullet">-</span> { <span class="hljs-attr">type:</span> <span class="hljs-string">text</span>, <span class="hljs-attr">text:</span> <span class="hljs-string">&quot;\${vqa_query}&quot;</span> }
    <span class="hljs-bullet">-</span> {
        <span class="hljs-attr">role:</span> <span class="hljs-string">assistant</span>,
        <span class="hljs-attr">content:</span> <span class="hljs-string">&quot;\${vqa}&quot;</span>,
        <span class="hljs-attr">stream:</span> <span class="hljs-string">high_level</span>,
        <span class="hljs-attr">target:</span> <span class="hljs-literal">true</span>,
        <span class="hljs-attr">if_present:</span> <span class="hljs-string">vqa</span>,
      }`,lang:"yaml",wrap:!1});var M=e(m,4);a(M,{title:"Layer 3 — training format",local:"layer-3--training-format",headingTag:"h2"});var g=e(M,4);s(g,{code:"c2FtcGxlJTVCJTIybWVzc2FnZXMlMjIlNUQlMEFzYW1wbGUlNUIlMjJtZXNzYWdlX3N0cmVhbXMlMjIlNUQlMEFzYW1wbGUlNUIlMjJ0YXJnZXRfbWVzc2FnZV9pbmRpY2VzJTIyJTVE",highlighted:`sample[<span class="hljs-string">&quot;messages&quot;</span>]
sample[<span class="hljs-string">&quot;message_streams&quot;</span>]
sample[<span class="hljs-string">&quot;target_message_indices&quot;</span>]`,lang:"python",wrap:!1});var y=e(g,4);a(y,{title:"Graceful absence",local:"graceful-absence",headingTag:"h2"});var b=e(y,4);A(b,{source:"https://github.com/huggingface/lerobot/blob/main/docs/source/language_and_recipes.mdx"}),z(2),w(J,t),S()}export{x as component};