/** * v3.1 language schema (lerobot#3467 + lerobot#3471). * * Each row in `language_persistent` and `language_events` has the same shape: * * { role, content, style, timestamp, tool_calls } * * Persistent styles (task_aug / subtask / plan / memory) live in * `language_persistent` and are broadcast across every frame in the episode. Event styles * (interjection / vqa) plus speech tool-call atoms (style=null) live in * `language_events` and only appear on the exact frames where they were * emitted. * * VQA assistant rows encode their answer as a JSON string in `content`, in one * of five shapes (bbox / keypoint / count / attribute / spatial), matching * `VQA_ANSWER_SHAPES` in lerobot's steerable-pipeline validator. */ export type Role = "user" | "assistant" | "system" | "tool"; export type LanguageStyle = | "task_aug" | "subtask" | "plan" | "memory" | "interjection" | "vqa"; export interface ToolCallFn { name: string; arguments: Record; } export interface ToolCall { type: "function"; function: ToolCallFn; } export interface LanguageAtom { role: Role; content: string | null; // null is reserved for tool-call-only atoms (speech). style: LanguageStyle | null; timestamp: number; /** * `observation.images.*` feature key when this atom is grounded against a * specific camera view (`vqa`, `trace`). `null` for camera-agnostic atoms * (`task_aug`, `subtask`, `plan`, `memory`, `motion`, `interjection`, speech). * Mirrors lerobot's row-level `camera` field (PR 3467). */ camera: string | null; tool_calls: ToolCall[] | null; } export const PERSISTENT_STYLES: ReadonlySet = new Set([ "task_aug", "subtask", "plan", "memory", ]); export const EVENT_STYLES: ReadonlySet = new Set([ "interjection", "vqa", ]); export function columnForStyle( style: LanguageStyle | null, ): "language_persistent" | "language_events" { if (style === null) return "language_events"; if (PERSISTENT_STYLES.has(style)) return "language_persistent"; if (EVENT_STYLES.has(style)) return "language_events"; throw new Error(`Unknown style: ${String(style)}`); } export function isSpeechAtom(a: LanguageAtom): boolean { return ( a.style === null && a.role === "assistant" && !!a.tool_calls && a.tool_calls.length > 0 && a.tool_calls[0]?.function?.name === "say" ); } export function speechText(a: LanguageAtom): string | null { if (!isSpeechAtom(a)) return null; const args = a.tool_calls?.[0]?.function?.arguments as | { text?: unknown } | undefined; return typeof args?.text === "string" ? args.text : null; } export function buildSpeechAtom(timestamp: number, text: string): LanguageAtom { return { role: "assistant", content: null, style: null, timestamp, camera: null, tool_calls: [ { type: "function", function: { name: "say", arguments: { text } }, }, ], }; } /** * Whether ``atom`` should render on the camera identified by ``cameraKey``. * * - row-level ``atom.camera`` is the source of truth (lerobot PR 3467); * ``null`` means camera-agnostic and renders everywhere; * a non-null value matches only its own camera. * - For backwards compatibility with annotations that were created before the * row-level field existed (visualizer-only payloads with an in-JSON * ``camera`` key inside the VQA answer), the caller is responsible for the * payload-level fallback — see ``video-overlay-canvas.tsx``. */ export function atomMatchesCamera( atom: LanguageAtom, cameraKey: string, ): boolean { return atom.camera == null || atom.camera === cameraKey; } // --- VQA answer shapes ------------------------------------------------------- export type VqaAnswer = | VqaBboxAnswer | VqaKeypointAnswer | VqaCountAnswer | VqaAttributeAnswer | VqaSpatialAnswer; export interface VqaBboxAnswer { detections: Array<{ label: string; bbox_format: "xyxy" | "xywh"; bbox: [number, number, number, number]; /** * Optional camera key for the visualizer (e.g. `observation.images.top`). * Not enforced by lerobot's writer but accepted as a passthrough JSON * field since the validator checks only required keys. */ camera?: string; }>; } export interface VqaKeypointAnswer { label: string; point_format: "xy"; point: [number, number]; camera?: string; } export interface VqaCountAnswer { label: string; count: number; note?: string; } export interface VqaAttributeAnswer { label: string; attribute: string; value: string; } export interface VqaSpatialAnswer { subject: string; relation: string; object: string; } export type VqaKind = "bbox" | "keypoint" | "count" | "attribute" | "spatial"; export function classifyVqa(answer: unknown): VqaKind | null { if (!answer || typeof answer !== "object") return null; const a = answer as Record; if (Array.isArray(a.detections)) return "bbox"; if (a.point_format && Array.isArray(a.point)) return "keypoint"; if (typeof a.count === "number" && typeof a.label === "string") return "count"; if (typeof a.attribute === "string" && a.value != null) return "attribute"; if (typeof a.subject === "string" && typeof a.relation === "string") return "spatial"; return null; } export function parseVqaAnswer( raw: string | null | undefined, ): VqaAnswer | null { if (!raw) return null; try { const parsed = JSON.parse(raw); return classifyVqa(parsed) ? (parsed as VqaAnswer) : null; } catch { return null; } } // --- Atom helpers used by the editor UI -------------------------------------- export interface EpisodeAtoms { persistent: LanguageAtom[]; events: LanguageAtom[]; } /** Group all atoms for an episode by their target column. */ export function partitionAtoms(atoms: LanguageAtom[]): EpisodeAtoms { const persistent: LanguageAtom[] = []; const events: LanguageAtom[] = []; for (const a of atoms) { if (columnForStyle(a.style) === "language_persistent") persistent.push(a); else events.push(a); } persistent.sort((a, b) => a.timestamp - b.timestamp); events.sort((a, b) => a.timestamp - b.timestamp); return { persistent, events }; } export function atomsByStyle( atoms: LanguageAtom[], style: LanguageStyle, ): LanguageAtom[] { return atoms.filter((a) => a.style === style); } export function activeAt( persistent: LanguageAtom[], style: LanguageStyle, t: number, ): LanguageAtom | null { let best: LanguageAtom | null = null; for (const a of persistent) { if (a.style !== style) continue; if (a.timestamp > t) break; best = a; } return best; } export function eventsAt( events: LanguageAtom[], t: number, windowSec = 1 / 60, ): LanguageAtom[] { return events.filter((a) => Math.abs(a.timestamp - t) <= windowSec); } /** * Snap a timestamp to the nearest source-frame timestamp. Linear scan; episodes * are typically a few thousand frames so this is fine without a tree. */ export function snapToFrame(frameTimestamps: number[], ts: number): number { if (!frameTimestamps.length) return ts; let best = frameTimestamps[0]; let dist = Math.abs(ts - best); for (let i = 1; i < frameTimestamps.length; i++) { const d = Math.abs(ts - frameTimestamps[i]); if (d < dist) { dist = d; best = frameTimestamps[i]; } } return best; }