Add packaged camera dataset sample

.gitattributes

@@ -14,6 +14,7 @@ assets/examples/droid_plus_lerobot_640x360_20260412/success/videos/observation.i
 assets/examples/droid_plus_lerobot_640x360_20260412/success/videos/observation.image.wrist_image_left/chunk-000/file-000.mp4 filter=lfs diff=lfs merge=lfs -text
 assets/examples/fractal20220817_data/videos/observation.images.image/chunk-000/file-000.mp4 filter=lfs diff=lfs merge=lfs -text
 assets/examples/av_v2_03292026_wdinfo/data/00000000.tar filter=lfs diff=lfs merge=lfs -text
+assets/examples/camera_benchmark_v3/data/00000000.tar filter=lfs diff=lfs merge=lfs -text
 assets/examples/fastumi/fastumi_single_arm/pour_coke/videos/observation.image.right_main_camera_rgb/chunk-000/file-000.mp4 filter=lfs diff=lfs merge=lfs -text
 assets/examples/AgiBotWorld-GEAR_20260208/agibot-offshelf/20251016_500h/gripper/task_1018/data/chunk-000/file_000.parquet filter=lfs diff=lfs merge=lfs -text
 assets/examples/AgiBotWorld-GEAR_20260208/agibot-offshelf/20251016_500h/gripper/task_1018/videos/observation.images.hand_left/chunk-000/file-000.mp4 filter=lfs diff=lfs merge=lfs -text

assets/examples/camera_benchmark_v3/data/00000000.tar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:816d31b913ba99e4b755efb1767e8ecf52fc5586451148416604eeec9a9c1743
+size 5201920

assets/examples/camera_benchmark_v3/wdinfo.json

@@ -0,0 +1,14 @@
+{
+  "data_keys": [
+    "pkl"
+  ],
+  "chunk_size": 1,
+  "root": "data",
+  "data_list": [
+    "00000000.tar"
+  ],
+  "data_list_key_count": [
+    1
+  ],
+  "total_key_count": 1
+}

cosmos-framework/cosmos_framework/data/vfm/action/action_viz/app_test.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 import numpy as np
 import pytest
 
@@ -12,6 +14,36 @@ from cosmos_framework.data.vfm.action.action_viz.state import (
     GenerationResult,
     write_generated_action,
 )
+from cosmos_framework.data.vfm.action.camera_benchmark_dataset import CameraBenchmarkDataset
+from cosmos_framework.data.vfm.action.urdf_visualizer.unified_action import ActionFormat
+from cosmos_framework.data.vfm.action.urdf_visualizer.viewer import _build_datasets
+
+
+@pytest.mark.L0
+def test_camera_dataset_registry_uses_validation_split() -> None:
+    entry = _build_datasets()["camera"]
+
+    assert entry.action_format is ActionFormat.EGO_9D
+    assert entry.dataset_class.endswith("camera_benchmark_dataset.CameraBenchmarkDataset")
+    assert entry.dataset_kwargs["root"].endswith("/camera_benchmark_v3")
+    assert entry.dataset_kwargs["split"] == "val"
+    assert entry.dataset_kwargs["shuffle"] is False
+    assert entry.dataset_kwargs["num_frames"] == 61
+    assert entry.dataset_kwargs["translation_scale"] == 10
+    assert "credential_path" not in entry.dataset_kwargs
+
+
+@pytest.mark.L0
+def test_camera_packaged_sample_loads() -> None:
+    repo_root = Path(__file__).resolve().parents[6]
+    dataset = CameraBenchmarkDataset(root=str(repo_root / "assets/examples/camera_benchmark_v3"), num_frames=61)
+
+    sample = dataset[0]
+
+    assert sample["video"].shape == (3, 61, 720, 1280)
+    assert sample["action"].shape == (60, 9)
+    assert sample["conditioning_fps"].item() == 30.0
+    assert isinstance(sample["ai_caption"], str)
 
 
 @pytest.mark.L0

cosmos-framework/cosmos_framework/data/vfm/action/action_viz/local_worker.py

@@ -575,8 +575,26 @@ def _write_conditioning_video(request: GenerationRequest, job_dir: Path, action_
 
     frame_count = min(len(frames), int(action_chunk_size) + 1)
     vision_path = job_dir / "conditioning.mp4"
-    imageio.mimsave(vision_path, list(frames[:frame_count]), fps=entry.fps, codec="libx264", macro_block_size=16)
-    return vision_path, int(entry.fps)
+    fps = _sample_conditioning_fps(sample, default=entry.fps)
+    imageio.mimsave(vision_path, list(frames[:frame_count]), fps=fps, codec="libx264", macro_block_size=16)
+    return vision_path, fps
+
+
+def _sample_conditioning_fps(sample: dict[str, Any], default: int) -> int:
+    raw = sample.get("conditioning_fps")
+    if raw is None:
+        return int(default)
+    try:
+        if hasattr(raw, "detach"):
+            raw = raw.detach().cpu().item()
+        elif hasattr(raw, "item"):
+            raw = raw.item()
+        fps = int(round(float(raw)))
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"Invalid sample conditioning_fps value {raw!r}") from exc
+    if fps <= 0:
+        raise ValueError(f"Invalid sample conditioning_fps value {raw!r}")
+    return fps
 
 
 def _write_action_values(request: GenerationRequest, job_dir: Path) -> tuple[Path, int, dict[str, Any]]:
@@ -601,6 +619,7 @@ def _domain_name_for_dataset(dataset: str) -> str:
     domains = {
         "av": "av",
         "bridge": "bridge_orig_lerobot",
+        "camera": "camera_pose",
         "droid": "droid_lerobot",
         "fractal": "fractal",
         "robomind_franka": "robomind-franka",

cosmos-framework/cosmos_framework/data/vfm/action/action_viz/local_worker_test.py

@@ -9,7 +9,10 @@ import pytest
 
 from cosmos_framework.data.vfm.action.action_viz.local_worker import (
     _copy_generation_outputs,
+    _domain_name_for_dataset,
     _patched_unipc_progress,
+    _sample_conditioning_fps,
+    _transform_action_values,
 )
 from cosmos_framework.data.vfm.action.action_viz.state import (
     BRIDGE_ACTION_DIM,
@@ -114,6 +117,27 @@ def test_copy_generation_outputs_requires_action_for_inverse_dynamics(tmp_path)
         _copy_generation_outputs(request, inference_dir, Path(request.output_dir))
 
 
+@pytest.mark.L0
+def test_camera_domain_and_pose_scale_normalization() -> None:
+    assert _domain_name_for_dataset("camera") == "camera_pose"
+
+    model_rows, summary = _transform_action_values(
+        [[0.1, -0.2, 0.3, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0]],
+        "camera",
+        to_model_space=True,
+    )
+
+    assert summary["method"] == "pose_scale"
+    assert summary["action_dim"] == 9
+    np.testing.assert_allclose(model_rows[0], [1.0, -2.0, 3.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0])
+
+
+@pytest.mark.L0
+def test_sample_conditioning_fps_prefers_sample_metadata() -> None:
+    assert _sample_conditioning_fps({"conditioning_fps": np.array(29.97, dtype=np.float32)}, default=10) == 30
+    assert _sample_conditioning_fps({}, default=10) == 10
+
+
 def _request(tmp_path, model_mode: str) -> GenerationRequest:
     return GenerationRequest(
         generation_id="gen",

cosmos-framework/cosmos_framework/data/vfm/action/action_viz/normalization_stats.json

@@ -11,6 +11,12 @@
     "q01": [-0.038884, -0.028667, -0.03784, 0.976292, -0.163098, -0.081545, -0.160193, 0.976322, -0.078872, 0.0],
     "q99": [0.039722, 0.029068, 0.026702, 1.0, 0.160195, 0.081655, 0.163227, 1.0, 0.095189, 1.0]
   },
+  "camera": {
+    "action_dim": 9,
+    "method": "pose_scale",
+    "rotation_scale": 1.0,
+    "translation_scale": 10.0
+  },
   "droid": {
     "action_dim": 10,
     "method": "quantile_rot",

cosmos-framework/cosmos_framework/data/vfm/action/camera_benchmark_dataset.py

@@ -0,0 +1,195 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Packaged camera benchmark samples for the action viewer."""
+
+from __future__ import annotations
+
+import json
+import pickle
+import tarfile
+from io import BytesIO
+from pathlib import Path
+from typing import Literal
+
+import av
+import numpy as np
+import torch
+from scipy.spatial.transform import Rotation as R
+from torch.utils.data import Dataset
+
+from cosmos_framework.data.vfm.action.domain_utils import get_domain_id
+from cosmos_framework.data.vfm.action.pose_utils import RotationConvention, pose_abs_to_rel
+
+
+class CameraBenchmarkDataset(Dataset):
+    """Load camera-pose validation samples packaged with the viewer assets."""
+
+    def __init__(
+        self,
+        root: str = "/app/assets/examples/camera_benchmark_v3",
+        mode: str = "policy",
+        split: str = "val",
+        shuffle: bool = False,
+        rotation_format: RotationConvention = "rot6d",
+        pose_convention: Literal["backward_anchored", "backward_framewise"] = "backward_framewise",
+        fix_caption: bool = False,
+        fix_caption_text: str = "The camera moves in a scene.",
+        num_frames: int = 17,
+        embodiment_type: str = "camera_pose",
+        **_: object,
+    ) -> None:
+        del shuffle
+        split = split.lower().strip()
+        if split not in {"val", "valid", "validation", "full"}:
+            raise ValueError("CameraBenchmarkDataset supports only the packaged validation split.")
+        if mode not in {"joint", "forward_dynamics", "inverse_dynamics", "policy", "image2video"}:
+            raise ValueError(f"Unsupported camera dataset mode {mode!r}")
+
+        self.root = Path(root)
+        if not self.root.exists():
+            raise ValueError(f"Camera benchmark root does not exist: {self.root}")
+        self.mode = mode
+        self.rotation_format = rotation_format
+        self.pose_convention = pose_convention
+        self.fix_caption = bool(fix_caption)
+        self.fix_caption_text = fix_caption_text
+        self.num_frames = int(num_frames)
+        self.domain_id = get_domain_id(embodiment_type)
+        self._tar_files, self._key_counts, self._total_key_count = _load_wdinfo(self.root)
+
+    def __len__(self) -> int:
+        return self._total_key_count
+
+    def __getitem__(self, index: int) -> dict[str, object]:
+        if index < 0:
+            index += len(self)
+        if index < 0 or index >= len(self):
+            raise IndexError(index)
+
+        tar_path, local_index = self._locate_record(index)
+        record = _load_record(tar_path, local_index)
+        metadata = record["metadata"]
+        video, fps = _decode_video(record["video"], self.num_frames)
+        camera_data = json.loads(record["camera"])
+        camera_to_world = _camera_to_world_poses(camera_data, len(video[0]))
+        action = pose_abs_to_rel(
+            camera_to_world,
+            rotation_format=self.rotation_format,
+            pose_convention=self.pose_convention,
+            translation_scale=1.0,
+            rotation_scale=1.0,
+        )
+        return {
+            "video": video,
+            "action": torch.from_numpy(action),
+            "conditioning_fps": torch.tensor(metadata.get("fps", fps), dtype=torch.float32),
+            "ai_caption": self.fix_caption_text if self.fix_caption else str(metadata["caption"]),
+            "mode": "policy" if self.mode == "joint" else self.mode,
+            "__key__": torch.tensor([index], dtype=torch.long),
+            "domain_id": torch.tensor(self.domain_id, dtype=torch.long),
+            "viewpoint": "ego_view",
+        }
+
+    def _locate_record(self, index: int) -> tuple[Path, int]:
+        offset = 0
+        for tar_path, key_count in zip(self._tar_files, self._key_counts, strict=True):
+            next_offset = offset + key_count
+            if index < next_offset:
+                return tar_path, index - offset
+            offset = next_offset
+        raise IndexError(index)
+
+
+def _load_wdinfo(root: Path) -> tuple[list[Path], list[int], int]:
+    wdinfo = json.loads((root / "wdinfo.json").read_text())
+    data_root = wdinfo.get("root", "")
+    data_list = wdinfo.get("data_list", [])
+    if not data_list:
+        raise RuntimeError(f"No camera benchmark tar files listed in {root / 'wdinfo.json'}")
+
+    chunk_size = int(wdinfo.get("chunk_size", 1))
+    key_counts = [int(value) for value in wdinfo.get("data_list_key_count", [chunk_size] * len(data_list))]
+    if len(key_counts) != len(data_list):
+        raise ValueError("camera benchmark wdinfo data_list_key_count must match data_list length")
+
+    tar_root = root / data_root if data_root else root
+    tar_files = [tar_root / filename for filename in data_list]
+    total_key_count = int(wdinfo.get("total_key_count", sum(key_counts)))
+    if total_key_count <= 0:
+        raise RuntimeError(f"No packaged camera benchmark samples found under {root}")
+    return tar_files, key_counts, total_key_count
+
+
+def _load_record(tar_path: Path, local_index: int) -> dict[str, object]:
+    with tarfile.open(tar_path, mode="r:*") as tar:
+        members = sorted(
+            (member for member in tar.getmembers() if member.isfile() and member.name.endswith(".pkl")),
+            key=lambda member: member.name,
+        )
+        if local_index >= len(members):
+            raise IndexError(local_index)
+        extracted = tar.extractfile(members[local_index])
+        if extracted is None:
+            raise RuntimeError(f"Failed to extract {members[local_index].name} from {tar_path}")
+        try:
+            return pickle.load(extracted)
+        finally:
+            extracted.close()
+
+
+def _decode_video(video_bytes: bytes, num_frames: int) -> tuple[torch.Tensor, float]:
+    container = av.open(BytesIO(video_bytes), mode="r")
+    try:
+        stream = next((stream for stream in container.streams.video), None)
+        if stream is None:
+            raise ValueError("Camera benchmark video has no video stream")
+        fps = float(stream.average_rate or stream.base_rate or 30.0)
+        frames = [frame.to_rgb().to_ndarray() for frame in container.decode(stream)]
+    finally:
+        container.close()
+
+    if num_frames > 0:
+        target_count = 1 + 4 * ((num_frames - 1) // 4)
+        if len(frames) < target_count:
+            raise ValueError(
+                f"Camera benchmark video has {len(frames)} frames, but {target_count} are required"
+            )
+        frames = frames[:target_count]
+    if len(frames) < 2:
+        raise ValueError("Camera benchmark video must contain at least two frames")
+
+    video = torch.from_numpy(np.stack(frames, axis=0)).permute(3, 0, 1, 2).contiguous()
+    return video, fps
+
+
+def _camera_to_world_poses(camera_data: dict[str, object], frame_count: int) -> np.ndarray:
+    camera = camera_data["camera"]
+    if not isinstance(camera, dict):
+        raise ValueError("Camera benchmark camera.json must contain a camera object")
+    world_to_camera = np.asarray(camera["pose_world2cam"], dtype=np.float32).reshape(-1, 7)
+    if len(world_to_camera) < frame_count:
+        raise ValueError(
+            f"Camera benchmark camera.json has {len(world_to_camera)} poses, "
+            f"but {frame_count} are required"
+        )
+    world_to_camera = _extrinsic_params_to_matrices(world_to_camera[:frame_count])
+    homogeneous = np.repeat(np.eye(4, dtype=np.float32)[None], frame_count, axis=0)
+    homogeneous[:, :3, :] = world_to_camera
+    return np.linalg.inv(homogeneous).astype(np.float32, copy=False)
+
+
+def _extrinsic_params_to_matrices(qxyzw_t: np.ndarray) -> np.ndarray:
+    """Convert ``[qx, qy, qz, qw, tx, ty, tz]`` rows to world-to-camera matrices."""
+
+    values = np.asarray(qxyzw_t, dtype=np.float32)
+    if values.ndim < 1 or values.shape[-1] != 7:
+        raise ValueError(f"Camera extrinsics must have trailing dim 7, got {values.shape}")
+    quat = values[..., :4]
+    norm = np.linalg.norm(quat, axis=-1, keepdims=True)
+    if np.any(norm < 1e-8):
+        raise ValueError("Camera extrinsics contain a zero-norm quaternion")
+    rotation = R.from_quat((quat / norm).reshape(-1, 4)).as_matrix().astype(np.float32)
+    rotation = rotation.reshape(*values.shape[:-1], 3, 3)
+    translation = values[..., 4:7, None]
+    return np.concatenate([rotation, translation], axis=-1).astype(np.float32, copy=False)

cosmos-framework/cosmos_framework/data/vfm/action/urdf_visualizer/action_datasets.py

@@ -11,6 +11,7 @@ from typing import Any, Callable
 
 from cosmos_framework.data.vfm.action.av_dataset import AVDataset
 from cosmos_framework.data.vfm.action.bridge_orig_lerobot_dataset import BridgeOrigLeRobotDataset
+from cosmos_framework.data.vfm.action.camera_benchmark_dataset import CameraBenchmarkDataset
 from cosmos_framework.data.vfm.action.droid_lerobot_dataset import DROIDLeRobotDataset
 from cosmos_framework.data.vfm.action.fractal import FractalLeRobotDataset
 from cosmos_framework.data.vfm.action.robomind_franka_dataset import RoboMINDFrankaDataset
@@ -42,6 +43,7 @@ def _env_path(*names: str, default: str) -> str:
 
 
 BRIDGE_ROOT = _env_path("BRIDGE_LEROBOT_ROOT", "DATASET_PATH", default="/app/assets/examples/bridge_lerobot_v3")
+CAMERA_ROOT = _env_path("CAMERA_ROOT", default="/app/assets/examples/camera_benchmark_v3")
 
 DATASET_BRIDGE_480 = L(dataset_entry)(
     name="bridge_20260501",
@@ -62,6 +64,25 @@ DATASET_BRIDGE_480 = L(dataset_entry)(
 )
 
 
+DATASET_CAMERA_480 = L(dataset_entry)(
+    name="camera_480_20260501",
+    dataset=L(CameraBenchmarkDataset)(
+        root=CAMERA_ROOT,
+        split="val",
+        shuffle=False,
+        fix_caption=False,
+        num_frames=61,
+        translation_scale=10,
+        max_action_translation_norm=10,
+        mode="policy",
+        rotation_format="rot6d",
+        pose_convention="backward_framewise",
+    ),
+    ratio=1,
+    resolution="480",
+)
+
+
 DATASET_FRACTAL_256 = L(dataset_entry)(
     name="fractal_20260501",
     dataset=L(FractalLeRobotDataset)(

cosmos-framework/cosmos_framework/data/vfm/action/urdf_visualizer/viewer.py

@@ -121,6 +121,7 @@ def _build_datasets() -> dict[str, DatasetEntry]:
     from cosmos_framework.data.vfm.action.urdf_visualizer.action_datasets import (
         DATASET_AV_480,
         DATASET_BRIDGE_480,
+        DATASET_CAMERA_480,
         DATASET_DROID_480,
         DATASET_FRACTAL_256,
         DATASET_ROBOMIND_FRANKA_480,
@@ -161,6 +162,13 @@ def _build_datasets() -> dict[str, DatasetEntry]:
             action_format=ActionFormat.SINGLE_ARM_10D,
             to_opencv=_BRIDGE_TO_OPENCV,
         ),
+        "camera": _lazycfg_to_entry(
+            DATASET_CAMERA_480,
+            robot_name="",
+            max_finger_width=0.0,
+            fps=30,
+            action_format=ActionFormat.EGO_9D,
+        ),
         "droid": _lazycfg_to_entry(
             DATASET_DROID_480,
             robot_name="franka_panda",
@@ -221,7 +229,7 @@ def _create_dataset(entry: DatasetEntry, chunk_length: int):
 
     kwargs = dict(entry.dataset_kwargs)
     kwargs["chunk_length"] = chunk_length
-    kwargs["split"] = "full"
+    kwargs.setdefault("split", "full")
     kwargs["mode"] = "policy"
     kwargs["enable_fast_init"] = True
 

start.sh

@@ -34,6 +34,7 @@ fi
 export COSMOS_VIEWER_ON_DEMAND_VIDEO="${COSMOS_VIEWER_ON_DEMAND_VIDEO:-0}"
 export COSMOS_VIEWER_DOWNLOAD_DATA="${COSMOS_VIEWER_DOWNLOAD_DATA:-0}"
 export BRIDGE_LEROBOT_ROOT="${BRIDGE_LEROBOT_ROOT:-/app/assets/examples/bridge_lerobot_v3}"
+export CAMERA_ROOT="${CAMERA_ROOT:-/app/assets/examples/camera_benchmark_v3}"
 export AV_ROOT="${AV_ROOT:-/app/assets/examples/av_v2_03292026_wdinfo}"
 export UMI_ROOT="${UMI_ROOT:-/app/assets/examples/fastumi/fastumi_single_arm/pour_coke}"
 export FRACTAL_ROOT="${FRACTAL_ROOT:-/app/assets/examples/fractal20220817_data}"
@@ -59,6 +60,7 @@ echo "ACTION_VIZ_PERSISTENT_RUNNER=${ACTION_VIZ_PERSISTENT_RUNNER}"
 echo "ACTION_VIZ_USE_TORCH_COMPILE=${ACTION_VIZ_USE_TORCH_COMPILE}"
 echo "PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF}"
 echo "BRIDGE_LEROBOT_ROOT=${BRIDGE_LEROBOT_ROOT}"
+echo "CAMERA_ROOT=${CAMERA_ROOT}"
 echo "COSMOS_VIEWER_ON_DEMAND_VIDEO=${COSMOS_VIEWER_ON_DEMAND_VIDEO}"
 echo "COSMOS_VIEWER_SHARE=${COSMOS_VIEWER_SHARE}"
 echo "AV_ROOT=${AV_ROOT}"