Add files using upload-large-folder tool

artifacts/logs/rlbench2_subset/multi_3cam_subset/BIMANUAL_PERACT/seed0/events.out.tfevents.1774229659.e52be9725b2e.1397.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9de379324d03cd07826de977c88b5b0af260f56775bac9274dbf4a7fa137206d
+size 88

artifacts/logs/rlbench2_subset/multi_3cam_subset/BIMANUAL_PERACT/seed0/events.out.tfevents.1774229898.e52be9725b2e.3678.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfe9cbe8ab18dd6bebc242d4fa4f080708080db28723815106734ce828d59aeb
+size 88

artifacts/logs/rlbench2_subset/multi_3cam_subset/BIMANUAL_PERACT/seed0/events.out.tfevents.1774229987.e52be9725b2e.3804.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44120d28c2f86549b5eb1f094f75390a1d5c487ec9a9b23cf5dc592f06c4ac75
+size 88

artifacts/logs/rlbench2_subset/multi_3cam_subset/BIMANUAL_PERACT/seed0/events.out.tfevents.1774230133.e52be9725b2e.3957.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29ae48bb1181a3bf46fd265ebc5e3bda8ebb738bb60779fef5ac9f5f6e535ee8
+size 88

artifacts/logs/rlbench2_subset/multi_3cam_subset/BIMANUAL_PERACT/seed0/events.out.tfevents.1774230462.e52be9725b2e.4130.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdc64fc04a005fbeae272ae65861ba291f73255668aa8f124c028ccf89402f7d
+size 88

artifacts/logs/rlbench2_subset/multi_3cam_subset/BIMANUAL_PERACT/seed0/weights/0/QAttentionAgent_layer0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:145ce79566eef7d9dff9d15ec3cec6636eaed8c1a6981adb20279e14e0d9b141
+size 332572639

artifacts/logs/rlbench2_subset3/multi_3cam_subset3/BIMANUAL_PERACT/seed0/events.out.tfevents.1774230948.e52be9725b2e.4400.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4075460938d070e509edd6a261928c25ba15d7b707f086273ece97ac31fb4603
+size 88

artifacts/logs/rlbench2_subset3/multi_3cam_subset3/BIMANUAL_PERACT/seed0/events.out.tfevents.1774231048.e52be9725b2e.4582.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72479e8a2b6c9cade8ffeb37429a6d8ddc3c0bd9678f7f09a8efd11aceca6458
+size 88

artifacts/logs/rlbench2_subset3/multi_3cam_subset3/BIMANUAL_PERACT/seed0/events.out.tfevents.1774231173.e52be9725b2e.4703.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14a6e4307c460255d3251b4563350e9aa484ca6cfd06af44da2d12d487a28207
+size 88

artifacts/logs/rlbench2_subset3/multi_3cam_subset3/BIMANUAL_PERACT/seed0/events.out.tfevents.1774231301.e52be9725b2e.4824.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a01b9dbac814fdfb0cb8bc364204ad053dc9f00eb58a9995a76974537f8625f
+size 88

artifacts/logs/rlbench2_subset3/multi_3cam_subset3/BIMANUAL_PERACT/seed0/events.out.tfevents.1774231438.e52be9725b2e.4951.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:250476f994aca860bf7fa083340dc9303eb5d395949fd61ead6e3f2bb7b7eab3
+size 140

artifacts/logs/rlbench2_subset3/multi_3cam_subset3/BIMANUAL_PERACT/seed0/weights/0/QAttentionAgent_layer0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2658c0b80b93d2701635898fe5abcc4303809f200920656c3c9a0a2fd4a193cc
+size 332572639

artifacts/outputs/reveal_runs/proxy_backbone_only/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb1090a52c89e840335b4736045d6153b8f1b827751d6b38fed2bd8494ed71e3
+size 2720837

artifacts/outputs/reveal_runs/proxy_backbone_only_clip/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e575c499b6b1595e062d4a374af9fe91ae5104b5acff0fc7cf6e3c71e94b4601
+size 723184260

artifacts/outputs/reveal_runs/proxy_reveal_state/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9aaca3280dc735adeac013f471183b09f9fffa203176e2a5a8cbb53f9a0a682
+size 3958745

artifacts/outputs/reveal_runs/proxy_reveal_state_clip/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af4a8d84b187a34450230256d85c34229679fcdea9ec0c622c2e6a9f66405e41
+size 734374962

artifacts/outputs/rlbench_custom/rlbench_subset3_backbone_only_clip/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f01315f74b4e069760ff32cd8660fc06236c29f0c2f1c64e2e95d899903c1001
+size 723184644

artifacts/outputs/rlbench_custom/rlbench_subset3_backbone_only_dummy/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3132aee5fbbff4e7c47cf8e870fabc99e8438623aab7de0b2f254cfc342cc81
+size 2720488

artifacts/outputs/rlbench_custom/rlbench_subset3_reveal_state_clip/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8c2e1de7e4246c3da2dd18cc0b4b9227300db701a7ae1291cc9027ec527ad5a
+size 734374898

artifacts/outputs/rlbench_custom/rlbench_subset3_reveal_state_dummy/checkpoint_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e6f095148284e79db7b8b114dce45edbc0e98c679992eda8c933ce097eba4a4
+size 3957308

code/reveal_vla_bimanual/sim_rlbench/dataset.py

@@ -0,0 +1,312 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import pickle
+from typing import Any, Sequence
+
+import numpy as np
+import torch
+from PIL import Image
+from torch.utils.data import Dataset
+
+
+THREE_CAMERAS: tuple[str, str, str] = ("front", "wrist_left", "wrist_right")
+
+
+def _normalize_quaternion_wxyz(quaternion: np.ndarray) -> np.ndarray:
+    quaternion = np.asarray(quaternion, dtype=np.float32)
+    return quaternion / max(float(np.linalg.norm(quaternion)), 1e-8)
+
+
+def _xyzw_to_wxyz(quaternion_xyzw: Sequence[float]) -> np.ndarray:
+    qx, qy, qz, qw = [float(value) for value in quaternion_xyzw]
+    return _normalize_quaternion_wxyz(np.array([qw, qx, qy, qz], dtype=np.float32))
+
+
+def _wxyz_to_xyzw(quaternion_wxyz: Sequence[float]) -> np.ndarray:
+    qw, qx, qy, qz = [float(value) for value in quaternion_wxyz]
+    return _normalize_quaternion_wxyz(np.array([qx, qy, qz, qw], dtype=np.float32))
+
+
+def _quat_multiply_wxyz(lhs: Sequence[float], rhs: Sequence[float]) -> np.ndarray:
+    lw, lx, ly, lz = [float(value) for value in lhs]
+    rw, rx, ry, rz = [float(value) for value in rhs]
+    return _normalize_quaternion_wxyz(
+        np.array(
+            [
+                lw * rw - lx * rx - ly * ry - lz * rz,
+                lw * rx + lx * rw + ly * rz - lz * ry,
+                lw * ry - lx * rz + ly * rw + lz * rx,
+                lw * rz + lx * ry - ly * rx + lz * rw,
+            ],
+            dtype=np.float32,
+        )
+    )
+
+
+def _quat_inverse_wxyz(quaternion_wxyz: Sequence[float]) -> np.ndarray:
+    qw, qx, qy, qz = [float(value) for value in quaternion_wxyz]
+    return _normalize_quaternion_wxyz(np.array([qw, -qx, -qy, -qz], dtype=np.float32))
+
+
+def _quat_to_rotvec_wxyz(quaternion_wxyz: Sequence[float]) -> np.ndarray:
+    quaternion_wxyz = _normalize_quaternion_wxyz(np.asarray(quaternion_wxyz, dtype=np.float32))
+    qw = float(np.clip(quaternion_wxyz[0], -1.0, 1.0))
+    xyz = quaternion_wxyz[1:]
+    sin_half = float(np.linalg.norm(xyz))
+    if sin_half < 1e-8:
+        return np.zeros(3, dtype=np.float32)
+    angle = 2.0 * np.arctan2(sin_half, qw)
+    axis = xyz / sin_half
+    return (axis * angle).astype(np.float32)
+
+
+def _rotvec_to_quat_wxyz(rotvec: Sequence[float]) -> np.ndarray:
+    rotvec = np.asarray(rotvec, dtype=np.float32)
+    angle = float(np.linalg.norm(rotvec))
+    if angle < 1e-8:
+        return np.array([1.0, 0.0, 0.0, 0.0], dtype=np.float32)
+    axis = rotvec / angle
+    half = angle * 0.5
+    sin_half = np.sin(half)
+    return _normalize_quaternion_wxyz(
+        np.array(
+            [np.cos(half), axis[0] * sin_half, axis[1] * sin_half, axis[2] * sin_half],
+            dtype=np.float32,
+        )
+    )
+
+
+def bimanual_proprio_from_obs(
+    obs: Any,
+    timestep: int,
+    episode_length: int,
+    target_dim: int = 32,
+) -> np.ndarray:
+    time_feature = np.array(
+        [(1.0 - (timestep / float(max(1, episode_length - 1)))) * 2.0 - 1.0],
+        dtype=np.float32,
+    )
+    base = np.concatenate(
+        [
+            np.asarray(obs.right.gripper_pose, dtype=np.float32),
+            np.asarray(obs.left.gripper_pose, dtype=np.float32),
+            np.asarray(obs.right.joint_positions, dtype=np.float32),
+            np.asarray(obs.left.joint_positions, dtype=np.float32),
+            np.array([float(obs.right.gripper_open), float(obs.left.gripper_open)], dtype=np.float32),
+            time_feature,
+        ],
+        axis=0,
+    )
+    if base.shape[0] >= target_dim:
+        return base[:target_dim]
+    padding = np.zeros(target_dim - base.shape[0], dtype=np.float32)
+    return np.concatenate([base, padding], axis=0)
+
+
+def delta_action_from_transition(current_obs: Any, next_obs: Any) -> np.ndarray:
+    action_parts: list[np.ndarray] = []
+    for arm_name in ("right", "left"):
+        current_arm = getattr(current_obs, arm_name)
+        next_arm = getattr(next_obs, arm_name)
+        current_pose = np.asarray(current_arm.gripper_pose, dtype=np.float32)
+        next_pose = np.asarray(next_arm.gripper_pose, dtype=np.float32)
+        position_delta = next_pose[:3] - current_pose[:3]
+        current_quat = _xyzw_to_wxyz(current_pose[3:])
+        next_quat = _xyzw_to_wxyz(next_pose[3:])
+        delta_quat = _quat_multiply_wxyz(next_quat, _quat_inverse_wxyz(current_quat))
+        delta_rotvec = _quat_to_rotvec_wxyz(delta_quat)
+        action_parts.append(
+            np.concatenate(
+                [
+                    position_delta.astype(np.float32),
+                    delta_rotvec.astype(np.float32),
+                    np.array([float(next_arm.gripper_open)], dtype=np.float32),
+                ],
+                axis=0,
+            )
+        )
+    return np.concatenate(action_parts, axis=0).astype(np.float32)
+
+
+def absolute_action_from_delta(current_obs: Any, delta_action: Sequence[float], ignore_collisions: bool = True) -> np.ndarray:
+    delta_action = np.asarray(delta_action, dtype=np.float32)
+    if delta_action.shape != (14,):
+        raise ValueError(f"Expected delta action shape (14,), received {delta_action.shape}")
+
+    env_action: list[float] = []
+    for arm_index, arm_name in enumerate(("right", "left")):
+        arm = getattr(current_obs, arm_name)
+        current_pose = np.asarray(arm.gripper_pose, dtype=np.float32)
+        offset = arm_index * 7
+        delta_position = delta_action[offset : offset + 3]
+        delta_rotvec = delta_action[offset + 3 : offset + 6]
+        gripper = float(delta_action[offset + 6] > 0.5)
+        current_quat = _xyzw_to_wxyz(current_pose[3:])
+        delta_quat = _rotvec_to_quat_wxyz(delta_rotvec)
+        next_quat = _quat_multiply_wxyz(delta_quat, current_quat)
+        next_pose = np.concatenate(
+            [
+                current_pose[:3] + delta_position,
+                _wxyz_to_xyzw(next_quat),
+            ],
+            axis=0,
+        )
+        env_action.extend(next_pose.tolist())
+        env_action.append(gripper)
+        env_action.append(float(ignore_collisions))
+    return np.asarray(env_action, dtype=np.float32)
+
+
+def stack_live_rgb_obs(obs: Any, cameras: Sequence[str] = THREE_CAMERAS, resolution: int = 224) -> torch.Tensor:
+    images: list[np.ndarray] = []
+    for camera_name in cameras:
+        rgb = np.asarray(obs.perception_data[f"{camera_name}_rgb"], dtype=np.uint8)
+        image = Image.fromarray(rgb)
+        if image.size != (resolution, resolution):
+            image = image.resize((resolution, resolution), Image.Resampling.BILINEAR)
+        images.append(np.asarray(image, dtype=np.uint8))
+    stacked = np.stack(images, axis=0)
+    return torch.from_numpy(stacked).permute(0, 3, 1, 2).float() / 255.0
+
+
+def language_goal_from_episode(episode_dir: Path) -> str:
+    description_path = episode_dir / "variation_descriptions.pkl"
+    with description_path.open("rb") as handle:
+        descriptions = pickle.load(handle)
+    if isinstance(descriptions, (list, tuple)) and descriptions:
+        return str(descriptions[0])
+    return str(descriptions)
+
+
+def load_episode_observations(episode_dir: Path) -> Any:
+    with (episode_dir / "low_dim_obs.pkl").open("rb") as handle:
+        return pickle.load(handle)
+
+
+@dataclass(frozen=True)
+class EpisodeRecord:
+    task: str
+    episode_index: int
+    episode_dir: Path
+    language_goal: str
+    observations: Any
+
+
+@dataclass(frozen=True)
+class SampleRecord:
+    episode_key: str
+    task: str
+    episode_index: int
+    step_index: int
+
+
+class RLBenchOfflineChunkDataset(Dataset[dict[str, Any]]):
+    def __init__(
+        self,
+        dataset_root: str | Path,
+        tasks: Sequence[str],
+        episode_indices: Sequence[int],
+        resolution: int = 224,
+        chunk_size: int = 8,
+        proprio_dim: int = 32,
+        cameras: Sequence[str] = THREE_CAMERAS,
+        max_samples: int | None = None,
+    ) -> None:
+        self.dataset_root = Path(dataset_root)
+        self.tasks = tuple(tasks)
+        self.episode_indices = tuple(int(index) for index in episode_indices)
+        self.resolution = int(resolution)
+        self.chunk_size = int(chunk_size)
+        self.proprio_dim = int(proprio_dim)
+        self.cameras = tuple(cameras)
+        self._episodes: dict[str, EpisodeRecord] = {}
+        self._samples: list[SampleRecord] = []
+
+        for task in self.tasks:
+            for episode_index in self.episode_indices:
+                episode_dir = self.dataset_root / task / "all_variations" / "episodes" / f"episode{episode_index}"
+                if not episode_dir.exists():
+                    continue
+                observations = load_episode_observations(episode_dir)
+                episode_key = f"{task}:episode{episode_index}"
+                self._episodes[episode_key] = EpisodeRecord(
+                    task=task,
+                    episode_index=episode_index,
+                    episode_dir=episode_dir,
+                    language_goal=language_goal_from_episode(episode_dir),
+                    observations=observations,
+                )
+                for step_index in range(max(0, len(observations) - 1)):
+                    self._samples.append(
+                        SampleRecord(
+                            episode_key=episode_key,
+                            task=task,
+                            episode_index=episode_index,
+                            step_index=step_index,
+                        )
+                    )
+                    if max_samples is not None and len(self._samples) >= max_samples:
+                        return
+
+    def __len__(self) -> int:
+        return len(self._samples)
+
+    def _load_rgb_stack(self, episode_dir: Path, step_index: int) -> torch.Tensor:
+        frames: list[np.ndarray] = []
+        for camera_name in self.cameras:
+            image_path = episode_dir / f"{camera_name}_rgb" / f"rgb_{step_index:04d}.png"
+            image = Image.open(image_path).convert("RGB")
+            if image.size != (self.resolution, self.resolution):
+                image = image.resize((self.resolution, self.resolution), Image.Resampling.BILINEAR)
+            frames.append(np.asarray(image, dtype=np.uint8))
+        stacked = np.stack(frames, axis=0)
+        return torch.from_numpy(stacked).permute(0, 3, 1, 2).float() / 255.0
+
+    def _action_chunk(self, observations: Any, start_index: int) -> torch.Tensor:
+        actions: list[np.ndarray] = []
+        last_valid = delta_action_from_transition(observations[-2], observations[-1])
+        for offset in range(self.chunk_size):
+            index = start_index + offset
+            if index < len(observations) - 1:
+                action = delta_action_from_transition(observations[index], observations[index + 1])
+                last_valid = action
+            else:
+                action = last_valid
+            actions.append(action)
+        return torch.from_numpy(np.stack(actions, axis=0))
+
+    def __getitem__(self, index: int) -> dict[str, Any]:
+        sample = self._samples[index]
+        episode = self._episodes[sample.episode_key]
+        observations = episode.observations
+        obs = observations[sample.step_index]
+        return {
+            "images": self._load_rgb_stack(episode.episode_dir, sample.step_index),
+            "proprio": torch.from_numpy(
+                bimanual_proprio_from_obs(
+                    obs,
+                    timestep=sample.step_index,
+                    episode_length=len(observations),
+                    target_dim=self.proprio_dim,
+                )
+            ),
+            "texts": episode.language_goal,
+            "action_chunk": self._action_chunk(observations, sample.step_index),
+            "task": sample.task,
+            "episode_index": sample.episode_index,
+            "step_index": sample.step_index,
+        }
+
+    def summary(self) -> dict[str, Any]:
+        return {
+            "dataset_root": str(self.dataset_root),
+            "tasks": list(self.tasks),
+            "episode_indices": list(self.episode_indices),
+            "num_episodes": len(self._episodes),
+            "num_samples": len(self._samples),
+            "resolution": self.resolution,
+            "chunk_size": self.chunk_size,
+            "proprio_dim": self.proprio_dim,
+        }

code/reveal_vla_bimanual/sim_rlbench/launch_smoke.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import argparse
+import json
+import os
+
+import numpy as np
+from helpers.observation_utils import create_obs_config
+from rlbench.action_modes.action_mode import BimanualJointPositionActionMode
+from rlbench.backend.utils import task_file_to_task_class
+from rlbench.environment import Environment
+
+from sim_rlbench.obs_adapter import extract_canonical_bimanual_obs
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--task", default="bimanual_lift_ball")
+    parser.add_argument("--resolution", type=int, default=224)
+    parser.add_argument("--display", default=None)
+    parser.add_argument("--headless", action="store_true", default=True)
+    parser.add_argument("--visible", action="store_true")
+    args = parser.parse_args()
+
+    headless = args.headless and not args.visible
+    cameras = ["front", "wrist_left", "wrist_right"]
+    obs_config = create_obs_config(
+        cameras,
+        [args.resolution, args.resolution],
+        "BIMANUAL_PERACT",
+        "bimanual",
+    )
+    env = Environment(
+        action_mode=BimanualJointPositionActionMode(),
+        obs_config=obs_config,
+        headless=headless,
+        robot_setup="dual_panda",
+    )
+
+    try:
+        env.launch()
+        task_cls = task_file_to_task_class(args.task, bimanual=True)
+        task = env.get_task(task_cls)
+        descriptions, obs = task.reset()
+        canonical = extract_canonical_bimanual_obs(
+            obs,
+            descriptions[0],
+            include_point_cloud=True,
+        )
+
+        right_joint_positions = np.asarray(obs.right.joint_positions, dtype=np.float32)
+        left_joint_positions = np.asarray(obs.left.joint_positions, dtype=np.float32)
+        right_open = float(obs.right.gripper_open)
+        left_open = float(obs.left.gripper_open)
+        action = np.concatenate(
+            [
+                right_joint_positions,
+                np.array([right_open], dtype=np.float32),
+                left_joint_positions,
+                np.array([left_open], dtype=np.float32),
+            ],
+            axis=0,
+        )
+        next_obs, reward, done = task.step(action)
+
+        payload = {
+            "display": args.display or os.environ.get("DISPLAY"),
+            "headless": headless,
+            "task": task.get_name(),
+            "description": descriptions[0],
+            "rgb_shapes": {k: list(v.shape) for k, v in canonical.rgb.items()},
+            "intrinsic_shapes": {
+                k: list(v.shape) for k, v in canonical.camera_intrinsics.items()
+            },
+            "extrinsic_shapes": {
+                k: list(v.shape) for k, v in canonical.camera_extrinsics.items()
+            },
+            "point_cloud_shapes": {
+                k: list(v.shape) for k, v in canonical.point_cloud.items()
+            },
+            "proprio_shape": list(canonical.proprio.shape),
+            "action_shape": list(action.shape),
+            "reward": float(reward),
+            "done": bool(done),
+            "front_rgb_shape_after_step": list(next_obs.perception_data["front_rgb"].shape),
+        }
+        print(json.dumps(payload, indent=2))
+    finally:
+        env.shutdown()
+
+
+if __name__ == "__main__":
+    main()