Add files using upload-large-folder tool

artifacts/outputs/rlbench_current/rlbench_lift_ball_backbone_only_clip_current_3train/config_resolved.yaml

@@ -0,0 +1,92 @@
+experiment_name: rlbench_lift_ball_backbone_only_clip_current_3train
+output_dir: /workspace/outputs/rlbench_current
+device: cuda
+seed: 17
+init_checkpoint: null
+init_strict: false
+data:
+  dataset_root: /workspace/data/rlbench2
+  tasks:
+  - bimanual_lift_ball
+  train_episodes:
+  - 0
+  - 1
+  - 2
+  val_episodes:
+  - 3
+  resolution: 224
+  chunk_horizon: 8
+  proprio_dim: 32
+  history_steps: 2
+optim:
+  epochs: 12
+  batch_size: 8
+  num_workers: 2
+  lr: 0.0002
+  weight_decay: 0.0001
+trainer:
+  policy_type: backbone_only
+  use_bf16: true
+  grad_clip_norm: 1.0
+  freeze_backbone: true
+  gradient_checkpointing: false
+  plan_during_train: false
+  plan_during_eval: false
+  support_mode_conditioning: true
+policy:
+  backbone:
+    model_name: openai/clip-vit-base-patch32
+    hidden_dim: 512
+    max_text_tokens: 32
+    freeze_backbone: true
+    gradient_checkpointing: false
+    use_dummy_backbone: false
+  fusion:
+    hidden_dim: 512
+    num_cameras: 3
+    num_layers: 4
+    num_heads: 8
+    ff_dim: 2048
+    dropout: 0.1
+    proprio_dim: 32
+    proprio_tokens: 1
+  memory:
+    hidden_dim: 512
+    history_steps: 2
+    num_layers: 1
+    dropout: 0.1
+  decoder:
+    hidden_dim: 512
+    num_heads: 8
+    num_layers: 4
+    ff_dim: 2048
+    dropout: 0.1
+    chunk_size: 8
+    action_dim: 14
+    num_candidates: 8
+  reveal_head:
+    hidden_dim: 512
+    num_support_modes: 3
+    num_approach_templates: 32
+    rollout_horizon: 5
+    belief_map_size: 32
+    predict_belief_map: true
+  world_model:
+    hidden_dim: 512
+    action_dim: 14
+    num_support_modes: 3
+    num_approach_templates: 32
+    rollout_horizon: 5
+  planner:
+    hidden_dim: 512
+    num_candidates: 8
+    action_dim: 14
+    utility_margin: 0.1
+loss_weights:
+  action: 1.0
+  support_mode: 0.0
+  corridor: 0.0
+  persistence: 0.0
+  disturbance: 0.0
+  world_model: 0.0
+  belief: 0.0

artifacts/outputs/rlbench_current/rlbench_lift_ball_backbone_only_clip_current_3train/metrics.json

@@ -0,0 +1,350 @@
+[
+  {
+    "epoch": 0,
+    "train": {
+      "action": 0.6453182360434786,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.6453182360434786,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.04164583683013916,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.04164583683013916,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 1,
+    "train": {
+      "action": 0.028159307454653244,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.028159307454653244,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.006555992768456539,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.006555992768456539,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 2,
+    "train": {
+      "action": 0.007879757025140398,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.007879757025140398,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.0027059585011253756,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0027059585011253756,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 3,
+    "train": {
+      "action": 0.005019047991075414,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.005019047991075414,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.002921265425781409,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.002921265425781409,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 4,
+    "train": {
+      "action": 0.004040793243280434,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.004040793243280434,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.0015864447845766942,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0015864447845766942,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 5,
+    "train": {
+      "action": 0.0030465805804000255,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0030465805804000255,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.0009213978890329599,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0009213978890329599,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 6,
+    "train": {
+      "action": 0.0023699305986272213,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0023699305986272213,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.0007997404201887548,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0007997404201887548,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 7,
+    "train": {
+      "action": 0.0020967414255868245,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0020967414255868245,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.0008208795178992052,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0008208795178992052,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 8,
+    "train": {
+      "action": 0.0017702074008102112,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0017702074008102112,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.0005620980557675163,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0005620980557675163,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 9,
+    "train": {
+      "action": 0.0015501773475292832,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0015501773475292832,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.0007885866759655376,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0007885866759655376,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 10,
+    "train": {
+      "action": 0.0014139077096107476,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0014139077096107476,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.000618226492467026,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.000618226492467026,
+      "world_model": 0.0
+    }
+  },
+  {
+    "epoch": 11,
+    "train": {
+      "action": 0.0012609842183821378,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0012609842183821378,
+      "world_model": 0.0
+    },
+    "val": {
+      "action": 0.0005350256532741089,
+      "planner_ranking": 0.0,
+      "planner_risk": 0.0,
+      "planner_success": 0.0,
+      "proposal_diversity": 0.0,
+      "proposal_ranking": 0.0,
+      "proposal_reconstruction": 0.0,
+      "proposal_success": 0.0,
+      "role_swap_consistency": 0.0,
+      "total": 0.0005350256532741089,
+      "world_model": 0.0
+    }
+  }
+]

artifacts/outputs/rlbench_current/rlbench_lift_ball_backbone_only_clip_current_3train/summary.json

@@ -0,0 +1,44 @@
+{
+  "experiment_name": "rlbench_lift_ball_backbone_only_clip_current_3train",
+  "device": "cuda",
+  "best_checkpoint": "/workspace/outputs/rlbench_current/rlbench_lift_ball_backbone_only_clip_current_3train/checkpoint_best.pt",
+  "final_train_total": 0.0012609842183821378,
+  "final_val_total": 0.0005350256532741089,
+  "train_dataset": {
+    "dataset_root": "/workspace/data/rlbench2",
+    "tasks": [
+      "bimanual_lift_ball"
+    ],
+    "episode_indices": [
+      0,
+      1,
+      2
+    ],
+    "num_episodes": 3,
+    "num_samples": 370,
+    "resolution": 224,
+    "chunk_size": 8,
+    "proprio_dim": 32,
+    "history_steps": 2
+  },
+  "val_dataset": {
+    "dataset_root": "/workspace/data/rlbench2",
+    "tasks": [
+      "bimanual_lift_ball"
+    ],
+    "episode_indices": [
+      3
+    ],
+    "num_episodes": 1,
+    "num_samples": 120,
+    "resolution": 224,
+    "chunk_size": 8,
+    "proprio_dim": 32,
+    "history_steps": 2
+  },
+  "init_info": null,
+  "planner_mode": "trainable",
+  "plan_enabled_for_train": false,
+  "plan_enabled_for_eval": false,
+  "frozen_modules": []
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_dual_push_buttons/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_dual_push_buttons --tasks bimanual_dual_push_buttons --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_dual_push_buttons/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_dual_push_buttons": {
+      "task_class": "BimanualDualPushButtons",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "push the olive and the orange buttons",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.197153091430664,
+                11.241825103759766,
+                11.236907005310059,
+                11.205011367797852
+              ],
+              "predicted_reocclusion": 0.5305227041244507,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.197153091430664,
+                11.241825103759766,
+                11.236907005310059,
+                11.205011367797852
+              ],
+              "predicted_reocclusion": 0.5305227041244507,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.197153091430664,
+                11.241825103759766,
+                11.236907005310059,
+                11.205011367797852
+              ],
+              "predicted_reocclusion": 0.5305227041244507,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.197153091430664,
+                11.241825103759766,
+                11.236907005310059,
+                11.205011367797852
+              ],
+              "predicted_reocclusion": 0.5305227041244507,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.1263427734375,
+                11.163692474365234,
+                11.160633087158203,
+                11.130797386169434
+              ],
+              "predicted_reocclusion": 0.5315501689910889,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.1263427734375,
+                11.163692474365234,
+                11.160633087158203,
+                11.130797386169434
+              ],
+              "predicted_reocclusion": 0.5315501689910889,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.1263427734375,
+                11.163692474365234,
+                11.160633087158203,
+                11.130797386169434
+              ],
+              "predicted_reocclusion": 0.5315501689910889,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.1263427734375,
+                11.163692474365234,
+                11.160633087158203,
+                11.130797386169434
+              ],
+              "predicted_reocclusion": 0.5315501689910889,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.077136039733887,
+                11.114724159240723,
+                11.111690521240234,
+                11.081738471984863
+              ],
+              "predicted_reocclusion": 0.5313586592674255,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.077136039733887,
+                11.114724159240723,
+                11.111690521240234,
+                11.081738471984863
+              ],
+              "predicted_reocclusion": 0.5313586592674255,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.077136039733887,
+                11.114724159240723,
+                11.111690521240234,
+                11.081738471984863
+              ],
+              "predicted_reocclusion": 0.5313586592674255,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.077136039733887,
+                11.114724159240723,
+                11.111690521240234,
+                11.081738471984863
+              ],
+              "predicted_reocclusion": 0.5313586592674255,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.042268753051758,
+                11.08004379272461,
+                11.07697868347168,
+                11.046956062316895
+              ],
+              "predicted_reocclusion": 0.5312807559967041,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.042268753051758,
+                11.08004379272461,
+                11.07697868347168,
+                11.046956062316895
+              ],
+              "predicted_reocclusion": 0.5312807559967041,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.042268753051758,
+                11.08004379272461,
+                11.07697868347168,
+                11.046956062316895
+              ],
+              "predicted_reocclusion": 0.5312807559967041,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.042268753051758,
+                11.08004379272461,
+                11.07697868347168,
+                11.046956062316895
+              ],
+              "predicted_reocclusion": 0.5312807559967041,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.03925895690918,
+                11.076944351196289,
+                11.073898315429688,
+                11.043900489807129
+              ],
+              "predicted_reocclusion": 0.5312473773956299,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.03925895690918,
+                11.076944351196289,
+                11.073898315429688,
+                11.043900489807129
+              ],
+              "predicted_reocclusion": 0.5312473773956299,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.03925895690918,
+                11.076944351196289,
+                11.073898315429688,
+                11.043900489807129
+              ],
+              "predicted_reocclusion": 0.5312473773956299,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.03925895690918,
+                11.076944351196289,
+                11.073898315429688,
+                11.043900489807129
+              ],
+              "predicted_reocclusion": 0.5312473773956299,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_dual_push_buttons/rollout_eval.md

@@ -0,0 +1,14 @@
+# RLBench Rollout Eval
+
+- Checkpoint: `/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt`
+- Plan requested: `True`
+- Plan applied: `True`
+- Support-mode conditioning: `True`
+- Task conditioning: `True`
+- Geometry enabled: `True`
+- World-model mode: `checkpoint_default`
+- Mean success: `0.000`
+
+## Per-task
+
+- `bimanual_dual_push_buttons`: mean_success=0.000, returns=[0.0]

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_dual_push_buttons/stderr.txt

@@ -0,0 +1,4 @@
+/workspace/envs/rlbench/lib/python3.10/site-packages/torch/nn/modules/transformer.py:306: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
+  warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
+qt.qpa.xcb: QXcbConnection: XCB error: 148 (Unknown), sequence: 181, resource id: 0, major code: 140 (Unknown), minor code: 20
+WARNING:root:not sure how _robot_shapes are used is used.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_dual_push_buttons/stdout.txt

@@ -0,0 +1,334 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_dual_push_buttons": {
+      "task_class": "BimanualDualPushButtons",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "push the olive and the orange buttons",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.197153091430664,
+                11.241825103759766,
+                11.236907005310059,
+                11.205011367797852
+              ],
+              "predicted_reocclusion": 0.5305227041244507,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.197153091430664,
+                11.241825103759766,
+                11.236907005310059,
+                11.205011367797852
+              ],
+              "predicted_reocclusion": 0.5305227041244507,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.197153091430664,
+                11.241825103759766,
+                11.236907005310059,
+                11.205011367797852
+              ],
+              "predicted_reocclusion": 0.5305227041244507,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.197153091430664,
+                11.241825103759766,
+                11.236907005310059,
+                11.205011367797852
+              ],
+              "predicted_reocclusion": 0.5305227041244507,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.1263427734375,
+                11.163692474365234,
+                11.160633087158203,
+                11.130797386169434
+              ],
+              "predicted_reocclusion": 0.5315501689910889,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.1263427734375,
+                11.163692474365234,
+                11.160633087158203,
+                11.130797386169434
+              ],
+              "predicted_reocclusion": 0.5315501689910889,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.1263427734375,
+                11.163692474365234,
+                11.160633087158203,
+                11.130797386169434
+              ],
+              "predicted_reocclusion": 0.5315501689910889,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.1263427734375,
+                11.163692474365234,
+                11.160633087158203,
+                11.130797386169434
+              ],
+              "predicted_reocclusion": 0.5315501689910889,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.077136039733887,
+                11.114724159240723,
+                11.111690521240234,
+                11.081738471984863
+              ],
+              "predicted_reocclusion": 0.5313586592674255,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.077136039733887,
+                11.114724159240723,
+                11.111690521240234,
+                11.081738471984863
+              ],
+              "predicted_reocclusion": 0.5313586592674255,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.077136039733887,
+                11.114724159240723,
+                11.111690521240234,
+                11.081738471984863
+              ],
+              "predicted_reocclusion": 0.5313586592674255,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.077136039733887,
+                11.114724159240723,
+                11.111690521240234,
+                11.081738471984863
+              ],
+              "predicted_reocclusion": 0.5313586592674255,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.042268753051758,
+                11.08004379272461,
+                11.07697868347168,
+                11.046956062316895
+              ],
+              "predicted_reocclusion": 0.5312807559967041,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.042268753051758,
+                11.08004379272461,
+                11.07697868347168,
+                11.046956062316895
+              ],
+              "predicted_reocclusion": 0.5312807559967041,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.042268753051758,
+                11.08004379272461,
+                11.07697868347168,
+                11.046956062316895
+              ],
+              "predicted_reocclusion": 0.5312807559967041,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.042268753051758,
+                11.08004379272461,
+                11.07697868347168,
+                11.046956062316895
+              ],
+              "predicted_reocclusion": 0.5312807559967041,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.03925895690918,
+                11.076944351196289,
+                11.073898315429688,
+                11.043900489807129
+              ],
+              "predicted_reocclusion": 0.5312473773956299,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.03925895690918,
+                11.076944351196289,
+                11.073898315429688,
+                11.043900489807129
+              ],
+              "predicted_reocclusion": 0.5312473773956299,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.03925895690918,
+                11.076944351196289,
+                11.073898315429688,
+                11.043900489807129
+              ],
+              "predicted_reocclusion": 0.5312473773956299,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "lift_support_layer",
+              "planner_scores": [
+                11.03925895690918,
+                11.076944351196289,
+                11.073898315429688,
+                11.043900489807129
+              ],
+              "predicted_reocclusion": 0.5312473773956299,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}
+[CoppeliaSim:loadinfo]   done.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item --tasks bimanual_handover_item --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_handover_item": {
+      "task_class": "BimanualHandoverItem",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "bring me the red item",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.330599784851074,
+                11.321919441223145,
+                11.334712982177734,
+                11.290019989013672
+              ],
+              "predicted_reocclusion": 0.45670241117477417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.330599784851074,
+                11.321919441223145,
+                11.334712982177734,
+                11.290019989013672
+              ],
+              "predicted_reocclusion": 0.45670241117477417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.330599784851074,
+                11.321919441223145,
+                11.334712982177734,
+                11.290019989013672
+              ],
+              "predicted_reocclusion": 0.45670241117477417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.330599784851074,
+                11.321919441223145,
+                11.334712982177734,
+                11.290019989013672
+              ],
+              "predicted_reocclusion": 0.45670241117477417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.277860641479492,
+                11.269150733947754,
+                11.281505584716797,
+                11.236681938171387
+              ],
+              "predicted_reocclusion": 0.4565059244632721,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.277860641479492,
+                11.269150733947754,
+                11.281505584716797,
+                11.236681938171387
+              ],
+              "predicted_reocclusion": 0.4565059244632721,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.277860641479492,
+                11.269150733947754,
+                11.281505584716797,
+                11.236681938171387
+              ],
+              "predicted_reocclusion": 0.4565059244632721,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.277860641479492,
+                11.269150733947754,
+                11.281505584716797,
+                11.236681938171387
+              ],
+              "predicted_reocclusion": 0.4565059244632721,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.233684539794922,
+                11.225020408630371,
+                11.237412452697754,
+                11.192520141601562
+              ],
+              "predicted_reocclusion": 0.4565500319004059,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.233684539794922,
+                11.225020408630371,
+                11.237412452697754,
+                11.192520141601562
+              ],
+              "predicted_reocclusion": 0.4565500319004059,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.233684539794922,
+                11.225020408630371,
+                11.237412452697754,
+                11.192520141601562
+              ],
+              "predicted_reocclusion": 0.4565500319004059,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.233684539794922,
+                11.225020408630371,
+                11.237412452697754,
+                11.192520141601562
+              ],
+              "predicted_reocclusion": 0.4565500319004059,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.203593254089355,
+                11.195001602172852,
+                11.207406044006348,
+                11.162463188171387
+              ],
+              "predicted_reocclusion": 0.45659494400024414,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.203593254089355,
+                11.195001602172852,
+                11.207406044006348,
+                11.162463188171387
+              ],
+              "predicted_reocclusion": 0.45659494400024414,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.203593254089355,
+                11.195001602172852,
+                11.207406044006348,
+                11.162463188171387
+              ],
+              "predicted_reocclusion": 0.45659494400024414,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.203593254089355,
+                11.195001602172852,
+                11.207406044006348,
+                11.162463188171387
+              ],
+              "predicted_reocclusion": 0.45659494400024414,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.192391395568848,
+                11.18377685546875,
+                11.196148872375488,
+                11.151208877563477
+              ],
+              "predicted_reocclusion": 0.45683759450912476,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.192391395568848,
+                11.18377685546875,
+                11.196148872375488,
+                11.151208877563477
+              ],
+              "predicted_reocclusion": 0.45683759450912476,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.192391395568848,
+                11.18377685546875,
+                11.196148872375488,
+                11.151208877563477
+              ],
+              "predicted_reocclusion": 0.45683759450912476,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.192391395568848,
+                11.18377685546875,
+                11.196148872375488,
+                11.151208877563477
+              ],
+              "predicted_reocclusion": 0.45683759450912476,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item/rollout_eval.md

@@ -0,0 +1,14 @@
+# RLBench Rollout Eval
+
+- Checkpoint: `/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt`
+- Plan requested: `True`
+- Plan applied: `True`
+- Support-mode conditioning: `True`
+- Task conditioning: `True`
+- Geometry enabled: `True`
+- World-model mode: `checkpoint_default`
+- Mean success: `0.000`
+
+## Per-task
+
+- `bimanual_handover_item`: mean_success=0.000, returns=[0.0]

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item/stderr.txt

@@ -0,0 +1,4 @@
+/workspace/envs/rlbench/lib/python3.10/site-packages/torch/nn/modules/transformer.py:306: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
+  warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
+qt.qpa.xcb: QXcbConnection: XCB error: 148 (Unknown), sequence: 181, resource id: 0, major code: 140 (Unknown), minor code: 20
+WARNING:root:not sure how _robot_shapes are used is used.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item/stdout.txt

@@ -0,0 +1,334 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_handover_item": {
+      "task_class": "BimanualHandoverItem",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "bring me the red item",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.330599784851074,
+                11.321919441223145,
+                11.334712982177734,
+                11.290019989013672
+              ],
+              "predicted_reocclusion": 0.45670241117477417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.330599784851074,
+                11.321919441223145,
+                11.334712982177734,
+                11.290019989013672
+              ],
+              "predicted_reocclusion": 0.45670241117477417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.330599784851074,
+                11.321919441223145,
+                11.334712982177734,
+                11.290019989013672
+              ],
+              "predicted_reocclusion": 0.45670241117477417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.330599784851074,
+                11.321919441223145,
+                11.334712982177734,
+                11.290019989013672
+              ],
+              "predicted_reocclusion": 0.45670241117477417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.277860641479492,
+                11.269150733947754,
+                11.281505584716797,
+                11.236681938171387
+              ],
+              "predicted_reocclusion": 0.4565059244632721,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.277860641479492,
+                11.269150733947754,
+                11.281505584716797,
+                11.236681938171387
+              ],
+              "predicted_reocclusion": 0.4565059244632721,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.277860641479492,
+                11.269150733947754,
+                11.281505584716797,
+                11.236681938171387
+              ],
+              "predicted_reocclusion": 0.4565059244632721,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.277860641479492,
+                11.269150733947754,
+                11.281505584716797,
+                11.236681938171387
+              ],
+              "predicted_reocclusion": 0.4565059244632721,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.233684539794922,
+                11.225020408630371,
+                11.237412452697754,
+                11.192520141601562
+              ],
+              "predicted_reocclusion": 0.4565500319004059,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.233684539794922,
+                11.225020408630371,
+                11.237412452697754,
+                11.192520141601562
+              ],
+              "predicted_reocclusion": 0.4565500319004059,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.233684539794922,
+                11.225020408630371,
+                11.237412452697754,
+                11.192520141601562
+              ],
+              "predicted_reocclusion": 0.4565500319004059,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.233684539794922,
+                11.225020408630371,
+                11.237412452697754,
+                11.192520141601562
+              ],
+              "predicted_reocclusion": 0.4565500319004059,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.203593254089355,
+                11.195001602172852,
+                11.207406044006348,
+                11.162463188171387
+              ],
+              "predicted_reocclusion": 0.45659494400024414,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.203593254089355,
+                11.195001602172852,
+                11.207406044006348,
+                11.162463188171387
+              ],
+              "predicted_reocclusion": 0.45659494400024414,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.203593254089355,
+                11.195001602172852,
+                11.207406044006348,
+                11.162463188171387
+              ],
+              "predicted_reocclusion": 0.45659494400024414,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.203593254089355,
+                11.195001602172852,
+                11.207406044006348,
+                11.162463188171387
+              ],
+              "predicted_reocclusion": 0.45659494400024414,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.192391395568848,
+                11.18377685546875,
+                11.196148872375488,
+                11.151208877563477
+              ],
+              "predicted_reocclusion": 0.45683759450912476,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.192391395568848,
+                11.18377685546875,
+                11.196148872375488,
+                11.151208877563477
+              ],
+              "predicted_reocclusion": 0.45683759450912476,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.192391395568848,
+                11.18377685546875,
+                11.196148872375488,
+                11.151208877563477
+              ],
+              "predicted_reocclusion": 0.45683759450912476,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.192391395568848,
+                11.18377685546875,
+                11.196148872375488,
+                11.151208877563477
+              ],
+              "predicted_reocclusion": 0.45683759450912476,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}
+[CoppeliaSim:loadinfo]   done.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item_easy/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item_easy --tasks bimanual_handover_item_easy --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item_easy/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_handover_item_easy": {
+      "task_class": "BimanualHandoverItemEasy",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "bring me the item",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.548981666564941,
+                11.52706241607666,
+                11.533926963806152,
+                11.543230056762695
+              ],
+              "predicted_reocclusion": 0.5040647983551025,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.548981666564941,
+                11.52706241607666,
+                11.533926963806152,
+                11.543230056762695
+              ],
+              "predicted_reocclusion": 0.5040647983551025,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.548981666564941,
+                11.52706241607666,
+                11.533926963806152,
+                11.543230056762695
+              ],
+              "predicted_reocclusion": 0.5040647983551025,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.548981666564941,
+                11.52706241607666,
+                11.533926963806152,
+                11.543230056762695
+              ],
+              "predicted_reocclusion": 0.5040647983551025,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.516067504882812,
+                11.490782737731934,
+                11.499794006347656,
+                11.509395599365234
+              ],
+              "predicted_reocclusion": 0.5022199749946594,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.516067504882812,
+                11.490782737731934,
+                11.499794006347656,
+                11.509395599365234
+              ],
+              "predicted_reocclusion": 0.5022199749946594,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.516067504882812,
+                11.490782737731934,
+                11.499794006347656,
+                11.509395599365234
+              ],
+              "predicted_reocclusion": 0.5022199749946594,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.516067504882812,
+                11.490782737731934,
+                11.499794006347656,
+                11.509395599365234
+              ],
+              "predicted_reocclusion": 0.5022199749946594,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.468103408813477,
+                11.442843437194824,
+                11.45191478729248,
+                11.461535453796387
+              ],
+              "predicted_reocclusion": 0.5022915601730347,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.468103408813477,
+                11.442843437194824,
+                11.45191478729248,
+                11.461535453796387
+              ],
+              "predicted_reocclusion": 0.5022915601730347,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.468103408813477,
+                11.442843437194824,
+                11.45191478729248,
+                11.461535453796387
+              ],
+              "predicted_reocclusion": 0.5022915601730347,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.468103408813477,
+                11.442843437194824,
+                11.45191478729248,
+                11.461535453796387
+              ],
+              "predicted_reocclusion": 0.5022915601730347,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.437320709228516,
+                11.412096977233887,
+                11.421077728271484,
+                11.430691719055176
+              ],
+              "predicted_reocclusion": 0.501986026763916,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.437320709228516,
+                11.412096977233887,
+                11.421077728271484,
+                11.430691719055176
+              ],
+              "predicted_reocclusion": 0.501986026763916,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.437320709228516,
+                11.412096977233887,
+                11.421077728271484,
+                11.430691719055176
+              ],
+              "predicted_reocclusion": 0.501986026763916,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.437320709228516,
+                11.412096977233887,
+                11.421077728271484,
+                11.430691719055176
+              ],
+              "predicted_reocclusion": 0.501986026763916,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.425673484802246,
+                11.400392532348633,
+                11.409513473510742,
+                11.419081687927246
+              ],
+              "predicted_reocclusion": 0.5022225975990295,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.425673484802246,
+                11.400392532348633,
+                11.409513473510742,
+                11.419081687927246
+              ],
+              "predicted_reocclusion": 0.5022225975990295,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.425673484802246,
+                11.400392532348633,
+                11.409513473510742,
+                11.419081687927246
+              ],
+              "predicted_reocclusion": 0.5022225975990295,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.425673484802246,
+                11.400392532348633,
+                11.409513473510742,
+                11.419081687927246
+              ],
+              "predicted_reocclusion": 0.5022225975990295,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item_easy/rollout_eval.md

@@ -0,0 +1,14 @@
+# RLBench Rollout Eval
+
+- Checkpoint: `/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt`
+- Plan requested: `True`
+- Plan applied: `True`
+- Support-mode conditioning: `True`
+- Task conditioning: `True`
+- Geometry enabled: `True`
+- World-model mode: `checkpoint_default`
+- Mean success: `0.000`
+
+## Per-task
+
+- `bimanual_handover_item_easy`: mean_success=0.000, returns=[0.0]

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item_easy/stderr.txt

@@ -0,0 +1,4 @@
+/workspace/envs/rlbench/lib/python3.10/site-packages/torch/nn/modules/transformer.py:306: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
+  warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
+qt.qpa.xcb: QXcbConnection: XCB error: 148 (Unknown), sequence: 181, resource id: 0, major code: 140 (Unknown), minor code: 20
+WARNING:root:not sure how _robot_shapes are used is used.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_handover_item_easy/stdout.txt

@@ -0,0 +1,334 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_handover_item_easy": {
+      "task_class": "BimanualHandoverItemEasy",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "bring me the item",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.548981666564941,
+                11.52706241607666,
+                11.533926963806152,
+                11.543230056762695
+              ],
+              "predicted_reocclusion": 0.5040647983551025,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.548981666564941,
+                11.52706241607666,
+                11.533926963806152,
+                11.543230056762695
+              ],
+              "predicted_reocclusion": 0.5040647983551025,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.548981666564941,
+                11.52706241607666,
+                11.533926963806152,
+                11.543230056762695
+              ],
+              "predicted_reocclusion": 0.5040647983551025,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.548981666564941,
+                11.52706241607666,
+                11.533926963806152,
+                11.543230056762695
+              ],
+              "predicted_reocclusion": 0.5040647983551025,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.516067504882812,
+                11.490782737731934,
+                11.499794006347656,
+                11.509395599365234
+              ],
+              "predicted_reocclusion": 0.5022199749946594,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.516067504882812,
+                11.490782737731934,
+                11.499794006347656,
+                11.509395599365234
+              ],
+              "predicted_reocclusion": 0.5022199749946594,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.516067504882812,
+                11.490782737731934,
+                11.499794006347656,
+                11.509395599365234
+              ],
+              "predicted_reocclusion": 0.5022199749946594,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.516067504882812,
+                11.490782737731934,
+                11.499794006347656,
+                11.509395599365234
+              ],
+              "predicted_reocclusion": 0.5022199749946594,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.468103408813477,
+                11.442843437194824,
+                11.45191478729248,
+                11.461535453796387
+              ],
+              "predicted_reocclusion": 0.5022915601730347,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.468103408813477,
+                11.442843437194824,
+                11.45191478729248,
+                11.461535453796387
+              ],
+              "predicted_reocclusion": 0.5022915601730347,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.468103408813477,
+                11.442843437194824,
+                11.45191478729248,
+                11.461535453796387
+              ],
+              "predicted_reocclusion": 0.5022915601730347,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.468103408813477,
+                11.442843437194824,
+                11.45191478729248,
+                11.461535453796387
+              ],
+              "predicted_reocclusion": 0.5022915601730347,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.437320709228516,
+                11.412096977233887,
+                11.421077728271484,
+                11.430691719055176
+              ],
+              "predicted_reocclusion": 0.501986026763916,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.437320709228516,
+                11.412096977233887,
+                11.421077728271484,
+                11.430691719055176
+              ],
+              "predicted_reocclusion": 0.501986026763916,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.437320709228516,
+                11.412096977233887,
+                11.421077728271484,
+                11.430691719055176
+              ],
+              "predicted_reocclusion": 0.501986026763916,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.437320709228516,
+                11.412096977233887,
+                11.421077728271484,
+                11.430691719055176
+              ],
+              "predicted_reocclusion": 0.501986026763916,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.425673484802246,
+                11.400392532348633,
+                11.409513473510742,
+                11.419081687927246
+              ],
+              "predicted_reocclusion": 0.5022225975990295,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.425673484802246,
+                11.400392532348633,
+                11.409513473510742,
+                11.419081687927246
+              ],
+              "predicted_reocclusion": 0.5022225975990295,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.425673484802246,
+                11.400392532348633,
+                11.409513473510742,
+                11.419081687927246
+              ],
+              "predicted_reocclusion": 0.5022225975990295,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.425673484802246,
+                11.400392532348633,
+                11.409513473510742,
+                11.419081687927246
+              ],
+              "predicted_reocclusion": 0.5022225975990295,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}
+[CoppeliaSim:loadinfo]   done.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_ball/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_ball --tasks bimanual_lift_ball --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_ball/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_lift_ball": {
+      "task_class": "BimanualLiftBall",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "Lift the ball",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.887824058532715,
+                10.873709678649902,
+                10.859504699707031,
+                10.897521018981934
+              ],
+              "predicted_reocclusion": 0.4867071211338043,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.887824058532715,
+                10.873709678649902,
+                10.859504699707031,
+                10.897521018981934
+              ],
+              "predicted_reocclusion": 0.4867071211338043,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.887824058532715,
+                10.873709678649902,
+                10.859504699707031,
+                10.897521018981934
+              ],
+              "predicted_reocclusion": 0.4867071211338043,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.887824058532715,
+                10.873709678649902,
+                10.859504699707031,
+                10.897521018981934
+              ],
+              "predicted_reocclusion": 0.4867071211338043,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.758891105651855,
+                10.743366241455078,
+                10.732953071594238,
+                10.769806861877441
+              ],
+              "predicted_reocclusion": 0.4885128140449524,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.758891105651855,
+                10.743366241455078,
+                10.732953071594238,
+                10.769806861877441
+              ],
+              "predicted_reocclusion": 0.4885128140449524,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.758891105651855,
+                10.743366241455078,
+                10.732953071594238,
+                10.769806861877441
+              ],
+              "predicted_reocclusion": 0.4885128140449524,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.758891105651855,
+                10.743366241455078,
+                10.732953071594238,
+                10.769806861877441
+              ],
+              "predicted_reocclusion": 0.4885128140449524,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.701985359191895,
+                10.686473846435547,
+                10.676054000854492,
+                10.712949752807617
+              ],
+              "predicted_reocclusion": 0.4885908365249634,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.701985359191895,
+                10.686473846435547,
+                10.676054000854492,
+                10.712949752807617
+              ],
+              "predicted_reocclusion": 0.4885908365249634,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.701985359191895,
+                10.686473846435547,
+                10.676054000854492,
+                10.712949752807617
+              ],
+              "predicted_reocclusion": 0.4885908365249634,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.701985359191895,
+                10.686473846435547,
+                10.676054000854492,
+                10.712949752807617
+              ],
+              "predicted_reocclusion": 0.4885908365249634,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.665305137634277,
+                10.649796485900879,
+                10.639298439025879,
+                10.67624568939209
+              ],
+              "predicted_reocclusion": 0.48868629336357117,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.665305137634277,
+                10.649796485900879,
+                10.639298439025879,
+                10.67624568939209
+              ],
+              "predicted_reocclusion": 0.48868629336357117,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.665305137634277,
+                10.649796485900879,
+                10.639298439025879,
+                10.67624568939209
+              ],
+              "predicted_reocclusion": 0.48868629336357117,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.665305137634277,
+                10.649796485900879,
+                10.639298439025879,
+                10.67624568939209
+              ],
+              "predicted_reocclusion": 0.48868629336357117,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.65845775604248,
+                10.64293384552002,
+                10.632391929626465,
+                10.669404983520508
+              ],
+              "predicted_reocclusion": 0.4886311888694763,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.65845775604248,
+                10.64293384552002,
+                10.632391929626465,
+                10.669404983520508
+              ],
+              "predicted_reocclusion": 0.4886311888694763,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.65845775604248,
+                10.64293384552002,
+                10.632391929626465,
+                10.669404983520508
+              ],
+              "predicted_reocclusion": 0.4886311888694763,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.65845775604248,
+                10.64293384552002,
+                10.632391929626465,
+                10.669404983520508
+              ],
+              "predicted_reocclusion": 0.4886311888694763,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_ball/rollout_eval.md

@@ -0,0 +1,14 @@
+# RLBench Rollout Eval
+
+- Checkpoint: `/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt`
+- Plan requested: `True`
+- Plan applied: `True`
+- Support-mode conditioning: `True`
+- Task conditioning: `True`
+- Geometry enabled: `True`
+- World-model mode: `checkpoint_default`
+- Mean success: `0.000`
+
+## Per-task
+
+- `bimanual_lift_ball`: mean_success=0.000, returns=[0.0]

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_ball/stderr.txt

@@ -0,0 +1,4 @@
+/workspace/envs/rlbench/lib/python3.10/site-packages/torch/nn/modules/transformer.py:306: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
+  warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
+qt.qpa.xcb: QXcbConnection: XCB error: 148 (Unknown), sequence: 181, resource id: 0, major code: 140 (Unknown), minor code: 20
+WARNING:root:not sure how _robot_shapes are used is used.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_ball/stdout.txt

@@ -0,0 +1,334 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_lift_ball": {
+      "task_class": "BimanualLiftBall",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "Lift the ball",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.887824058532715,
+                10.873709678649902,
+                10.859504699707031,
+                10.897521018981934
+              ],
+              "predicted_reocclusion": 0.4867071211338043,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.887824058532715,
+                10.873709678649902,
+                10.859504699707031,
+                10.897521018981934
+              ],
+              "predicted_reocclusion": 0.4867071211338043,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.887824058532715,
+                10.873709678649902,
+                10.859504699707031,
+                10.897521018981934
+              ],
+              "predicted_reocclusion": 0.4867071211338043,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.887824058532715,
+                10.873709678649902,
+                10.859504699707031,
+                10.897521018981934
+              ],
+              "predicted_reocclusion": 0.4867071211338043,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.758891105651855,
+                10.743366241455078,
+                10.732953071594238,
+                10.769806861877441
+              ],
+              "predicted_reocclusion": 0.4885128140449524,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.758891105651855,
+                10.743366241455078,
+                10.732953071594238,
+                10.769806861877441
+              ],
+              "predicted_reocclusion": 0.4885128140449524,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.758891105651855,
+                10.743366241455078,
+                10.732953071594238,
+                10.769806861877441
+              ],
+              "predicted_reocclusion": 0.4885128140449524,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.758891105651855,
+                10.743366241455078,
+                10.732953071594238,
+                10.769806861877441
+              ],
+              "predicted_reocclusion": 0.4885128140449524,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.701985359191895,
+                10.686473846435547,
+                10.676054000854492,
+                10.712949752807617
+              ],
+              "predicted_reocclusion": 0.4885908365249634,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.701985359191895,
+                10.686473846435547,
+                10.676054000854492,
+                10.712949752807617
+              ],
+              "predicted_reocclusion": 0.4885908365249634,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.701985359191895,
+                10.686473846435547,
+                10.676054000854492,
+                10.712949752807617
+              ],
+              "predicted_reocclusion": 0.4885908365249634,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.701985359191895,
+                10.686473846435547,
+                10.676054000854492,
+                10.712949752807617
+              ],
+              "predicted_reocclusion": 0.4885908365249634,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.665305137634277,
+                10.649796485900879,
+                10.639298439025879,
+                10.67624568939209
+              ],
+              "predicted_reocclusion": 0.48868629336357117,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.665305137634277,
+                10.649796485900879,
+                10.639298439025879,
+                10.67624568939209
+              ],
+              "predicted_reocclusion": 0.48868629336357117,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.665305137634277,
+                10.649796485900879,
+                10.639298439025879,
+                10.67624568939209
+              ],
+              "predicted_reocclusion": 0.48868629336357117,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.665305137634277,
+                10.649796485900879,
+                10.639298439025879,
+                10.67624568939209
+              ],
+              "predicted_reocclusion": 0.48868629336357117,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.65845775604248,
+                10.64293384552002,
+                10.632391929626465,
+                10.669404983520508
+              ],
+              "predicted_reocclusion": 0.4886311888694763,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.65845775604248,
+                10.64293384552002,
+                10.632391929626465,
+                10.669404983520508
+              ],
+              "predicted_reocclusion": 0.4886311888694763,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.65845775604248,
+                10.64293384552002,
+                10.632391929626465,
+                10.669404983520508
+              ],
+              "predicted_reocclusion": 0.4886311888694763,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                10.65845775604248,
+                10.64293384552002,
+                10.632391929626465,
+                10.669404983520508
+              ],
+              "predicted_reocclusion": 0.4886311888694763,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}
+[CoppeliaSim:loadinfo]   done.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_tray/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_tray --tasks bimanual_lift_tray --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_tray/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_lift_tray": {
+      "task_class": "BimanualLiftTray",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "Lift the tray",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.135418891906738,
+                11.09164047241211,
+                11.122170448303223,
+                11.109566688537598
+              ],
+              "predicted_reocclusion": 0.574530839920044,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.135418891906738,
+                11.09164047241211,
+                11.122170448303223,
+                11.109566688537598
+              ],
+              "predicted_reocclusion": 0.574530839920044,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.135418891906738,
+                11.09164047241211,
+                11.122170448303223,
+                11.109566688537598
+              ],
+              "predicted_reocclusion": 0.574530839920044,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.135418891906738,
+                11.09164047241211,
+                11.122170448303223,
+                11.109566688537598
+              ],
+              "predicted_reocclusion": 0.574530839920044,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.017287254333496,
+                10.974169731140137,
+                11.004316329956055,
+                10.99146556854248
+              ],
+              "predicted_reocclusion": 0.5756664276123047,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.017287254333496,
+                10.974169731140137,
+                11.004316329956055,
+                10.99146556854248
+              ],
+              "predicted_reocclusion": 0.5756664276123047,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.017287254333496,
+                10.974169731140137,
+                11.004316329956055,
+                10.99146556854248
+              ],
+              "predicted_reocclusion": 0.5756664276123047,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.017287254333496,
+                10.974169731140137,
+                11.004316329956055,
+                10.99146556854248
+              ],
+              "predicted_reocclusion": 0.5756664276123047,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.96334457397461,
+                10.920077323913574,
+                10.950352668762207,
+                10.937488555908203
+              ],
+              "predicted_reocclusion": 0.5757571458816528,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.96334457397461,
+                10.920077323913574,
+                10.950352668762207,
+                10.937488555908203
+              ],
+              "predicted_reocclusion": 0.5757571458816528,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.96334457397461,
+                10.920077323913574,
+                10.950352668762207,
+                10.937488555908203
+              ],
+              "predicted_reocclusion": 0.5757571458816528,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.96334457397461,
+                10.920077323913574,
+                10.950352668762207,
+                10.937488555908203
+              ],
+              "predicted_reocclusion": 0.5757571458816528,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.923962593078613,
+                10.88067626953125,
+                10.911008834838867,
+                10.898168563842773
+              ],
+              "predicted_reocclusion": 0.5757627487182617,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.923962593078613,
+                10.88067626953125,
+                10.911008834838867,
+                10.898168563842773
+              ],
+              "predicted_reocclusion": 0.5757627487182617,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.923962593078613,
+                10.88067626953125,
+                10.911008834838867,
+                10.898168563842773
+              ],
+              "predicted_reocclusion": 0.5757627487182617,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.923962593078613,
+                10.88067626953125,
+                10.911008834838867,
+                10.898168563842773
+              ],
+              "predicted_reocclusion": 0.5757627487182617,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.910428047180176,
+                10.86708927154541,
+                10.897433280944824,
+                10.88456916809082
+              ],
+              "predicted_reocclusion": 0.5759365558624268,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.910428047180176,
+                10.86708927154541,
+                10.897433280944824,
+                10.88456916809082
+              ],
+              "predicted_reocclusion": 0.5759365558624268,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.910428047180176,
+                10.86708927154541,
+                10.897433280944824,
+                10.88456916809082
+              ],
+              "predicted_reocclusion": 0.5759365558624268,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.910428047180176,
+                10.86708927154541,
+                10.897433280944824,
+                10.88456916809082
+              ],
+              "predicted_reocclusion": 0.5759365558624268,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_tray/rollout_eval.md

@@ -0,0 +1,14 @@
+# RLBench Rollout Eval
+
+- Checkpoint: `/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt`
+- Plan requested: `True`
+- Plan applied: `True`
+- Support-mode conditioning: `True`
+- Task conditioning: `True`
+- Geometry enabled: `True`
+- World-model mode: `checkpoint_default`
+- Mean success: `0.000`
+
+## Per-task
+
+- `bimanual_lift_tray`: mean_success=0.000, returns=[0.0]

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_tray/stderr.txt

@@ -0,0 +1,4 @@
+/workspace/envs/rlbench/lib/python3.10/site-packages/torch/nn/modules/transformer.py:306: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
+  warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
+qt.qpa.xcb: QXcbConnection: XCB error: 148 (Unknown), sequence: 181, resource id: 0, major code: 140 (Unknown), minor code: 20
+WARNING:root:not sure how _robot_shapes are used is used.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_lift_tray/stdout.txt

@@ -0,0 +1,334 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_lift_tray": {
+      "task_class": "BimanualLiftTray",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "Lift the tray",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.135418891906738,
+                11.09164047241211,
+                11.122170448303223,
+                11.109566688537598
+              ],
+              "predicted_reocclusion": 0.574530839920044,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.135418891906738,
+                11.09164047241211,
+                11.122170448303223,
+                11.109566688537598
+              ],
+              "predicted_reocclusion": 0.574530839920044,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.135418891906738,
+                11.09164047241211,
+                11.122170448303223,
+                11.109566688537598
+              ],
+              "predicted_reocclusion": 0.574530839920044,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.135418891906738,
+                11.09164047241211,
+                11.122170448303223,
+                11.109566688537598
+              ],
+              "predicted_reocclusion": 0.574530839920044,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.017287254333496,
+                10.974169731140137,
+                11.004316329956055,
+                10.99146556854248
+              ],
+              "predicted_reocclusion": 0.5756664276123047,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.017287254333496,
+                10.974169731140137,
+                11.004316329956055,
+                10.99146556854248
+              ],
+              "predicted_reocclusion": 0.5756664276123047,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.017287254333496,
+                10.974169731140137,
+                11.004316329956055,
+                10.99146556854248
+              ],
+              "predicted_reocclusion": 0.5756664276123047,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.017287254333496,
+                10.974169731140137,
+                11.004316329956055,
+                10.99146556854248
+              ],
+              "predicted_reocclusion": 0.5756664276123047,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.96334457397461,
+                10.920077323913574,
+                10.950352668762207,
+                10.937488555908203
+              ],
+              "predicted_reocclusion": 0.5757571458816528,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.96334457397461,
+                10.920077323913574,
+                10.950352668762207,
+                10.937488555908203
+              ],
+              "predicted_reocclusion": 0.5757571458816528,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.96334457397461,
+                10.920077323913574,
+                10.950352668762207,
+                10.937488555908203
+              ],
+              "predicted_reocclusion": 0.5757571458816528,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.96334457397461,
+                10.920077323913574,
+                10.950352668762207,
+                10.937488555908203
+              ],
+              "predicted_reocclusion": 0.5757571458816528,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.923962593078613,
+                10.88067626953125,
+                10.911008834838867,
+                10.898168563842773
+              ],
+              "predicted_reocclusion": 0.5757627487182617,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.923962593078613,
+                10.88067626953125,
+                10.911008834838867,
+                10.898168563842773
+              ],
+              "predicted_reocclusion": 0.5757627487182617,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.923962593078613,
+                10.88067626953125,
+                10.911008834838867,
+                10.898168563842773
+              ],
+              "predicted_reocclusion": 0.5757627487182617,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.923962593078613,
+                10.88067626953125,
+                10.911008834838867,
+                10.898168563842773
+              ],
+              "predicted_reocclusion": 0.5757627487182617,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.910428047180176,
+                10.86708927154541,
+                10.897433280944824,
+                10.88456916809082
+              ],
+              "predicted_reocclusion": 0.5759365558624268,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.910428047180176,
+                10.86708927154541,
+                10.897433280944824,
+                10.88456916809082
+              ],
+              "predicted_reocclusion": 0.5759365558624268,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.910428047180176,
+                10.86708927154541,
+                10.897433280944824,
+                10.88456916809082
+              ],
+              "predicted_reocclusion": 0.5759365558624268,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                10.910428047180176,
+                10.86708927154541,
+                10.897433280944824,
+                10.88456916809082
+              ],
+              "predicted_reocclusion": 0.5759365558624268,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}
+[CoppeliaSim:loadinfo]   done.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_laptop/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_laptop --tasks bimanual_pick_laptop --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_laptop/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_pick_laptop": {
+      "task_class": "BimanualPickLaptop",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "pick up the laptop",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.561836242675781,
+                11.543346405029297,
+                11.573431015014648,
+                11.548807144165039
+              ],
+              "predicted_reocclusion": 0.524105429649353,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.561836242675781,
+                11.543346405029297,
+                11.573431015014648,
+                11.548807144165039
+              ],
+              "predicted_reocclusion": 0.524105429649353,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.561836242675781,
+                11.543346405029297,
+                11.573431015014648,
+                11.548807144165039
+              ],
+              "predicted_reocclusion": 0.524105429649353,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.561836242675781,
+                11.543346405029297,
+                11.573431015014648,
+                11.548807144165039
+              ],
+              "predicted_reocclusion": 0.524105429649353,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.497159004211426,
+                11.483813285827637,
+                11.510025978088379,
+                11.486665725708008
+              ],
+              "predicted_reocclusion": 0.5259946584701538,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.497159004211426,
+                11.483813285827637,
+                11.510025978088379,
+                11.486665725708008
+              ],
+              "predicted_reocclusion": 0.5259946584701538,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.497159004211426,
+                11.483813285827637,
+                11.510025978088379,
+                11.486665725708008
+              ],
+              "predicted_reocclusion": 0.5259946584701538,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.497159004211426,
+                11.483813285827637,
+                11.510025978088379,
+                11.486665725708008
+              ],
+              "predicted_reocclusion": 0.5259946584701538,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.45930290222168,
+                11.445694923400879,
+                11.472105979919434,
+                11.448662757873535
+              ],
+              "predicted_reocclusion": 0.5262089371681213,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.45930290222168,
+                11.445694923400879,
+                11.472105979919434,
+                11.448662757873535
+              ],
+              "predicted_reocclusion": 0.5262089371681213,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.45930290222168,
+                11.445694923400879,
+                11.472105979919434,
+                11.448662757873535
+              ],
+              "predicted_reocclusion": 0.5262089371681213,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.45930290222168,
+                11.445694923400879,
+                11.472105979919434,
+                11.448662757873535
+              ],
+              "predicted_reocclusion": 0.5262089371681213,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.430659294128418,
+                11.417139053344727,
+                11.443557739257812,
+                11.420016288757324
+              ],
+              "predicted_reocclusion": 0.5259823203086853,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.430659294128418,
+                11.417139053344727,
+                11.443557739257812,
+                11.420016288757324
+              ],
+              "predicted_reocclusion": 0.5259823203086853,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.430659294128418,
+                11.417139053344727,
+                11.443557739257812,
+                11.420016288757324
+              ],
+              "predicted_reocclusion": 0.5259823203086853,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.430659294128418,
+                11.417139053344727,
+                11.443557739257812,
+                11.420016288757324
+              ],
+              "predicted_reocclusion": 0.5259823203086853,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.418904304504395,
+                11.405465126037598,
+                11.431793212890625,
+                11.40832233428955
+              ],
+              "predicted_reocclusion": 0.5259734392166138,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.418904304504395,
+                11.405465126037598,
+                11.431793212890625,
+                11.40832233428955
+              ],
+              "predicted_reocclusion": 0.5259734392166138,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.418904304504395,
+                11.405465126037598,
+                11.431793212890625,
+                11.40832233428955
+              ],
+              "predicted_reocclusion": 0.5259734392166138,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.418904304504395,
+                11.405465126037598,
+                11.431793212890625,
+                11.40832233428955
+              ],
+              "predicted_reocclusion": 0.5259734392166138,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_laptop/rollout_eval.md

@@ -0,0 +1,14 @@
+# RLBench Rollout Eval
+
+- Checkpoint: `/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt`
+- Plan requested: `True`
+- Plan applied: `True`
+- Support-mode conditioning: `True`
+- Task conditioning: `True`
+- Geometry enabled: `True`
+- World-model mode: `checkpoint_default`
+- Mean success: `0.000`
+
+## Per-task
+
+- `bimanual_pick_laptop`: mean_success=0.000, returns=[0.0]

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_laptop/stderr.txt

@@ -0,0 +1,8 @@
+/workspace/envs/rlbench/lib/python3.10/site-packages/torch/nn/modules/transformer.py:306: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
+  warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
+qt.qpa.xcb: QXcbConnection: XCB error: 148 (Unknown), sequence: 181, resource id: 0, major code: 140 (Unknown), minor code: 20
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint5 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_pick_laptop. Infeasible episode. Can't reach waypoint ((False, 'waypoint5 - DualPanda'), (True, '')).
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint1 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_pick_laptop. Infeasible episode. Can't reach waypoint ((False, 'waypoint1 - DualPanda'), (True, '')).

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_laptop/stdout.txt

@@ -0,0 +1,334 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_pick_laptop": {
+      "task_class": "BimanualPickLaptop",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "pick up the laptop",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.561836242675781,
+                11.543346405029297,
+                11.573431015014648,
+                11.548807144165039
+              ],
+              "predicted_reocclusion": 0.524105429649353,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.561836242675781,
+                11.543346405029297,
+                11.573431015014648,
+                11.548807144165039
+              ],
+              "predicted_reocclusion": 0.524105429649353,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.561836242675781,
+                11.543346405029297,
+                11.573431015014648,
+                11.548807144165039
+              ],
+              "predicted_reocclusion": 0.524105429649353,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.561836242675781,
+                11.543346405029297,
+                11.573431015014648,
+                11.548807144165039
+              ],
+              "predicted_reocclusion": 0.524105429649353,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.497159004211426,
+                11.483813285827637,
+                11.510025978088379,
+                11.486665725708008
+              ],
+              "predicted_reocclusion": 0.5259946584701538,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.497159004211426,
+                11.483813285827637,
+                11.510025978088379,
+                11.486665725708008
+              ],
+              "predicted_reocclusion": 0.5259946584701538,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.497159004211426,
+                11.483813285827637,
+                11.510025978088379,
+                11.486665725708008
+              ],
+              "predicted_reocclusion": 0.5259946584701538,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.497159004211426,
+                11.483813285827637,
+                11.510025978088379,
+                11.486665725708008
+              ],
+              "predicted_reocclusion": 0.5259946584701538,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.45930290222168,
+                11.445694923400879,
+                11.472105979919434,
+                11.448662757873535
+              ],
+              "predicted_reocclusion": 0.5262089371681213,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.45930290222168,
+                11.445694923400879,
+                11.472105979919434,
+                11.448662757873535
+              ],
+              "predicted_reocclusion": 0.5262089371681213,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.45930290222168,
+                11.445694923400879,
+                11.472105979919434,
+                11.448662757873535
+              ],
+              "predicted_reocclusion": 0.5262089371681213,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.45930290222168,
+                11.445694923400879,
+                11.472105979919434,
+                11.448662757873535
+              ],
+              "predicted_reocclusion": 0.5262089371681213,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.430659294128418,
+                11.417139053344727,
+                11.443557739257812,
+                11.420016288757324
+              ],
+              "predicted_reocclusion": 0.5259823203086853,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.430659294128418,
+                11.417139053344727,
+                11.443557739257812,
+                11.420016288757324
+              ],
+              "predicted_reocclusion": 0.5259823203086853,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.430659294128418,
+                11.417139053344727,
+                11.443557739257812,
+                11.420016288757324
+              ],
+              "predicted_reocclusion": 0.5259823203086853,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.430659294128418,
+                11.417139053344727,
+                11.443557739257812,
+                11.420016288757324
+              ],
+              "predicted_reocclusion": 0.5259823203086853,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.418904304504395,
+                11.405465126037598,
+                11.431793212890625,
+                11.40832233428955
+              ],
+              "predicted_reocclusion": 0.5259734392166138,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.418904304504395,
+                11.405465126037598,
+                11.431793212890625,
+                11.40832233428955
+              ],
+              "predicted_reocclusion": 0.5259734392166138,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.418904304504395,
+                11.405465126037598,
+                11.431793212890625,
+                11.40832233428955
+              ],
+              "predicted_reocclusion": 0.5259734392166138,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "slide_occluder",
+              "planner_scores": [
+                11.418904304504395,
+                11.405465126037598,
+                11.431793212890625,
+                11.40832233428955
+              ],
+              "predicted_reocclusion": 0.5259734392166138,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}
+[CoppeliaSim:loadinfo]   done.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_plate/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_plate --tasks bimanual_pick_plate --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_plate/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_pick_plate": {
+      "task_class": "BimanualPickPlate",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "pick up the plate",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.643865585327148,
+                11.657587051391602,
+                11.617194175720215,
+                11.66508674621582
+              ],
+              "predicted_reocclusion": 0.49792808294296265,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.643865585327148,
+                11.657587051391602,
+                11.617194175720215,
+                11.66508674621582
+              ],
+              "predicted_reocclusion": 0.49792808294296265,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.643865585327148,
+                11.657587051391602,
+                11.617194175720215,
+                11.66508674621582
+              ],
+              "predicted_reocclusion": 0.49792808294296265,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.643865585327148,
+                11.657587051391602,
+                11.617194175720215,
+                11.66508674621582
+              ],
+              "predicted_reocclusion": 0.49792808294296265,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.560833930969238,
+                11.572925567626953,
+                11.534008979797363,
+                11.579452514648438
+              ],
+              "predicted_reocclusion": 0.5003998279571533,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.560833930969238,
+                11.572925567626953,
+                11.534008979797363,
+                11.579452514648438
+              ],
+              "predicted_reocclusion": 0.5003998279571533,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.560833930969238,
+                11.572925567626953,
+                11.534008979797363,
+                11.579452514648438
+              ],
+              "predicted_reocclusion": 0.5003998279571533,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.560833930969238,
+                11.572925567626953,
+                11.534008979797363,
+                11.579452514648438
+              ],
+              "predicted_reocclusion": 0.5003998279571533,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.516202926635742,
+                11.528287887573242,
+                11.48935317993164,
+                11.534825325012207
+              ],
+              "predicted_reocclusion": 0.5006614923477173,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.516202926635742,
+                11.528287887573242,
+                11.48935317993164,
+                11.534825325012207
+              ],
+              "predicted_reocclusion": 0.5006614923477173,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.516202926635742,
+                11.528287887573242,
+                11.48935317993164,
+                11.534825325012207
+              ],
+              "predicted_reocclusion": 0.5006614923477173,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.516202926635742,
+                11.528287887573242,
+                11.48935317993164,
+                11.534825325012207
+              ],
+              "predicted_reocclusion": 0.5006614923477173,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.4873628616333,
+                11.499344825744629,
+                11.460457801818848,
+                11.505942344665527
+              ],
+              "predicted_reocclusion": 0.5007470846176147,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.4873628616333,
+                11.499344825744629,
+                11.460457801818848,
+                11.505942344665527
+              ],
+              "predicted_reocclusion": 0.5007470846176147,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.4873628616333,
+                11.499344825744629,
+                11.460457801818848,
+                11.505942344665527
+              ],
+              "predicted_reocclusion": 0.5007470846176147,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.4873628616333,
+                11.499344825744629,
+                11.460457801818848,
+                11.505942344665527
+              ],
+              "predicted_reocclusion": 0.5007470846176147,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.481613159179688,
+                11.493622779846191,
+                11.454720497131348,
+                11.5001802444458
+              ],
+              "predicted_reocclusion": 0.5007752180099487,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.481613159179688,
+                11.493622779846191,
+                11.454720497131348,
+                11.5001802444458
+              ],
+              "predicted_reocclusion": 0.5007752180099487,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.481613159179688,
+                11.493622779846191,
+                11.454720497131348,
+                11.5001802444458
+              ],
+              "predicted_reocclusion": 0.5007752180099487,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.481613159179688,
+                11.493622779846191,
+                11.454720497131348,
+                11.5001802444458
+              ],
+              "predicted_reocclusion": 0.5007752180099487,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_plate/rollout_eval.md

@@ -0,0 +1,14 @@
+# RLBench Rollout Eval
+
+- Checkpoint: `/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt`
+- Plan requested: `True`
+- Plan applied: `True`
+- Support-mode conditioning: `True`
+- Task conditioning: `True`
+- Geometry enabled: `True`
+- World-model mode: `checkpoint_default`
+- Mean success: `0.000`
+
+## Per-task
+
+- `bimanual_pick_plate`: mean_success=0.000, returns=[0.0]

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_plate/stderr.txt

@@ -0,0 +1,10 @@
+/workspace/envs/rlbench/lib/python3.10/site-packages/torch/nn/modules/transformer.py:306: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
+  warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
+qt.qpa.xcb: QXcbConnection: XCB error: 148 (Unknown), sequence: 181, resource id: 0, major code: 140 (Unknown), minor code: 20
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint1 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_pick_plate. Infeasible episode. Can't reach waypoint ((False, 'waypoint1 - DualPanda'), (True, '')).
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint1 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_pick_plate. Infeasible episode. Can't reach waypoint ((False, 'waypoint1 - DualPanda'), (True, '')).
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint1 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_pick_plate. Infeasible episode. Can't reach waypoint ((False, 'waypoint1 - DualPanda'), (True, '')).

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_pick_plate/stdout.txt

@@ -0,0 +1,334 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_pick_plate": {
+      "task_class": "BimanualPickPlate",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "pick up the plate",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.643865585327148,
+                11.657587051391602,
+                11.617194175720215,
+                11.66508674621582
+              ],
+              "predicted_reocclusion": 0.49792808294296265,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.643865585327148,
+                11.657587051391602,
+                11.617194175720215,
+                11.66508674621582
+              ],
+              "predicted_reocclusion": 0.49792808294296265,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.643865585327148,
+                11.657587051391602,
+                11.617194175720215,
+                11.66508674621582
+              ],
+              "predicted_reocclusion": 0.49792808294296265,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.643865585327148,
+                11.657587051391602,
+                11.617194175720215,
+                11.66508674621582
+              ],
+              "predicted_reocclusion": 0.49792808294296265,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.560833930969238,
+                11.572925567626953,
+                11.534008979797363,
+                11.579452514648438
+              ],
+              "predicted_reocclusion": 0.5003998279571533,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.560833930969238,
+                11.572925567626953,
+                11.534008979797363,
+                11.579452514648438
+              ],
+              "predicted_reocclusion": 0.5003998279571533,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.560833930969238,
+                11.572925567626953,
+                11.534008979797363,
+                11.579452514648438
+              ],
+              "predicted_reocclusion": 0.5003998279571533,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.560833930969238,
+                11.572925567626953,
+                11.534008979797363,
+                11.579452514648438
+              ],
+              "predicted_reocclusion": 0.5003998279571533,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.516202926635742,
+                11.528287887573242,
+                11.48935317993164,
+                11.534825325012207
+              ],
+              "predicted_reocclusion": 0.5006614923477173,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.516202926635742,
+                11.528287887573242,
+                11.48935317993164,
+                11.534825325012207
+              ],
+              "predicted_reocclusion": 0.5006614923477173,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.516202926635742,
+                11.528287887573242,
+                11.48935317993164,
+                11.534825325012207
+              ],
+              "predicted_reocclusion": 0.5006614923477173,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.516202926635742,
+                11.528287887573242,
+                11.48935317993164,
+                11.534825325012207
+              ],
+              "predicted_reocclusion": 0.5006614923477173,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.4873628616333,
+                11.499344825744629,
+                11.460457801818848,
+                11.505942344665527
+              ],
+              "predicted_reocclusion": 0.5007470846176147,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.4873628616333,
+                11.499344825744629,
+                11.460457801818848,
+                11.505942344665527
+              ],
+              "predicted_reocclusion": 0.5007470846176147,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.4873628616333,
+                11.499344825744629,
+                11.460457801818848,
+                11.505942344665527
+              ],
+              "predicted_reocclusion": 0.5007470846176147,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.4873628616333,
+                11.499344825744629,
+                11.460457801818848,
+                11.505942344665527
+              ],
+              "predicted_reocclusion": 0.5007470846176147,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.481613159179688,
+                11.493622779846191,
+                11.454720497131348,
+                11.5001802444458
+              ],
+              "predicted_reocclusion": 0.5007752180099487,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.481613159179688,
+                11.493622779846191,
+                11.454720497131348,
+                11.5001802444458
+              ],
+              "predicted_reocclusion": 0.5007752180099487,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.481613159179688,
+                11.493622779846191,
+                11.454720497131348,
+                11.5001802444458
+              ],
+              "predicted_reocclusion": 0.5007752180099487,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "maintain_opening",
+              "planner_scores": [
+                11.481613159179688,
+                11.493622779846191,
+                11.454720497131348,
+                11.5001802444458
+              ],
+              "predicted_reocclusion": 0.5007752180099487,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}
+[CoppeliaSim:loadinfo]   done.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_push_box/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_push_box --tasks bimanual_push_box --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_push_box/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_push_box": {
+      "task_class": "BimanualPushBox",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "push the box to the red area",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.716897964477539,
+                11.697670936584473,
+                11.714740753173828,
+                11.69885540008545
+              ],
+              "predicted_reocclusion": 0.5334509611129761,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.716897964477539,
+                11.697670936584473,
+                11.714740753173828,
+                11.69885540008545
+              ],
+              "predicted_reocclusion": 0.5334509611129761,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.716897964477539,
+                11.697670936584473,
+                11.714740753173828,
+                11.69885540008545
+              ],
+              "predicted_reocclusion": 0.5334509611129761,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.716897964477539,
+                11.697670936584473,
+                11.714740753173828,
+                11.69885540008545
+              ],
+              "predicted_reocclusion": 0.5334509611129761,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.642910957336426,
+                11.622575759887695,
+                11.638394355773926,
+                11.624726295471191
+              ],
+              "predicted_reocclusion": 0.5352821350097656,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.642910957336426,
+                11.622575759887695,
+                11.638394355773926,
+                11.624726295471191
+              ],
+              "predicted_reocclusion": 0.5352821350097656,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.642910957336426,
+                11.622575759887695,
+                11.638394355773926,
+                11.624726295471191
+              ],
+              "predicted_reocclusion": 0.5352821350097656,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.642910957336426,
+                11.622575759887695,
+                11.638394355773926,
+                11.624726295471191
+              ],
+              "predicted_reocclusion": 0.5352821350097656,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.608431816101074,
+                11.588105201721191,
+                11.603989601135254,
+                11.590292930603027
+              ],
+              "predicted_reocclusion": 0.5354688167572021,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.608431816101074,
+                11.588105201721191,
+                11.603989601135254,
+                11.590292930603027
+              ],
+              "predicted_reocclusion": 0.5354688167572021,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.608431816101074,
+                11.588105201721191,
+                11.603989601135254,
+                11.590292930603027
+              ],
+              "predicted_reocclusion": 0.5354688167572021,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.608431816101074,
+                11.588105201721191,
+                11.603989601135254,
+                11.590292930603027
+              ],
+              "predicted_reocclusion": 0.5354688167572021,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.58913516998291,
+                11.568829536437988,
+                11.584684371948242,
+                11.57098388671875
+              ],
+              "predicted_reocclusion": 0.5352646112442017,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.58913516998291,
+                11.568829536437988,
+                11.584684371948242,
+                11.57098388671875
+              ],
+              "predicted_reocclusion": 0.5352646112442017,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.58913516998291,
+                11.568829536437988,
+                11.584684371948242,
+                11.57098388671875
+              ],
+              "predicted_reocclusion": 0.5352646112442017,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.58913516998291,
+                11.568829536437988,
+                11.584684371948242,
+                11.57098388671875
+              ],
+              "predicted_reocclusion": 0.5352646112442017,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.579277992248535,
+                11.558948516845703,
+                11.574824333190918,
+                11.561141014099121
+              ],
+              "predicted_reocclusion": 0.5352468490600586,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.579277992248535,
+                11.558948516845703,
+                11.574824333190918,
+                11.561141014099121
+              ],
+              "predicted_reocclusion": 0.5352468490600586,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.579277992248535,
+                11.558948516845703,
+                11.574824333190918,
+                11.561141014099121
+              ],
+              "predicted_reocclusion": 0.5352468490600586,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.579277992248535,
+                11.558948516845703,
+                11.574824333190918,
+                11.561141014099121
+              ],
+              "predicted_reocclusion": 0.5352468490600586,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_push_box/rollout_eval.md

@@ -0,0 +1,14 @@
+# RLBench Rollout Eval
+
+- Checkpoint: `/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt`
+- Plan requested: `True`
+- Plan applied: `True`
+- Support-mode conditioning: `True`
+- Task conditioning: `True`
+- Geometry enabled: `True`
+- World-model mode: `checkpoint_default`
+- Mean success: `0.000`
+
+## Per-task
+
+- `bimanual_push_box`: mean_success=0.000, returns=[0.0]

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_push_box/stderr.txt

@@ -0,0 +1,4 @@
+/workspace/envs/rlbench/lib/python3.10/site-packages/torch/nn/modules/transformer.py:306: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
+  warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
+qt.qpa.xcb: QXcbConnection: XCB error: 148 (Unknown), sequence: 181, resource id: 0, major code: 140 (Unknown), minor code: 20
+WARNING:root:not sure how _robot_shapes are used is used.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_push_box/stdout.txt

@@ -0,0 +1,334 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_push_box": {
+      "task_class": "BimanualPushBox",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "push the box to the red area",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.716897964477539,
+                11.697670936584473,
+                11.714740753173828,
+                11.69885540008545
+              ],
+              "predicted_reocclusion": 0.5334509611129761,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.716897964477539,
+                11.697670936584473,
+                11.714740753173828,
+                11.69885540008545
+              ],
+              "predicted_reocclusion": 0.5334509611129761,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.716897964477539,
+                11.697670936584473,
+                11.714740753173828,
+                11.69885540008545
+              ],
+              "predicted_reocclusion": 0.5334509611129761,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.716897964477539,
+                11.697670936584473,
+                11.714740753173828,
+                11.69885540008545
+              ],
+              "predicted_reocclusion": 0.5334509611129761,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.642910957336426,
+                11.622575759887695,
+                11.638394355773926,
+                11.624726295471191
+              ],
+              "predicted_reocclusion": 0.5352821350097656,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.642910957336426,
+                11.622575759887695,
+                11.638394355773926,
+                11.624726295471191
+              ],
+              "predicted_reocclusion": 0.5352821350097656,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.642910957336426,
+                11.622575759887695,
+                11.638394355773926,
+                11.624726295471191
+              ],
+              "predicted_reocclusion": 0.5352821350097656,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.642910957336426,
+                11.622575759887695,
+                11.638394355773926,
+                11.624726295471191
+              ],
+              "predicted_reocclusion": 0.5352821350097656,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.608431816101074,
+                11.588105201721191,
+                11.603989601135254,
+                11.590292930603027
+              ],
+              "predicted_reocclusion": 0.5354688167572021,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.608431816101074,
+                11.588105201721191,
+                11.603989601135254,
+                11.590292930603027
+              ],
+              "predicted_reocclusion": 0.5354688167572021,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.608431816101074,
+                11.588105201721191,
+                11.603989601135254,
+                11.590292930603027
+              ],
+              "predicted_reocclusion": 0.5354688167572021,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.608431816101074,
+                11.588105201721191,
+                11.603989601135254,
+                11.590292930603027
+              ],
+              "predicted_reocclusion": 0.5354688167572021,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.58913516998291,
+                11.568829536437988,
+                11.584684371948242,
+                11.57098388671875
+              ],
+              "predicted_reocclusion": 0.5352646112442017,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.58913516998291,
+                11.568829536437988,
+                11.584684371948242,
+                11.57098388671875
+              ],
+              "predicted_reocclusion": 0.5352646112442017,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.58913516998291,
+                11.568829536437988,
+                11.584684371948242,
+                11.57098388671875
+              ],
+              "predicted_reocclusion": 0.5352646112442017,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.58913516998291,
+                11.568829536437988,
+                11.584684371948242,
+                11.57098388671875
+              ],
+              "predicted_reocclusion": 0.5352646112442017,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.579277992248535,
+                11.558948516845703,
+                11.574824333190918,
+                11.561141014099121
+              ],
+              "predicted_reocclusion": 0.5352468490600586,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.579277992248535,
+                11.558948516845703,
+                11.574824333190918,
+                11.561141014099121
+              ],
+              "predicted_reocclusion": 0.5352468490600586,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.579277992248535,
+                11.558948516845703,
+                11.574824333190918,
+                11.561141014099121
+              ],
+              "predicted_reocclusion": 0.5352468490600586,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.579277992248535,
+                11.558948516845703,
+                11.574824333190918,
+                11.561141014099121
+              ],
+              "predicted_reocclusion": 0.5352468490600586,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}
+[CoppeliaSim:loadinfo]   done.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_put_bottle_in_fridge/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_put_bottle_in_fridge --tasks bimanual_put_bottle_in_fridge --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_put_bottle_in_fridge/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_put_bottle_in_fridge": {
+      "task_class": "BimanualPutBottleInFridge",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "put bottle in fridge",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.617284774780273,
+                11.603991508483887,
+                11.592961311340332,
+                11.604387283325195
+              ],
+              "predicted_reocclusion": 0.46648383140563965,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.617284774780273,
+                11.603991508483887,
+                11.592961311340332,
+                11.604387283325195
+              ],
+              "predicted_reocclusion": 0.46648383140563965,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.617284774780273,
+                11.603991508483887,
+                11.592961311340332,
+                11.604387283325195
+              ],
+              "predicted_reocclusion": 0.46648383140563965,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.617284774780273,
+                11.603991508483887,
+                11.592961311340332,
+                11.604387283325195
+              ],
+              "predicted_reocclusion": 0.46648383140563965,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.504376411437988,
+                11.491621971130371,
+                11.483019828796387,
+                11.493010520935059
+              ],
+              "predicted_reocclusion": 0.47061675786972046,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.504376411437988,
+                11.491621971130371,
+                11.483019828796387,
+                11.493010520935059
+              ],
+              "predicted_reocclusion": 0.47061675786972046,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.504376411437988,
+                11.491621971130371,
+                11.483019828796387,
+                11.493010520935059
+              ],
+              "predicted_reocclusion": 0.47061675786972046,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.504376411437988,
+                11.491621971130371,
+                11.483019828796387,
+                11.493010520935059
+              ],
+              "predicted_reocclusion": 0.47061675786972046,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.45065975189209,
+                11.437807083129883,
+                11.42929744720459,
+                11.439268112182617
+              ],
+              "predicted_reocclusion": 0.47058993577957153,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.45065975189209,
+                11.437807083129883,
+                11.42929744720459,
+                11.439268112182617
+              ],
+              "predicted_reocclusion": 0.47058993577957153,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.45065975189209,
+                11.437807083129883,
+                11.42929744720459,
+                11.439268112182617
+              ],
+              "predicted_reocclusion": 0.47058993577957153,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.45065975189209,
+                11.437807083129883,
+                11.42929744720459,
+                11.439268112182617
+              ],
+              "predicted_reocclusion": 0.47058993577957153,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.416655540466309,
+                11.403790473937988,
+                11.39507007598877,
+                11.405183792114258
+              ],
+              "predicted_reocclusion": 0.47055739164352417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.416655540466309,
+                11.403790473937988,
+                11.39507007598877,
+                11.405183792114258
+              ],
+              "predicted_reocclusion": 0.47055739164352417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.416655540466309,
+                11.403790473937988,
+                11.39507007598877,
+                11.405183792114258
+              ],
+              "predicted_reocclusion": 0.47055739164352417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.416655540466309,
+                11.403790473937988,
+                11.39507007598877,
+                11.405183792114258
+              ],
+              "predicted_reocclusion": 0.47055739164352417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.40277099609375,
+                11.389852523803711,
+                11.381258010864258,
+                11.391311645507812
+              ],
+              "predicted_reocclusion": 0.4707050621509552,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.40277099609375,
+                11.389852523803711,
+                11.381258010864258,
+                11.391311645507812
+              ],
+              "predicted_reocclusion": 0.4707050621509552,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.40277099609375,
+                11.389852523803711,
+                11.381258010864258,
+                11.391311645507812
+              ],
+              "predicted_reocclusion": 0.4707050621509552,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.40277099609375,
+                11.389852523803711,
+                11.381258010864258,
+                11.391311645507812
+              ],
+              "predicted_reocclusion": 0.4707050621509552,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_put_bottle_in_fridge/rollout_eval.md

@@ -0,0 +1,14 @@
+# RLBench Rollout Eval
+
+- Checkpoint: `/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt`
+- Plan requested: `True`
+- Plan applied: `True`
+- Support-mode conditioning: `True`
+- Task conditioning: `True`
+- Geometry enabled: `True`
+- World-model mode: `checkpoint_default`
+- Mean success: `0.000`
+
+## Per-task
+
+- `bimanual_put_bottle_in_fridge`: mean_success=0.000, returns=[0.0]

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_put_bottle_in_fridge/stderr.txt

@@ -0,0 +1,4 @@
+/workspace/envs/rlbench/lib/python3.10/site-packages/torch/nn/modules/transformer.py:306: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True
+  warnings.warn(f"enable_nested_tensor is True, but self.use_nested_tensor is False because {why_not_sparsity_fast_path}")
+qt.qpa.xcb: QXcbConnection: XCB error: 148 (Unknown), sequence: 181, resource id: 0, major code: 140 (Unknown), minor code: 20
+WARNING:root:not sure how _robot_shapes are used is used.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_put_bottle_in_fridge/stdout.txt

@@ -0,0 +1,335 @@
+[ 0.05       -0.27500001  0.75199997]
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_put_bottle_in_fridge": {
+      "task_class": "BimanualPutBottleInFridge",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "put bottle in fridge",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.617284774780273,
+                11.603991508483887,
+                11.592961311340332,
+                11.604387283325195
+              ],
+              "predicted_reocclusion": 0.46648383140563965,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.617284774780273,
+                11.603991508483887,
+                11.592961311340332,
+                11.604387283325195
+              ],
+              "predicted_reocclusion": 0.46648383140563965,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.617284774780273,
+                11.603991508483887,
+                11.592961311340332,
+                11.604387283325195
+              ],
+              "predicted_reocclusion": 0.46648383140563965,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.617284774780273,
+                11.603991508483887,
+                11.592961311340332,
+                11.604387283325195
+              ],
+              "predicted_reocclusion": 0.46648383140563965,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.504376411437988,
+                11.491621971130371,
+                11.483019828796387,
+                11.493010520935059
+              ],
+              "predicted_reocclusion": 0.47061675786972046,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.504376411437988,
+                11.491621971130371,
+                11.483019828796387,
+                11.493010520935059
+              ],
+              "predicted_reocclusion": 0.47061675786972046,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.504376411437988,
+                11.491621971130371,
+                11.483019828796387,
+                11.493010520935059
+              ],
+              "predicted_reocclusion": 0.47061675786972046,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.504376411437988,
+                11.491621971130371,
+                11.483019828796387,
+                11.493010520935059
+              ],
+              "predicted_reocclusion": 0.47061675786972046,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.45065975189209,
+                11.437807083129883,
+                11.42929744720459,
+                11.439268112182617
+              ],
+              "predicted_reocclusion": 0.47058993577957153,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.45065975189209,
+                11.437807083129883,
+                11.42929744720459,
+                11.439268112182617
+              ],
+              "predicted_reocclusion": 0.47058993577957153,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.45065975189209,
+                11.437807083129883,
+                11.42929744720459,
+                11.439268112182617
+              ],
+              "predicted_reocclusion": 0.47058993577957153,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.45065975189209,
+                11.437807083129883,
+                11.42929744720459,
+                11.439268112182617
+              ],
+              "predicted_reocclusion": 0.47058993577957153,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.416655540466309,
+                11.403790473937988,
+                11.39507007598877,
+                11.405183792114258
+              ],
+              "predicted_reocclusion": 0.47055739164352417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.416655540466309,
+                11.403790473937988,
+                11.39507007598877,
+                11.405183792114258
+              ],
+              "predicted_reocclusion": 0.47055739164352417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.416655540466309,
+                11.403790473937988,
+                11.39507007598877,
+                11.405183792114258
+              ],
+              "predicted_reocclusion": 0.47055739164352417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.416655540466309,
+                11.403790473937988,
+                11.39507007598877,
+                11.405183792114258
+              ],
+              "predicted_reocclusion": 0.47055739164352417,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.40277099609375,
+                11.389852523803711,
+                11.381258010864258,
+                11.391311645507812
+              ],
+              "predicted_reocclusion": 0.4707050621509552,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.40277099609375,
+                11.389852523803711,
+                11.381258010864258,
+                11.391311645507812
+              ],
+              "predicted_reocclusion": 0.4707050621509552,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.40277099609375,
+                11.389852523803711,
+                11.381258010864258,
+                11.391311645507812
+              ],
+              "predicted_reocclusion": 0.4707050621509552,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.40277099609375,
+                11.389852523803711,
+                11.381258010864258,
+                11.391311645507812
+              ],
+              "predicted_reocclusion": 0.4707050621509552,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}
+[CoppeliaSim:loadinfo]   done.

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_put_item_in_drawer/command.txt

@@ -0,0 +1 @@
+/workspace/envs/rlbench/bin/python -m eval.run_rlbench_rollout_eval --checkpoint /workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt --output-dir /workspace/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_put_item_in_drawer --tasks bimanual_put_item_in_drawer --episodes-per-task 1 --episode-length 20 --resolution 224 --device cuda --chunk-commit-steps 4 --headless --plan --allow-unsupervised-planning

artifacts/reports/peract2_baseline_ep1/baseline_rgbd_seed17_plan_split/bimanual_put_item_in_drawer/rollout_eval.json

@@ -0,0 +1,333 @@
+{
+  "checkpoint": "/workspace/VLAarchtests/artifacts/outputs/r3d/proxy_interaction_r3d_stage3_clip_rgbd_seed17/checkpoint_best.pt",
+  "plan_requested": true,
+  "plan_applied": true,
+  "planner_mode": "trainable",
+  "support_mode_conditioning": true,
+  "task_conditioning": true,
+  "geometry_enabled": true,
+  "world_model_mode": "checkpoint_default",
+  "episodes_per_task": 1,
+  "episode_length": 20,
+  "resolution": 224,
+  "reset_retries": 20,
+  "cameras": [
+    "front",
+    "wrist_left",
+    "wrist_right"
+  ],
+  "tasks": {
+    "bimanual_put_item_in_drawer": {
+      "task_class": "BimanualPutItemInDrawer",
+      "successes": [
+        0.0
+      ],
+      "returns": [
+        0.0
+      ],
+      "path_recoveries": [
+        0
+      ],
+      "noop_fallbacks": [
+        0
+      ],
+      "reset_retries": [
+        0
+      ],
+      "episode_traces": [
+        {
+          "language_goal": "put the item in the bottom drawer",
+          "steps": [
+            {
+              "timestep": 0,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.668746948242188,
+                11.627811431884766,
+                11.656004905700684,
+                11.657785415649414
+              ],
+              "predicted_reocclusion": 0.493838369846344,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 1,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.668746948242188,
+                11.627811431884766,
+                11.656004905700684,
+                11.657785415649414
+              ],
+              "predicted_reocclusion": 0.493838369846344,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 2,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.668746948242188,
+                11.627811431884766,
+                11.656004905700684,
+                11.657785415649414
+              ],
+              "predicted_reocclusion": 0.493838369846344,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 3,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.668746948242188,
+                11.627811431884766,
+                11.656004905700684,
+                11.657785415649414
+              ],
+              "predicted_reocclusion": 0.493838369846344,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 4,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.599308013916016,
+                11.5580415725708,
+                11.584209442138672,
+                11.587956428527832
+              ],
+              "predicted_reocclusion": 0.4928761422634125,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 5,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.599308013916016,
+                11.5580415725708,
+                11.584209442138672,
+                11.587956428527832
+              ],
+              "predicted_reocclusion": 0.4928761422634125,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 6,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.599308013916016,
+                11.5580415725708,
+                11.584209442138672,
+                11.587956428527832
+              ],
+              "predicted_reocclusion": 0.4928761422634125,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 7,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.599308013916016,
+                11.5580415725708,
+                11.584209442138672,
+                11.587956428527832
+              ],
+              "predicted_reocclusion": 0.4928761422634125,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 8,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.564525604248047,
+                11.523283958435059,
+                11.549537658691406,
+                11.55319595336914
+              ],
+              "predicted_reocclusion": 0.4930340647697449,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 9,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.564525604248047,
+                11.523283958435059,
+                11.549537658691406,
+                11.55319595336914
+              ],
+              "predicted_reocclusion": 0.4930340647697449,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 10,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.564525604248047,
+                11.523283958435059,
+                11.549537658691406,
+                11.55319595336914
+              ],
+              "predicted_reocclusion": 0.4930340647697449,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 11,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.564525604248047,
+                11.523283958435059,
+                11.549537658691406,
+                11.55319595336914
+              ],
+              "predicted_reocclusion": 0.4930340647697449,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 12,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.54377555847168,
+                11.502500534057617,
+                11.528753280639648,
+                11.532443046569824
+              ],
+              "predicted_reocclusion": 0.49306720495224,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 13,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.54377555847168,
+                11.502500534057617,
+                11.528753280639648,
+                11.532443046569824
+              ],
+              "predicted_reocclusion": 0.49306720495224,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 14,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.54377555847168,
+                11.502500534057617,
+                11.528753280639648,
+                11.532443046569824
+              ],
+              "predicted_reocclusion": 0.49306720495224,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 15,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.54377555847168,
+                11.502500534057617,
+                11.528753280639648,
+                11.532443046569824
+              ],
+              "predicted_reocclusion": 0.49306720495224,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 16,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.537642478942871,
+                11.496336936950684,
+                11.522603034973145,
+                11.526345252990723
+              ],
+              "predicted_reocclusion": 0.4930623769760132,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 17,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.537642478942871,
+                11.496336936950684,
+                11.522603034973145,
+                11.526345252990723
+              ],
+              "predicted_reocclusion": 0.4930623769760132,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 18,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.537642478942871,
+                11.496336936950684,
+                11.522603034973145,
+                11.526345252990723
+              ],
+              "predicted_reocclusion": 0.4930623769760132,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            },
+            {
+              "timestep": 19,
+              "chosen_macro_mode": "widen_opening",
+              "planner_scores": [
+                11.537642478942871,
+                11.496336936950684,
+                11.522603034973145,
+                11.526345252990723
+              ],
+              "predicted_reocclusion": 0.4930623769760132,
+              "support_mode_conditioning": true,
+              "path_recoveries": 0,
+              "noop_fallbacks": 0
+            }
+          ],
+          "success": 0.0,
+          "return": 0.0,
+          "path_recoveries": 0,
+          "noop_fallbacks": 0
+        }
+      ],
+      "mean_success": 0.0,
+      "mean_return": 0.0
+    }
+  },
+  "mean_success": 0.0
+}