Add files using upload-large-folder tool
Browse files- code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_current.yaml +92 -0
- code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_current_3train.yaml +92 -0
- code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_current_4train.yaml +92 -0
- code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_current_wide.yaml +94 -0
- code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_step1.yaml +95 -0
- code/reveal_vla_bimanual/train/configs/rlbench_push_box_backbone_only_clip_step1.yaml +95 -0
- code/reveal_vla_bimanual/train/configs/rlbench_subset3_backbone_only_clip_current.yaml +94 -0
- code/reveal_vla_bimanual/train/configs/rlbench_subset3_backbone_only_clip_current_common23.yaml +94 -0
- code/reveal_vla_bimanual/train/configs/rlbench_subset3_backbone_only_clip_current_valid9.yaml +94 -0
- code/reveal_vla_bimanual/train/losses.py +5 -1
- code/reveal_vla_bimanual/train/run_rlbench_experiment.py +2 -0
- environment/README.md +7 -0
code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_current.yaml
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
experiment_name: rlbench_lift_ball_backbone_only_clip_current
|
| 2 |
+
output_dir: /workspace/outputs/rlbench_current
|
| 3 |
+
device: cuda
|
| 4 |
+
seed: 17
|
| 5 |
+
init_checkpoint: null
|
| 6 |
+
init_strict: false
|
| 7 |
+
|
| 8 |
+
data:
|
| 9 |
+
dataset_root: /workspace/data/rlbench2
|
| 10 |
+
tasks: [bimanual_lift_ball]
|
| 11 |
+
train_episodes: [0]
|
| 12 |
+
val_episodes: [1]
|
| 13 |
+
resolution: 224
|
| 14 |
+
chunk_horizon: 8
|
| 15 |
+
proprio_dim: 32
|
| 16 |
+
history_steps: 2
|
| 17 |
+
|
| 18 |
+
optim:
|
| 19 |
+
epochs: 10
|
| 20 |
+
batch_size: 8
|
| 21 |
+
num_workers: 0
|
| 22 |
+
lr: 0.0002
|
| 23 |
+
weight_decay: 0.0001
|
| 24 |
+
|
| 25 |
+
trainer:
|
| 26 |
+
policy_type: backbone_only
|
| 27 |
+
use_bf16: true
|
| 28 |
+
grad_clip_norm: 1.0
|
| 29 |
+
freeze_backbone: true
|
| 30 |
+
gradient_checkpointing: false
|
| 31 |
+
plan_during_train: false
|
| 32 |
+
plan_during_eval: false
|
| 33 |
+
support_mode_conditioning: true
|
| 34 |
+
|
| 35 |
+
policy:
|
| 36 |
+
backbone:
|
| 37 |
+
model_name: openai/clip-vit-base-patch32
|
| 38 |
+
hidden_dim: 512
|
| 39 |
+
max_text_tokens: 32
|
| 40 |
+
freeze_backbone: true
|
| 41 |
+
gradient_checkpointing: false
|
| 42 |
+
use_dummy_backbone: false
|
| 43 |
+
fusion:
|
| 44 |
+
hidden_dim: 512
|
| 45 |
+
num_cameras: 3
|
| 46 |
+
num_layers: 4
|
| 47 |
+
num_heads: 8
|
| 48 |
+
ff_dim: 2048
|
| 49 |
+
dropout: 0.1
|
| 50 |
+
proprio_dim: 32
|
| 51 |
+
proprio_tokens: 1
|
| 52 |
+
memory:
|
| 53 |
+
hidden_dim: 512
|
| 54 |
+
history_steps: 2
|
| 55 |
+
num_layers: 1
|
| 56 |
+
dropout: 0.1
|
| 57 |
+
decoder:
|
| 58 |
+
hidden_dim: 512
|
| 59 |
+
num_heads: 8
|
| 60 |
+
num_layers: 4
|
| 61 |
+
ff_dim: 2048
|
| 62 |
+
dropout: 0.1
|
| 63 |
+
chunk_size: 8
|
| 64 |
+
action_dim: 14
|
| 65 |
+
num_candidates: 8
|
| 66 |
+
reveal_head:
|
| 67 |
+
hidden_dim: 512
|
| 68 |
+
num_support_modes: 3
|
| 69 |
+
num_approach_templates: 32
|
| 70 |
+
rollout_horizon: 5
|
| 71 |
+
belief_map_size: 32
|
| 72 |
+
predict_belief_map: true
|
| 73 |
+
world_model:
|
| 74 |
+
hidden_dim: 512
|
| 75 |
+
action_dim: 14
|
| 76 |
+
num_support_modes: 3
|
| 77 |
+
num_approach_templates: 32
|
| 78 |
+
rollout_horizon: 5
|
| 79 |
+
planner:
|
| 80 |
+
hidden_dim: 512
|
| 81 |
+
num_candidates: 8
|
| 82 |
+
action_dim: 14
|
| 83 |
+
utility_margin: 0.1
|
| 84 |
+
|
| 85 |
+
loss_weights:
|
| 86 |
+
action: 1.0
|
| 87 |
+
support_mode: 0.0
|
| 88 |
+
corridor: 0.0
|
| 89 |
+
persistence: 0.0
|
| 90 |
+
disturbance: 0.0
|
| 91 |
+
world_model: 0.0
|
| 92 |
+
belief: 0.0
|
code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_current_3train.yaml
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
experiment_name: rlbench_lift_ball_backbone_only_clip_current_3train
|
| 2 |
+
output_dir: /workspace/outputs/rlbench_current
|
| 3 |
+
device: cuda
|
| 4 |
+
seed: 17
|
| 5 |
+
init_checkpoint: null
|
| 6 |
+
init_strict: false
|
| 7 |
+
|
| 8 |
+
data:
|
| 9 |
+
dataset_root: /workspace/data/rlbench2
|
| 10 |
+
tasks: [bimanual_lift_ball]
|
| 11 |
+
train_episodes: [0, 1, 2]
|
| 12 |
+
val_episodes: [3]
|
| 13 |
+
resolution: 224
|
| 14 |
+
chunk_horizon: 8
|
| 15 |
+
proprio_dim: 32
|
| 16 |
+
history_steps: 2
|
| 17 |
+
|
| 18 |
+
optim:
|
| 19 |
+
epochs: 12
|
| 20 |
+
batch_size: 8
|
| 21 |
+
num_workers: 2
|
| 22 |
+
lr: 0.0002
|
| 23 |
+
weight_decay: 0.0001
|
| 24 |
+
|
| 25 |
+
trainer:
|
| 26 |
+
policy_type: backbone_only
|
| 27 |
+
use_bf16: true
|
| 28 |
+
grad_clip_norm: 1.0
|
| 29 |
+
freeze_backbone: true
|
| 30 |
+
gradient_checkpointing: false
|
| 31 |
+
plan_during_train: false
|
| 32 |
+
plan_during_eval: false
|
| 33 |
+
support_mode_conditioning: true
|
| 34 |
+
|
| 35 |
+
policy:
|
| 36 |
+
backbone:
|
| 37 |
+
model_name: openai/clip-vit-base-patch32
|
| 38 |
+
hidden_dim: 512
|
| 39 |
+
max_text_tokens: 32
|
| 40 |
+
freeze_backbone: true
|
| 41 |
+
gradient_checkpointing: false
|
| 42 |
+
use_dummy_backbone: false
|
| 43 |
+
fusion:
|
| 44 |
+
hidden_dim: 512
|
| 45 |
+
num_cameras: 3
|
| 46 |
+
num_layers: 4
|
| 47 |
+
num_heads: 8
|
| 48 |
+
ff_dim: 2048
|
| 49 |
+
dropout: 0.1
|
| 50 |
+
proprio_dim: 32
|
| 51 |
+
proprio_tokens: 1
|
| 52 |
+
memory:
|
| 53 |
+
hidden_dim: 512
|
| 54 |
+
history_steps: 2
|
| 55 |
+
num_layers: 1
|
| 56 |
+
dropout: 0.1
|
| 57 |
+
decoder:
|
| 58 |
+
hidden_dim: 512
|
| 59 |
+
num_heads: 8
|
| 60 |
+
num_layers: 4
|
| 61 |
+
ff_dim: 2048
|
| 62 |
+
dropout: 0.1
|
| 63 |
+
chunk_size: 8
|
| 64 |
+
action_dim: 14
|
| 65 |
+
num_candidates: 8
|
| 66 |
+
reveal_head:
|
| 67 |
+
hidden_dim: 512
|
| 68 |
+
num_support_modes: 3
|
| 69 |
+
num_approach_templates: 32
|
| 70 |
+
rollout_horizon: 5
|
| 71 |
+
belief_map_size: 32
|
| 72 |
+
predict_belief_map: true
|
| 73 |
+
world_model:
|
| 74 |
+
hidden_dim: 512
|
| 75 |
+
action_dim: 14
|
| 76 |
+
num_support_modes: 3
|
| 77 |
+
num_approach_templates: 32
|
| 78 |
+
rollout_horizon: 5
|
| 79 |
+
planner:
|
| 80 |
+
hidden_dim: 512
|
| 81 |
+
num_candidates: 8
|
| 82 |
+
action_dim: 14
|
| 83 |
+
utility_margin: 0.1
|
| 84 |
+
|
| 85 |
+
loss_weights:
|
| 86 |
+
action: 1.0
|
| 87 |
+
support_mode: 0.0
|
| 88 |
+
corridor: 0.0
|
| 89 |
+
persistence: 0.0
|
| 90 |
+
disturbance: 0.0
|
| 91 |
+
world_model: 0.0
|
| 92 |
+
belief: 0.0
|
code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_current_4train.yaml
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
experiment_name: rlbench_lift_ball_backbone_only_clip_current_4train
|
| 2 |
+
output_dir: /workspace/outputs/rlbench_current
|
| 3 |
+
device: cuda
|
| 4 |
+
seed: 17
|
| 5 |
+
init_checkpoint: null
|
| 6 |
+
init_strict: false
|
| 7 |
+
|
| 8 |
+
data:
|
| 9 |
+
dataset_root: /workspace/data/rlbench2
|
| 10 |
+
tasks: [bimanual_lift_ball]
|
| 11 |
+
train_episodes: [0, 1, 2, 3]
|
| 12 |
+
val_episodes: [4]
|
| 13 |
+
resolution: 224
|
| 14 |
+
chunk_horizon: 8
|
| 15 |
+
proprio_dim: 32
|
| 16 |
+
history_steps: 2
|
| 17 |
+
|
| 18 |
+
optim:
|
| 19 |
+
epochs: 10
|
| 20 |
+
batch_size: 8
|
| 21 |
+
num_workers: 2
|
| 22 |
+
lr: 0.0002
|
| 23 |
+
weight_decay: 0.0001
|
| 24 |
+
|
| 25 |
+
trainer:
|
| 26 |
+
policy_type: backbone_only
|
| 27 |
+
use_bf16: true
|
| 28 |
+
grad_clip_norm: 1.0
|
| 29 |
+
freeze_backbone: true
|
| 30 |
+
gradient_checkpointing: false
|
| 31 |
+
plan_during_train: false
|
| 32 |
+
plan_during_eval: false
|
| 33 |
+
support_mode_conditioning: true
|
| 34 |
+
|
| 35 |
+
policy:
|
| 36 |
+
backbone:
|
| 37 |
+
model_name: openai/clip-vit-base-patch32
|
| 38 |
+
hidden_dim: 512
|
| 39 |
+
max_text_tokens: 32
|
| 40 |
+
freeze_backbone: true
|
| 41 |
+
gradient_checkpointing: false
|
| 42 |
+
use_dummy_backbone: false
|
| 43 |
+
fusion:
|
| 44 |
+
hidden_dim: 512
|
| 45 |
+
num_cameras: 3
|
| 46 |
+
num_layers: 4
|
| 47 |
+
num_heads: 8
|
| 48 |
+
ff_dim: 2048
|
| 49 |
+
dropout: 0.1
|
| 50 |
+
proprio_dim: 32
|
| 51 |
+
proprio_tokens: 1
|
| 52 |
+
memory:
|
| 53 |
+
hidden_dim: 512
|
| 54 |
+
history_steps: 2
|
| 55 |
+
num_layers: 1
|
| 56 |
+
dropout: 0.1
|
| 57 |
+
decoder:
|
| 58 |
+
hidden_dim: 512
|
| 59 |
+
num_heads: 8
|
| 60 |
+
num_layers: 4
|
| 61 |
+
ff_dim: 2048
|
| 62 |
+
dropout: 0.1
|
| 63 |
+
chunk_size: 8
|
| 64 |
+
action_dim: 14
|
| 65 |
+
num_candidates: 8
|
| 66 |
+
reveal_head:
|
| 67 |
+
hidden_dim: 512
|
| 68 |
+
num_support_modes: 3
|
| 69 |
+
num_approach_templates: 32
|
| 70 |
+
rollout_horizon: 5
|
| 71 |
+
belief_map_size: 32
|
| 72 |
+
predict_belief_map: true
|
| 73 |
+
world_model:
|
| 74 |
+
hidden_dim: 512
|
| 75 |
+
action_dim: 14
|
| 76 |
+
num_support_modes: 3
|
| 77 |
+
num_approach_templates: 32
|
| 78 |
+
rollout_horizon: 5
|
| 79 |
+
planner:
|
| 80 |
+
hidden_dim: 512
|
| 81 |
+
num_candidates: 8
|
| 82 |
+
action_dim: 14
|
| 83 |
+
utility_margin: 0.1
|
| 84 |
+
|
| 85 |
+
loss_weights:
|
| 86 |
+
action: 1.0
|
| 87 |
+
support_mode: 0.0
|
| 88 |
+
corridor: 0.0
|
| 89 |
+
persistence: 0.0
|
| 90 |
+
disturbance: 0.0
|
| 91 |
+
world_model: 0.0
|
| 92 |
+
belief: 0.0
|
code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_current_wide.yaml
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
experiment_name: rlbench_lift_ball_backbone_only_clip_current_wide
|
| 2 |
+
output_dir: /workspace/outputs/rlbench_current
|
| 3 |
+
device: cuda
|
| 4 |
+
seed: 17
|
| 5 |
+
init_checkpoint: /workspace/outputs/rlbench_current/rlbench_lift_ball_backbone_only_clip_current_3train/checkpoint_best.pt
|
| 6 |
+
init_strict: true
|
| 7 |
+
|
| 8 |
+
data:
|
| 9 |
+
dataset_root: /workspace/data/rlbench2
|
| 10 |
+
tasks: [bimanual_lift_ball]
|
| 11 |
+
train_episodes: [0, 1, 2, 3, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
| 12 |
+
val_episodes: [32, 33, 34, 35, 36, 37, 38, 39]
|
| 13 |
+
resolution: 224
|
| 14 |
+
chunk_horizon: 8
|
| 15 |
+
proprio_dim: 32
|
| 16 |
+
history_steps: 2
|
| 17 |
+
max_train_samples: 4096
|
| 18 |
+
max_val_samples: 2048
|
| 19 |
+
|
| 20 |
+
optim:
|
| 21 |
+
epochs: 8
|
| 22 |
+
batch_size: 16
|
| 23 |
+
num_workers: 4
|
| 24 |
+
lr: 0.0001
|
| 25 |
+
weight_decay: 0.0001
|
| 26 |
+
|
| 27 |
+
trainer:
|
| 28 |
+
policy_type: backbone_only
|
| 29 |
+
use_bf16: true
|
| 30 |
+
grad_clip_norm: 1.0
|
| 31 |
+
freeze_backbone: true
|
| 32 |
+
gradient_checkpointing: false
|
| 33 |
+
plan_during_train: false
|
| 34 |
+
plan_during_eval: false
|
| 35 |
+
support_mode_conditioning: true
|
| 36 |
+
|
| 37 |
+
policy:
|
| 38 |
+
backbone:
|
| 39 |
+
model_name: openai/clip-vit-base-patch32
|
| 40 |
+
hidden_dim: 512
|
| 41 |
+
max_text_tokens: 32
|
| 42 |
+
freeze_backbone: true
|
| 43 |
+
gradient_checkpointing: false
|
| 44 |
+
use_dummy_backbone: false
|
| 45 |
+
fusion:
|
| 46 |
+
hidden_dim: 512
|
| 47 |
+
num_cameras: 3
|
| 48 |
+
num_layers: 4
|
| 49 |
+
num_heads: 8
|
| 50 |
+
ff_dim: 2048
|
| 51 |
+
dropout: 0.1
|
| 52 |
+
proprio_dim: 32
|
| 53 |
+
proprio_tokens: 1
|
| 54 |
+
memory:
|
| 55 |
+
hidden_dim: 512
|
| 56 |
+
history_steps: 2
|
| 57 |
+
num_layers: 1
|
| 58 |
+
dropout: 0.1
|
| 59 |
+
decoder:
|
| 60 |
+
hidden_dim: 512
|
| 61 |
+
num_heads: 8
|
| 62 |
+
num_layers: 4
|
| 63 |
+
ff_dim: 2048
|
| 64 |
+
dropout: 0.1
|
| 65 |
+
chunk_size: 8
|
| 66 |
+
action_dim: 14
|
| 67 |
+
num_candidates: 8
|
| 68 |
+
reveal_head:
|
| 69 |
+
hidden_dim: 512
|
| 70 |
+
num_support_modes: 3
|
| 71 |
+
num_approach_templates: 32
|
| 72 |
+
rollout_horizon: 5
|
| 73 |
+
belief_map_size: 32
|
| 74 |
+
predict_belief_map: true
|
| 75 |
+
world_model:
|
| 76 |
+
hidden_dim: 512
|
| 77 |
+
action_dim: 14
|
| 78 |
+
num_support_modes: 3
|
| 79 |
+
num_approach_templates: 32
|
| 80 |
+
rollout_horizon: 5
|
| 81 |
+
planner:
|
| 82 |
+
hidden_dim: 512
|
| 83 |
+
num_candidates: 8
|
| 84 |
+
action_dim: 14
|
| 85 |
+
utility_margin: 0.1
|
| 86 |
+
|
| 87 |
+
loss_weights:
|
| 88 |
+
action: 1.0
|
| 89 |
+
support_mode: 0.0
|
| 90 |
+
corridor: 0.0
|
| 91 |
+
persistence: 0.0
|
| 92 |
+
disturbance: 0.0
|
| 93 |
+
world_model: 0.0
|
| 94 |
+
belief: 0.0
|
code/reveal_vla_bimanual/train/configs/rlbench_lift_ball_backbone_only_clip_step1.yaml
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
experiment_name: rlbench_lift_ball_backbone_only_clip_step1
|
| 2 |
+
output_dir: /workspace/outputs/rlbench_current
|
| 3 |
+
device: cuda
|
| 4 |
+
seed: 17
|
| 5 |
+
init_checkpoint: /workspace/outputs/rlbench_current/rlbench_lift_ball_backbone_only_clip_current_wide/checkpoint_stable.pt
|
| 6 |
+
init_strict: true
|
| 7 |
+
|
| 8 |
+
data:
|
| 9 |
+
dataset_root: /workspace/data/rlbench2
|
| 10 |
+
tasks: [bimanual_lift_ball]
|
| 11 |
+
train_episodes: [0, 1, 2, 3, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31]
|
| 12 |
+
val_episodes: [32, 33, 34, 35, 36, 37, 38, 39]
|
| 13 |
+
resolution: 224
|
| 14 |
+
chunk_horizon: 8
|
| 15 |
+
supervise_action_steps: 1
|
| 16 |
+
proprio_dim: 32
|
| 17 |
+
history_steps: 2
|
| 18 |
+
max_train_samples: 4096
|
| 19 |
+
max_val_samples: 2048
|
| 20 |
+
|
| 21 |
+
optim:
|
| 22 |
+
epochs: 8
|
| 23 |
+
batch_size: 16
|
| 24 |
+
num_workers: 4
|
| 25 |
+
lr: 0.00005
|
| 26 |
+
weight_decay: 0.0001
|
| 27 |
+
|
| 28 |
+
trainer:
|
| 29 |
+
policy_type: backbone_only
|
| 30 |
+
use_bf16: true
|
| 31 |
+
grad_clip_norm: 1.0
|
| 32 |
+
freeze_backbone: true
|
| 33 |
+
gradient_checkpointing: false
|
| 34 |
+
plan_during_train: false
|
| 35 |
+
plan_during_eval: false
|
| 36 |
+
support_mode_conditioning: true
|
| 37 |
+
|
| 38 |
+
policy:
|
| 39 |
+
backbone:
|
| 40 |
+
model_name: openai/clip-vit-base-patch32
|
| 41 |
+
hidden_dim: 512
|
| 42 |
+
max_text_tokens: 32
|
| 43 |
+
freeze_backbone: true
|
| 44 |
+
gradient_checkpointing: false
|
| 45 |
+
use_dummy_backbone: false
|
| 46 |
+
fusion:
|
| 47 |
+
hidden_dim: 512
|
| 48 |
+
num_cameras: 3
|
| 49 |
+
num_layers: 4
|
| 50 |
+
num_heads: 8
|
| 51 |
+
ff_dim: 2048
|
| 52 |
+
dropout: 0.1
|
| 53 |
+
proprio_dim: 32
|
| 54 |
+
proprio_tokens: 1
|
| 55 |
+
memory:
|
| 56 |
+
hidden_dim: 512
|
| 57 |
+
history_steps: 2
|
| 58 |
+
num_layers: 1
|
| 59 |
+
dropout: 0.1
|
| 60 |
+
decoder:
|
| 61 |
+
hidden_dim: 512
|
| 62 |
+
num_heads: 8
|
| 63 |
+
num_layers: 4
|
| 64 |
+
ff_dim: 2048
|
| 65 |
+
dropout: 0.1
|
| 66 |
+
chunk_size: 8
|
| 67 |
+
action_dim: 14
|
| 68 |
+
num_candidates: 8
|
| 69 |
+
reveal_head:
|
| 70 |
+
hidden_dim: 512
|
| 71 |
+
num_support_modes: 3
|
| 72 |
+
num_approach_templates: 32
|
| 73 |
+
rollout_horizon: 5
|
| 74 |
+
belief_map_size: 32
|
| 75 |
+
predict_belief_map: true
|
| 76 |
+
world_model:
|
| 77 |
+
hidden_dim: 512
|
| 78 |
+
action_dim: 14
|
| 79 |
+
num_support_modes: 3
|
| 80 |
+
num_approach_templates: 32
|
| 81 |
+
rollout_horizon: 5
|
| 82 |
+
planner:
|
| 83 |
+
hidden_dim: 512
|
| 84 |
+
num_candidates: 8
|
| 85 |
+
action_dim: 14
|
| 86 |
+
utility_margin: 0.1
|
| 87 |
+
|
| 88 |
+
loss_weights:
|
| 89 |
+
action: 1.0
|
| 90 |
+
support_mode: 0.0
|
| 91 |
+
corridor: 0.0
|
| 92 |
+
persistence: 0.0
|
| 93 |
+
disturbance: 0.0
|
| 94 |
+
world_model: 0.0
|
| 95 |
+
belief: 0.0
|
code/reveal_vla_bimanual/train/configs/rlbench_push_box_backbone_only_clip_step1.yaml
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
experiment_name: rlbench_push_box_backbone_only_clip_step1
|
| 2 |
+
output_dir: /workspace/outputs/rlbench_current
|
| 3 |
+
device: cuda
|
| 4 |
+
seed: 17
|
| 5 |
+
init_checkpoint: /workspace/outputs/rlbench_current/rlbench_subset3_backbone_only_clip_current_common23/checkpoint_stable.pt
|
| 6 |
+
init_strict: true
|
| 7 |
+
|
| 8 |
+
data:
|
| 9 |
+
dataset_root: /workspace/data/rlbench2
|
| 10 |
+
tasks: [bimanual_push_box]
|
| 11 |
+
train_episodes: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79]
|
| 12 |
+
val_episodes: [80, 81, 82, 83, 84, 85, 86, 87, 88, 89]
|
| 13 |
+
resolution: 224
|
| 14 |
+
chunk_horizon: 8
|
| 15 |
+
supervise_action_steps: 1
|
| 16 |
+
proprio_dim: 32
|
| 17 |
+
history_steps: 2
|
| 18 |
+
max_train_samples: 12288
|
| 19 |
+
max_val_samples: 2048
|
| 20 |
+
|
| 21 |
+
optim:
|
| 22 |
+
epochs: 6
|
| 23 |
+
batch_size: 32
|
| 24 |
+
num_workers: 8
|
| 25 |
+
lr: 0.00005
|
| 26 |
+
weight_decay: 0.0001
|
| 27 |
+
|
| 28 |
+
trainer:
|
| 29 |
+
policy_type: backbone_only
|
| 30 |
+
use_bf16: true
|
| 31 |
+
grad_clip_norm: 1.0
|
| 32 |
+
freeze_backbone: true
|
| 33 |
+
gradient_checkpointing: false
|
| 34 |
+
plan_during_train: false
|
| 35 |
+
plan_during_eval: false
|
| 36 |
+
support_mode_conditioning: true
|
| 37 |
+
|
| 38 |
+
policy:
|
| 39 |
+
backbone:
|
| 40 |
+
model_name: openai/clip-vit-base-patch32
|
| 41 |
+
hidden_dim: 512
|
| 42 |
+
max_text_tokens: 32
|
| 43 |
+
freeze_backbone: true
|
| 44 |
+
gradient_checkpointing: false
|
| 45 |
+
use_dummy_backbone: false
|
| 46 |
+
fusion:
|
| 47 |
+
hidden_dim: 512
|
| 48 |
+
num_cameras: 3
|
| 49 |
+
num_layers: 4
|
| 50 |
+
num_heads: 8
|
| 51 |
+
ff_dim: 2048
|
| 52 |
+
dropout: 0.1
|
| 53 |
+
proprio_dim: 32
|
| 54 |
+
proprio_tokens: 1
|
| 55 |
+
memory:
|
| 56 |
+
hidden_dim: 512
|
| 57 |
+
history_steps: 2
|
| 58 |
+
num_layers: 1
|
| 59 |
+
dropout: 0.1
|
| 60 |
+
decoder:
|
| 61 |
+
hidden_dim: 512
|
| 62 |
+
num_heads: 8
|
| 63 |
+
num_layers: 4
|
| 64 |
+
ff_dim: 2048
|
| 65 |
+
dropout: 0.1
|
| 66 |
+
chunk_size: 8
|
| 67 |
+
action_dim: 14
|
| 68 |
+
num_candidates: 8
|
| 69 |
+
reveal_head:
|
| 70 |
+
hidden_dim: 512
|
| 71 |
+
num_support_modes: 3
|
| 72 |
+
num_approach_templates: 32
|
| 73 |
+
rollout_horizon: 5
|
| 74 |
+
belief_map_size: 32
|
| 75 |
+
predict_belief_map: true
|
| 76 |
+
world_model:
|
| 77 |
+
hidden_dim: 512
|
| 78 |
+
action_dim: 14
|
| 79 |
+
num_support_modes: 3
|
| 80 |
+
num_approach_templates: 32
|
| 81 |
+
rollout_horizon: 5
|
| 82 |
+
planner:
|
| 83 |
+
hidden_dim: 512
|
| 84 |
+
num_candidates: 8
|
| 85 |
+
action_dim: 14
|
| 86 |
+
utility_margin: 0.1
|
| 87 |
+
|
| 88 |
+
loss_weights:
|
| 89 |
+
action: 1.0
|
| 90 |
+
support_mode: 0.0
|
| 91 |
+
corridor: 0.0
|
| 92 |
+
persistence: 0.0
|
| 93 |
+
disturbance: 0.0
|
| 94 |
+
world_model: 0.0
|
| 95 |
+
belief: 0.0
|
code/reveal_vla_bimanual/train/configs/rlbench_subset3_backbone_only_clip_current.yaml
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
experiment_name: rlbench_subset3_backbone_only_clip_current
|
| 2 |
+
output_dir: /workspace/outputs/rlbench_current
|
| 3 |
+
device: cuda
|
| 4 |
+
seed: 17
|
| 5 |
+
init_checkpoint: null
|
| 6 |
+
init_strict: false
|
| 7 |
+
|
| 8 |
+
data:
|
| 9 |
+
dataset_root: /workspace/data/rlbench2
|
| 10 |
+
tasks: [bimanual_lift_ball, bimanual_push_box, bimanual_dual_push_buttons]
|
| 11 |
+
train_episodes: [0, 1, 2, 3]
|
| 12 |
+
val_episodes: [4]
|
| 13 |
+
resolution: 224
|
| 14 |
+
chunk_horizon: 8
|
| 15 |
+
proprio_dim: 32
|
| 16 |
+
history_steps: 2
|
| 17 |
+
max_train_samples: 2048
|
| 18 |
+
max_val_samples: 512
|
| 19 |
+
|
| 20 |
+
optim:
|
| 21 |
+
epochs: 6
|
| 22 |
+
batch_size: 8
|
| 23 |
+
num_workers: 2
|
| 24 |
+
lr: 0.0002
|
| 25 |
+
weight_decay: 0.0001
|
| 26 |
+
|
| 27 |
+
trainer:
|
| 28 |
+
policy_type: backbone_only
|
| 29 |
+
use_bf16: true
|
| 30 |
+
grad_clip_norm: 1.0
|
| 31 |
+
freeze_backbone: true
|
| 32 |
+
gradient_checkpointing: false
|
| 33 |
+
plan_during_train: false
|
| 34 |
+
plan_during_eval: false
|
| 35 |
+
support_mode_conditioning: true
|
| 36 |
+
|
| 37 |
+
policy:
|
| 38 |
+
backbone:
|
| 39 |
+
model_name: openai/clip-vit-base-patch32
|
| 40 |
+
hidden_dim: 512
|
| 41 |
+
max_text_tokens: 32
|
| 42 |
+
freeze_backbone: true
|
| 43 |
+
gradient_checkpointing: false
|
| 44 |
+
use_dummy_backbone: false
|
| 45 |
+
fusion:
|
| 46 |
+
hidden_dim: 512
|
| 47 |
+
num_cameras: 3
|
| 48 |
+
num_layers: 4
|
| 49 |
+
num_heads: 8
|
| 50 |
+
ff_dim: 2048
|
| 51 |
+
dropout: 0.1
|
| 52 |
+
proprio_dim: 32
|
| 53 |
+
proprio_tokens: 1
|
| 54 |
+
memory:
|
| 55 |
+
hidden_dim: 512
|
| 56 |
+
history_steps: 2
|
| 57 |
+
num_layers: 1
|
| 58 |
+
dropout: 0.1
|
| 59 |
+
decoder:
|
| 60 |
+
hidden_dim: 512
|
| 61 |
+
num_heads: 8
|
| 62 |
+
num_layers: 4
|
| 63 |
+
ff_dim: 2048
|
| 64 |
+
dropout: 0.1
|
| 65 |
+
chunk_size: 8
|
| 66 |
+
action_dim: 14
|
| 67 |
+
num_candidates: 8
|
| 68 |
+
reveal_head:
|
| 69 |
+
hidden_dim: 512
|
| 70 |
+
num_support_modes: 3
|
| 71 |
+
num_approach_templates: 32
|
| 72 |
+
rollout_horizon: 5
|
| 73 |
+
belief_map_size: 32
|
| 74 |
+
predict_belief_map: true
|
| 75 |
+
world_model:
|
| 76 |
+
hidden_dim: 512
|
| 77 |
+
action_dim: 14
|
| 78 |
+
num_support_modes: 3
|
| 79 |
+
num_approach_templates: 32
|
| 80 |
+
rollout_horizon: 5
|
| 81 |
+
planner:
|
| 82 |
+
hidden_dim: 512
|
| 83 |
+
num_candidates: 8
|
| 84 |
+
action_dim: 14
|
| 85 |
+
utility_margin: 0.1
|
| 86 |
+
|
| 87 |
+
loss_weights:
|
| 88 |
+
action: 1.0
|
| 89 |
+
support_mode: 0.0
|
| 90 |
+
corridor: 0.0
|
| 91 |
+
persistence: 0.0
|
| 92 |
+
disturbance: 0.0
|
| 93 |
+
world_model: 0.0
|
| 94 |
+
belief: 0.0
|
code/reveal_vla_bimanual/train/configs/rlbench_subset3_backbone_only_clip_current_common23.yaml
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
experiment_name: rlbench_subset3_backbone_only_clip_current_common23
|
| 2 |
+
output_dir: /workspace/outputs/rlbench_current
|
| 3 |
+
device: cuda
|
| 4 |
+
seed: 17
|
| 5 |
+
init_checkpoint: /workspace/outputs/rlbench_current/rlbench_subset3_backbone_only_clip_current_valid9/checkpoint_best.pt
|
| 6 |
+
init_strict: true
|
| 7 |
+
|
| 8 |
+
data:
|
| 9 |
+
dataset_root: /workspace/data/rlbench2
|
| 10 |
+
tasks: [bimanual_lift_ball, bimanual_push_box, bimanual_dual_push_buttons]
|
| 11 |
+
train_episodes: [0, 1, 2, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
|
| 12 |
+
val_episodes: [26, 27, 28, 29]
|
| 13 |
+
resolution: 224
|
| 14 |
+
chunk_horizon: 8
|
| 15 |
+
proprio_dim: 32
|
| 16 |
+
history_steps: 2
|
| 17 |
+
max_train_samples: 8192
|
| 18 |
+
max_val_samples: 2048
|
| 19 |
+
|
| 20 |
+
optim:
|
| 21 |
+
epochs: 8
|
| 22 |
+
batch_size: 16
|
| 23 |
+
num_workers: 4
|
| 24 |
+
lr: 0.0001
|
| 25 |
+
weight_decay: 0.0001
|
| 26 |
+
|
| 27 |
+
trainer:
|
| 28 |
+
policy_type: backbone_only
|
| 29 |
+
use_bf16: true
|
| 30 |
+
grad_clip_norm: 1.0
|
| 31 |
+
freeze_backbone: true
|
| 32 |
+
gradient_checkpointing: false
|
| 33 |
+
plan_during_train: false
|
| 34 |
+
plan_during_eval: false
|
| 35 |
+
support_mode_conditioning: true
|
| 36 |
+
|
| 37 |
+
policy:
|
| 38 |
+
backbone:
|
| 39 |
+
model_name: openai/clip-vit-base-patch32
|
| 40 |
+
hidden_dim: 512
|
| 41 |
+
max_text_tokens: 32
|
| 42 |
+
freeze_backbone: true
|
| 43 |
+
gradient_checkpointing: false
|
| 44 |
+
use_dummy_backbone: false
|
| 45 |
+
fusion:
|
| 46 |
+
hidden_dim: 512
|
| 47 |
+
num_cameras: 3
|
| 48 |
+
num_layers: 4
|
| 49 |
+
num_heads: 8
|
| 50 |
+
ff_dim: 2048
|
| 51 |
+
dropout: 0.1
|
| 52 |
+
proprio_dim: 32
|
| 53 |
+
proprio_tokens: 1
|
| 54 |
+
memory:
|
| 55 |
+
hidden_dim: 512
|
| 56 |
+
history_steps: 2
|
| 57 |
+
num_layers: 1
|
| 58 |
+
dropout: 0.1
|
| 59 |
+
decoder:
|
| 60 |
+
hidden_dim: 512
|
| 61 |
+
num_heads: 8
|
| 62 |
+
num_layers: 4
|
| 63 |
+
ff_dim: 2048
|
| 64 |
+
dropout: 0.1
|
| 65 |
+
chunk_size: 8
|
| 66 |
+
action_dim: 14
|
| 67 |
+
num_candidates: 8
|
| 68 |
+
reveal_head:
|
| 69 |
+
hidden_dim: 512
|
| 70 |
+
num_support_modes: 3
|
| 71 |
+
num_approach_templates: 32
|
| 72 |
+
rollout_horizon: 5
|
| 73 |
+
belief_map_size: 32
|
| 74 |
+
predict_belief_map: true
|
| 75 |
+
world_model:
|
| 76 |
+
hidden_dim: 512
|
| 77 |
+
action_dim: 14
|
| 78 |
+
num_support_modes: 3
|
| 79 |
+
num_approach_templates: 32
|
| 80 |
+
rollout_horizon: 5
|
| 81 |
+
planner:
|
| 82 |
+
hidden_dim: 512
|
| 83 |
+
num_candidates: 8
|
| 84 |
+
action_dim: 14
|
| 85 |
+
utility_margin: 0.1
|
| 86 |
+
|
| 87 |
+
loss_weights:
|
| 88 |
+
action: 1.0
|
| 89 |
+
support_mode: 0.0
|
| 90 |
+
corridor: 0.0
|
| 91 |
+
persistence: 0.0
|
| 92 |
+
disturbance: 0.0
|
| 93 |
+
world_model: 0.0
|
| 94 |
+
belief: 0.0
|
code/reveal_vla_bimanual/train/configs/rlbench_subset3_backbone_only_clip_current_valid9.yaml
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
experiment_name: rlbench_subset3_backbone_only_clip_current_valid9
|
| 2 |
+
output_dir: /workspace/outputs/rlbench_current
|
| 3 |
+
device: cuda
|
| 4 |
+
seed: 17
|
| 5 |
+
init_checkpoint: /workspace/outputs/rlbench_current/rlbench_lift_ball_backbone_only_clip_current_3train/checkpoint_best.pt
|
| 6 |
+
init_strict: true
|
| 7 |
+
|
| 8 |
+
data:
|
| 9 |
+
dataset_root: /workspace/data/rlbench2
|
| 10 |
+
tasks: [bimanual_lift_ball, bimanual_push_box, bimanual_dual_push_buttons]
|
| 11 |
+
train_episodes: [0, 1, 10, 11, 12, 13, 14, 15]
|
| 12 |
+
val_episodes: [16]
|
| 13 |
+
resolution: 224
|
| 14 |
+
chunk_horizon: 8
|
| 15 |
+
proprio_dim: 32
|
| 16 |
+
history_steps: 2
|
| 17 |
+
max_train_samples: 4096
|
| 18 |
+
max_val_samples: 1024
|
| 19 |
+
|
| 20 |
+
optim:
|
| 21 |
+
epochs: 8
|
| 22 |
+
batch_size: 16
|
| 23 |
+
num_workers: 4
|
| 24 |
+
lr: 0.0001
|
| 25 |
+
weight_decay: 0.0001
|
| 26 |
+
|
| 27 |
+
trainer:
|
| 28 |
+
policy_type: backbone_only
|
| 29 |
+
use_bf16: true
|
| 30 |
+
grad_clip_norm: 1.0
|
| 31 |
+
freeze_backbone: true
|
| 32 |
+
gradient_checkpointing: false
|
| 33 |
+
plan_during_train: false
|
| 34 |
+
plan_during_eval: false
|
| 35 |
+
support_mode_conditioning: true
|
| 36 |
+
|
| 37 |
+
policy:
|
| 38 |
+
backbone:
|
| 39 |
+
model_name: openai/clip-vit-base-patch32
|
| 40 |
+
hidden_dim: 512
|
| 41 |
+
max_text_tokens: 32
|
| 42 |
+
freeze_backbone: true
|
| 43 |
+
gradient_checkpointing: false
|
| 44 |
+
use_dummy_backbone: false
|
| 45 |
+
fusion:
|
| 46 |
+
hidden_dim: 512
|
| 47 |
+
num_cameras: 3
|
| 48 |
+
num_layers: 4
|
| 49 |
+
num_heads: 8
|
| 50 |
+
ff_dim: 2048
|
| 51 |
+
dropout: 0.1
|
| 52 |
+
proprio_dim: 32
|
| 53 |
+
proprio_tokens: 1
|
| 54 |
+
memory:
|
| 55 |
+
hidden_dim: 512
|
| 56 |
+
history_steps: 2
|
| 57 |
+
num_layers: 1
|
| 58 |
+
dropout: 0.1
|
| 59 |
+
decoder:
|
| 60 |
+
hidden_dim: 512
|
| 61 |
+
num_heads: 8
|
| 62 |
+
num_layers: 4
|
| 63 |
+
ff_dim: 2048
|
| 64 |
+
dropout: 0.1
|
| 65 |
+
chunk_size: 8
|
| 66 |
+
action_dim: 14
|
| 67 |
+
num_candidates: 8
|
| 68 |
+
reveal_head:
|
| 69 |
+
hidden_dim: 512
|
| 70 |
+
num_support_modes: 3
|
| 71 |
+
num_approach_templates: 32
|
| 72 |
+
rollout_horizon: 5
|
| 73 |
+
belief_map_size: 32
|
| 74 |
+
predict_belief_map: true
|
| 75 |
+
world_model:
|
| 76 |
+
hidden_dim: 512
|
| 77 |
+
action_dim: 14
|
| 78 |
+
num_support_modes: 3
|
| 79 |
+
num_approach_templates: 32
|
| 80 |
+
rollout_horizon: 5
|
| 81 |
+
planner:
|
| 82 |
+
hidden_dim: 512
|
| 83 |
+
num_candidates: 8
|
| 84 |
+
action_dim: 14
|
| 85 |
+
utility_margin: 0.1
|
| 86 |
+
|
| 87 |
+
loss_weights:
|
| 88 |
+
action: 1.0
|
| 89 |
+
support_mode: 0.0
|
| 90 |
+
corridor: 0.0
|
| 91 |
+
persistence: 0.0
|
| 92 |
+
disturbance: 0.0
|
| 93 |
+
world_model: 0.0
|
| 94 |
+
belief: 0.0
|
code/reveal_vla_bimanual/train/losses.py
CHANGED
|
@@ -309,7 +309,11 @@ def compute_total_loss(
|
|
| 309 |
) -> dict[str, Tensor]:
|
| 310 |
weights = weights or LossWeights()
|
| 311 |
losses = {
|
| 312 |
-
"action": chunk_bc_loss(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 313 |
}
|
| 314 |
total = weights.action * losses["action"]
|
| 315 |
|
|
|
|
| 309 |
) -> dict[str, Tensor]:
|
| 310 |
weights = weights or LossWeights()
|
| 311 |
losses = {
|
| 312 |
+
"action": chunk_bc_loss(
|
| 313 |
+
model_output["action_mean"],
|
| 314 |
+
batch["action_chunk"],
|
| 315 |
+
mask=batch.get("action_mask"),
|
| 316 |
+
),
|
| 317 |
}
|
| 318 |
total = weights.action * losses["action"]
|
| 319 |
|
code/reveal_vla_bimanual/train/run_rlbench_experiment.py
CHANGED
|
@@ -74,6 +74,7 @@ def main() -> None:
|
|
| 74 |
chunk_size=int(cfg.data.chunk_horizon),
|
| 75 |
proprio_dim=int(cfg.data.proprio_dim),
|
| 76 |
history_steps=int(cfg.data.get("history_steps", 2)),
|
|
|
|
| 77 |
max_samples=cfg.data.get("max_train_samples"),
|
| 78 |
)
|
| 79 |
val_dataset = RLBenchOfflineChunkDataset(
|
|
@@ -84,6 +85,7 @@ def main() -> None:
|
|
| 84 |
chunk_size=int(cfg.data.chunk_horizon),
|
| 85 |
proprio_dim=int(cfg.data.proprio_dim),
|
| 86 |
history_steps=int(cfg.data.get("history_steps", 2)),
|
|
|
|
| 87 |
max_samples=cfg.data.get("max_val_samples"),
|
| 88 |
)
|
| 89 |
train_loader = _make_loader(
|
|
|
|
| 74 |
chunk_size=int(cfg.data.chunk_horizon),
|
| 75 |
proprio_dim=int(cfg.data.proprio_dim),
|
| 76 |
history_steps=int(cfg.data.get("history_steps", 2)),
|
| 77 |
+
supervise_action_steps=cfg.data.get("supervise_action_steps"),
|
| 78 |
max_samples=cfg.data.get("max_train_samples"),
|
| 79 |
)
|
| 80 |
val_dataset = RLBenchOfflineChunkDataset(
|
|
|
|
| 85 |
chunk_size=int(cfg.data.chunk_horizon),
|
| 86 |
proprio_dim=int(cfg.data.proprio_dim),
|
| 87 |
history_steps=int(cfg.data.get("history_steps", 2)),
|
| 88 |
+
supervise_action_steps=cfg.data.get("supervise_action_steps"),
|
| 89 |
max_samples=cfg.data.get("max_val_samples"),
|
| 90 |
)
|
| 91 |
train_loader = _make_loader(
|
environment/README.md
CHANGED
|
@@ -28,6 +28,11 @@ This directory contains the machine snapshot and setup helpers for the runpod no
|
|
| 28 |
- bundle-aware bootstrap script for a matching `/workspace` machine
|
| 29 |
- `validate_same_machine.sh`
|
| 30 |
- validation helper that runs `glxinfo`, RLBench import smoke, and `open_drawer` launch smoke
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
## Default Layout
|
| 33 |
|
|
@@ -35,6 +40,7 @@ This directory contains the machine snapshot and setup helpers for the runpod no
|
|
| 35 |
- bundled project code: `/workspace/VLAarchtests/code/reveal_vla_bimanual`
|
| 36 |
- runtime symlink target used by the setup scripts: `/workspace/reveal_vla_bimanual`
|
| 37 |
- third-party checkout root: `/workspace/third_party`
|
|
|
|
| 38 |
|
| 39 |
## Validation Commands
|
| 40 |
|
|
@@ -42,4 +48,5 @@ This directory contains the machine snapshot and setup helpers for the runpod no
|
|
| 42 |
DISPLAY=:99 glxinfo -B
|
| 43 |
/workspace/.tools/micromamba/bin/micromamba run -r /workspace/.micromamba -p /workspace/envs/rlbench python -m sim_rlbench.launch_smoke --headless
|
| 44 |
/workspace/VLAarchtests/environment/validate_same_machine.sh
|
|
|
|
| 45 |
```
|
|
|
|
| 28 |
- bundle-aware bootstrap script for a matching `/workspace` machine
|
| 29 |
- `validate_same_machine.sh`
|
| 30 |
- validation helper that runs `glxinfo`, RLBench import smoke, and `open_drawer` launch smoke
|
| 31 |
+
- `hf_cli_version.txt`
|
| 32 |
+
- raw `hf version` output from the upload session
|
| 33 |
+
- Hugging Face CLI used for upload in this session
|
| 34 |
+
- install command: `curl -LsSf https://hf.co/cli/install.sh | bash`
|
| 35 |
+
- install path on this node: `/workspace/.local/bin/hf`
|
| 36 |
|
| 37 |
## Default Layout
|
| 38 |
|
|
|
|
| 40 |
- bundled project code: `/workspace/VLAarchtests/code/reveal_vla_bimanual`
|
| 41 |
- runtime symlink target used by the setup scripts: `/workspace/reveal_vla_bimanual`
|
| 42 |
- third-party checkout root: `/workspace/third_party`
|
| 43 |
+
- handoff instructions copied into the bundle: `/workspace/VLAarchtests/handoff/instructions.md`
|
| 44 |
|
| 45 |
## Validation Commands
|
| 46 |
|
|
|
|
| 48 |
DISPLAY=:99 glxinfo -B
|
| 49 |
/workspace/.tools/micromamba/bin/micromamba run -r /workspace/.micromamba -p /workspace/envs/rlbench python -m sim_rlbench.launch_smoke --headless
|
| 50 |
/workspace/VLAarchtests/environment/validate_same_machine.sh
|
| 51 |
+
env PATH="/workspace/.local/bin:$PATH" hf version
|
| 52 |
```
|