Upload folder using huggingface_hub

l10_4tasks/config.yaml

@@ -0,0 +1,523 @@
+callbacks:
+  rollout_lh:
+    tasks:
+      _target_: calvin_env.envs.tasks.Tasks
+      tasks:
+        rotate_red_block_right:
+        - rotate_object
+        - block_red
+        - -60
+        rotate_red_block_left:
+        - rotate_object
+        - block_red
+        - 60
+        rotate_blue_block_right:
+        - rotate_object
+        - block_blue
+        - -60
+        rotate_blue_block_left:
+        - rotate_object
+        - block_blue
+        - 60
+        rotate_pink_block_right:
+        - rotate_object
+        - block_pink
+        - -60
+        rotate_pink_block_left:
+        - rotate_object
+        - block_pink
+        - 60
+        push_red_block_right:
+        - push_object
+        - block_red
+        - 0.1
+        - 0
+        push_red_block_left:
+        - push_object
+        - block_red
+        - -0.1
+        - 0
+        push_blue_block_right:
+        - push_object
+        - block_blue
+        - 0.1
+        - 0
+        push_blue_block_left:
+        - push_object
+        - block_blue
+        - -0.1
+        - 0
+        push_pink_block_right:
+        - push_object
+        - block_pink
+        - 0.1
+        - 0
+        push_pink_block_left:
+        - push_object
+        - block_pink
+        - -0.1
+        - 0
+        move_slider_left:
+        - move_door_rel
+        - base__slide
+        - 0.15
+        move_slider_right:
+        - move_door_rel
+        - base__slide
+        - -0.15
+        open_drawer:
+        - move_door_rel
+        - base__drawer
+        - 0.12
+        close_drawer:
+        - move_door_rel
+        - base__drawer
+        - -0.12
+        lift_red_block_table:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - base_link
+        lift_red_block_slider:
+        - lift_object
+        - block_red
+        - 0.03
+        - table
+        - plank_link
+        lift_red_block_drawer:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - drawer_link
+        lift_blue_block_table:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - base_link
+        lift_blue_block_slider:
+        - lift_object
+        - block_blue
+        - 0.03
+        - table
+        - plank_link
+        lift_blue_block_drawer:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - drawer_link
+        lift_pink_block_table:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - base_link
+        lift_pink_block_slider:
+        - lift_object
+        - block_pink
+        - 0.03
+        - table
+        - plank_link
+        lift_pink_block_drawer:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - drawer_link
+        place_in_slider:
+        - place_object
+        - table
+        - plank_link
+        place_in_drawer:
+        - place_object
+        - table
+        - drawer_link
+        stack_block:
+        - stack_objects
+        unstack_block:
+        - unstack_objects
+        turn_on_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 0
+        - 1
+        turn_off_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 1
+        - 0
+        turn_on_led:
+        - toggle_light
+        - led
+        - 0
+        - 1
+        turn_off_led:
+        - toggle_light
+        - led
+        - 1
+        - 0
+        push_into_drawer:
+        - push_object_into
+        - - block_red
+          - block_blue
+          - block_pink
+        - table
+        - base_link
+        - table
+        - drawer_link
+    val_annotations:
+      rotate_red_block_right:
+      - take the red block and rotate it to the right
+      rotate_red_block_left:
+      - take the red block and rotate it to the left
+      rotate_blue_block_right:
+      - take the blue block and rotate it to the right
+      rotate_blue_block_left:
+      - take the blue block and rotate it to the left
+      rotate_pink_block_right:
+      - take the pink block and rotate it to the right
+      rotate_pink_block_left:
+      - take the pink block and rotate it to the left
+      push_red_block_right:
+      - go push the red block right
+      push_red_block_left:
+      - go push the red block left
+      push_blue_block_right:
+      - go push the blue block right
+      push_blue_block_left:
+      - go push the blue block left
+      push_pink_block_right:
+      - go push the pink block right
+      push_pink_block_left:
+      - go push the pink block left
+      move_slider_left:
+      - push the sliding door to the left side
+      move_slider_right:
+      - push the sliding door to the right side
+      open_drawer:
+      - pull the handle to open the drawer
+      close_drawer:
+      - push the handle to close the drawer
+      lift_red_block_table:
+      - grasp and lift the red block
+      lift_blue_block_table:
+      - grasp and lift the blue block
+      lift_pink_block_table:
+      - grasp and lift the pink block
+      lift_red_block_slider:
+      - lift the red block from the sliding cabinet
+      lift_blue_block_slider:
+      - lift the blue block from the sliding cabinet
+      lift_pink_block_slider:
+      - lift the pink block from the sliding cabinet
+      lift_red_block_drawer:
+      - Take the red block from the drawer
+      lift_blue_block_drawer:
+      - Take the blue block from the drawer
+      lift_pink_block_drawer:
+      - Take the pink block from the drawer
+      place_in_slider:
+      - store the grasped block in the sliding cabinet
+      place_in_drawer:
+      - store the grasped block in the drawer
+      push_into_drawer:
+      - slide the block that it falls into the drawer
+      stack_block:
+      - stack the grasped block
+      unstack_block:
+      - remove the stacked block
+      turn_on_lightbulb:
+      - use the switch to turn on the light bulb
+      turn_off_lightbulb:
+      - use the switch to turn off the light bulb
+      turn_on_led:
+      - press the button to turn on the led light
+      turn_off_led:
+      - press the button to turn off the led light
+    _target_: mode.rollout.rollout_long_horizon.RolloutLongHorizon
+    _recursive_: false
+    env_cfg:
+      _target_: mode.wrappers.hulc_wrapper.HulcWrapper
+    skip_epochs: ${rollout_lh_skip_epochs}
+    rollout_freq: 5
+    num_videos: 0
+    num_sequences: 1000
+    replan_freq: 30
+    ep_len: 360
+    empty_cache: false
+    log_video_to_file: false
+    save_dir: ./videos
+    lang_folder: ${lang_folder}
+    debug: false
+  ema:
+    _target_: mode.callbacks.ema.EMA
+    decay: 0.999
+    start_step: 0
+    save_ema_weights_in_callback_state: true
+    evaluate_ema_weights_instead: true
+    power: 0.6666666666666666
+    inv_gamma: 1.0
+    min_value: 0.0
+    max_value: 0.9999
+  checkpoint:
+    _target_: pytorch_lightning.callbacks.ModelCheckpoint
+    verbose: true
+    dirpath: saved_models
+    filename: '{epoch:02d}'
+    every_n_epochs: ${callbacks.rollout_lh.rollout_freq}
+datamodule:
+  transforms:
+    train:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 10
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 4
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+    val:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+  _target_: mode.datasets.hulc_data_module.HulcDataModule
+  _recursive_: false
+  root_data_dir: ${root_data_dir}
+  action_space: 7
+  action_max:
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  action_min:
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1
+  shuffle_val: false
+  observation_space:
+    rgb_obs:
+    - rgb_static
+    - rgb_gripper
+    depth_obs: []
+    state_obs:
+    - robot_obs
+    actions:
+    - rel_actions
+    language:
+    - language
+  proprioception_dims:
+    n_state_obs: 8
+    keep_indices:
+    - - 0
+      - 7
+    - - 14
+      - 15
+    robot_orientation_idx:
+    - 3
+    - 6
+    normalize: true
+    normalize_robot_orientation: true
+  datasets:
+    lang_dataset:
+      _target_: mode.datasets.disk_dataset.ExtendedDiskDataset
+      key: lang
+      save_format: npz
+      batch_size: ${batch_size}
+      min_window_size: ${act_seq_len}
+      max_window_size: ${act_seq_len}
+      proprio_state: ${datamodule.proprioception_dims}
+      obs_space: ${datamodule.observation_space}
+      skip_frames: 1
+      pad: false
+      lang_folder: ${lang_folder}
+      aux_lang_loss_window: 8
+      num_workers: ${num_workers}
+      action_seq_len: ${act_seq_len}
+      obs_seq_len: ${obs_seq_len}
+      future_range: 1
+      use_extracted_rel_actions: ${use_extracted_rel_actions}
+model:
+  _target_: mode.models.mode_agent.MoDEAgent
+  _recursive_: false
+  multistep: ${multistep}
+  use_lr_scheduler: true
+  entropy_gamma: 0.01
+  router_z_delta: 0.0
+  use_proprio: false
+  seed: ${seed}
+  sampler_type: ddim
+  num_sampling_steps: 10
+  sigma_data: 0.5
+  sigma_min: 0.001
+  sigma_max: 80
+  noise_scheduler: exponential
+  sigma_sample_density_type: loglogistic
+  ckpt_path: /home/yiming/pretrained_models/MoDE/MoDE_Pretrained
+  start_from_pretrained: true
+  act_window_size: ${act_seq_len}
+  latent_dim: 1024
+  obs_enc_dim: ${obs_dim}
+  cond_dim: 512
+  resnet_type: '50'
+  optimizer:
+    _target_: torch.optim.AdamW
+    transformer_weight_decay: 0.05
+    obs_encoder_weight_decay: 0.05
+    learning_rate: 0.0001
+    betas:
+    - 0.9
+    - 0.95
+  lr_scheduler:
+    lr_scheduler:
+      init_lr: 0.0001
+      init_lr_scale: 0.1
+      final_lr_scale: 1.0e-06
+      total_steps: 45000
+      phase_ratio: (0.02, 0.08, 0.9)
+      lr: 0.0001
+  model:
+    _target_: mode.models.edm_diffusion.score_wrappers.GCDenoiser
+    _recursive_: false
+    sigma_data: ${model.sigma_data}
+    inner_model:
+      _target_: mode.models.networks.modedit.MoDeDiT
+      action_dim: ${datamodule.action_space}
+      goal_dim: ${model.cond_dim}
+      obs_dim: 2048
+      goal_conditioned: true
+      causal: true
+      use_custom_attn_mask: false
+      use_proprio: ${model.use_proprio}
+      state_dim: ${proprio_dims}
+      embed_dim: ${model.latent_dim}
+      n_layers: 10
+      goal_seq_len: 1
+      obs_seq_len: ${obs_seq_len}
+      action_seq_len: ${act_seq_len}
+      embed_pdrob: 0
+      goal_drop: 0.1
+      attn_pdrop: 0.3
+      mlp_pdrop: 0.1
+      n_heads: 8
+      device: ${device}
+      linear_output: true
+      cond_router: true
+      num_experts: 4
+      top_k: 2
+      router_normalize: true
+      use_goal_in_routing: false
+      use_argmax: false
+      use_shared_expert: false
+      use_noise_token_as_input: true
+      init_style: olmoe
+  language_goal:
+    _target_: mode.models.networks.clip_lang_encoder.LangClip
+    _recursive_: false
+    model_name: ${clip_lang_model_name}
+root_data_dir: data/lebai
+lang_folder: lang_clip_resnet50
+vis_clip_model_name: ViT-B/16
+clip_lang_model_name: ViT-B/32
+log_dir: ./logs
+slurm: false
+seed: 421
+device: cuda
+batch_size: 64
+devices: 4
+act_dim: 7
+proprio_dims: 7
+obs_dim: 512
+goal_dim: 512
+obs_seq_len: 1
+act_seq_len: 32
+multistep: 32
+p_last_state: 0
+gen_img_res: 112
+max_epochs: 100
+rollout_lh_skip_epochs: 9
+num_workers: 12
+benchmark_name: lebai_mode
+use_extracted_rel_actions: true
+trainer:
+  devices: ${devices}
+  precision: bf16
+  max_epochs: ${max_epochs}
+  sync_batchnorm: true
+  accelerator: gpu
+  strategy: ddp
+logger:
+  _target_: pytorch_lightning.loggers.WandbLogger
+  save_dir: .
+  name: logger
+  group: mode
+  log_model: false
+  project: ${benchmark_name}
+  entity: yimingwu0
+  id: ???

l10_4tasks/model_cleaned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:202a4f0edcff5ea0733939d018dc7edd587862a0dcfc37f3b2384b176b08d271
+size 3115403448

l10_ds1_only/config.yaml

@@ -0,0 +1,523 @@
+callbacks:
+  rollout_lh:
+    tasks:
+      _target_: calvin_env.envs.tasks.Tasks
+      tasks:
+        rotate_red_block_right:
+        - rotate_object
+        - block_red
+        - -60
+        rotate_red_block_left:
+        - rotate_object
+        - block_red
+        - 60
+        rotate_blue_block_right:
+        - rotate_object
+        - block_blue
+        - -60
+        rotate_blue_block_left:
+        - rotate_object
+        - block_blue
+        - 60
+        rotate_pink_block_right:
+        - rotate_object
+        - block_pink
+        - -60
+        rotate_pink_block_left:
+        - rotate_object
+        - block_pink
+        - 60
+        push_red_block_right:
+        - push_object
+        - block_red
+        - 0.1
+        - 0
+        push_red_block_left:
+        - push_object
+        - block_red
+        - -0.1
+        - 0
+        push_blue_block_right:
+        - push_object
+        - block_blue
+        - 0.1
+        - 0
+        push_blue_block_left:
+        - push_object
+        - block_blue
+        - -0.1
+        - 0
+        push_pink_block_right:
+        - push_object
+        - block_pink
+        - 0.1
+        - 0
+        push_pink_block_left:
+        - push_object
+        - block_pink
+        - -0.1
+        - 0
+        move_slider_left:
+        - move_door_rel
+        - base__slide
+        - 0.15
+        move_slider_right:
+        - move_door_rel
+        - base__slide
+        - -0.15
+        open_drawer:
+        - move_door_rel
+        - base__drawer
+        - 0.12
+        close_drawer:
+        - move_door_rel
+        - base__drawer
+        - -0.12
+        lift_red_block_table:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - base_link
+        lift_red_block_slider:
+        - lift_object
+        - block_red
+        - 0.03
+        - table
+        - plank_link
+        lift_red_block_drawer:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - drawer_link
+        lift_blue_block_table:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - base_link
+        lift_blue_block_slider:
+        - lift_object
+        - block_blue
+        - 0.03
+        - table
+        - plank_link
+        lift_blue_block_drawer:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - drawer_link
+        lift_pink_block_table:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - base_link
+        lift_pink_block_slider:
+        - lift_object
+        - block_pink
+        - 0.03
+        - table
+        - plank_link
+        lift_pink_block_drawer:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - drawer_link
+        place_in_slider:
+        - place_object
+        - table
+        - plank_link
+        place_in_drawer:
+        - place_object
+        - table
+        - drawer_link
+        stack_block:
+        - stack_objects
+        unstack_block:
+        - unstack_objects
+        turn_on_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 0
+        - 1
+        turn_off_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 1
+        - 0
+        turn_on_led:
+        - toggle_light
+        - led
+        - 0
+        - 1
+        turn_off_led:
+        - toggle_light
+        - led
+        - 1
+        - 0
+        push_into_drawer:
+        - push_object_into
+        - - block_red
+          - block_blue
+          - block_pink
+        - table
+        - base_link
+        - table
+        - drawer_link
+    val_annotations:
+      rotate_red_block_right:
+      - take the red block and rotate it to the right
+      rotate_red_block_left:
+      - take the red block and rotate it to the left
+      rotate_blue_block_right:
+      - take the blue block and rotate it to the right
+      rotate_blue_block_left:
+      - take the blue block and rotate it to the left
+      rotate_pink_block_right:
+      - take the pink block and rotate it to the right
+      rotate_pink_block_left:
+      - take the pink block and rotate it to the left
+      push_red_block_right:
+      - go push the red block right
+      push_red_block_left:
+      - go push the red block left
+      push_blue_block_right:
+      - go push the blue block right
+      push_blue_block_left:
+      - go push the blue block left
+      push_pink_block_right:
+      - go push the pink block right
+      push_pink_block_left:
+      - go push the pink block left
+      move_slider_left:
+      - push the sliding door to the left side
+      move_slider_right:
+      - push the sliding door to the right side
+      open_drawer:
+      - pull the handle to open the drawer
+      close_drawer:
+      - push the handle to close the drawer
+      lift_red_block_table:
+      - grasp and lift the red block
+      lift_blue_block_table:
+      - grasp and lift the blue block
+      lift_pink_block_table:
+      - grasp and lift the pink block
+      lift_red_block_slider:
+      - lift the red block from the sliding cabinet
+      lift_blue_block_slider:
+      - lift the blue block from the sliding cabinet
+      lift_pink_block_slider:
+      - lift the pink block from the sliding cabinet
+      lift_red_block_drawer:
+      - Take the red block from the drawer
+      lift_blue_block_drawer:
+      - Take the blue block from the drawer
+      lift_pink_block_drawer:
+      - Take the pink block from the drawer
+      place_in_slider:
+      - store the grasped block in the sliding cabinet
+      place_in_drawer:
+      - store the grasped block in the drawer
+      push_into_drawer:
+      - slide the block that it falls into the drawer
+      stack_block:
+      - stack the grasped block
+      unstack_block:
+      - remove the stacked block
+      turn_on_lightbulb:
+      - use the switch to turn on the light bulb
+      turn_off_lightbulb:
+      - use the switch to turn off the light bulb
+      turn_on_led:
+      - press the button to turn on the led light
+      turn_off_led:
+      - press the button to turn off the led light
+    _target_: mode.rollout.rollout_long_horizon.RolloutLongHorizon
+    _recursive_: false
+    env_cfg:
+      _target_: mode.wrappers.hulc_wrapper.HulcWrapper
+    skip_epochs: ${rollout_lh_skip_epochs}
+    rollout_freq: 5
+    num_videos: 0
+    num_sequences: 1000
+    replan_freq: 30
+    ep_len: 360
+    empty_cache: false
+    log_video_to_file: false
+    save_dir: ./videos
+    lang_folder: ${lang_folder}
+    debug: false
+  ema:
+    _target_: mode.callbacks.ema.EMA
+    decay: 0.999
+    start_step: 0
+    save_ema_weights_in_callback_state: true
+    evaluate_ema_weights_instead: true
+    power: 0.6666666666666666
+    inv_gamma: 1.0
+    min_value: 0.0
+    max_value: 0.9999
+  checkpoint:
+    _target_: pytorch_lightning.callbacks.ModelCheckpoint
+    verbose: true
+    dirpath: saved_models
+    filename: '{epoch:02d}'
+    every_n_epochs: ${callbacks.rollout_lh.rollout_freq}
+datamodule:
+  transforms:
+    train:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 10
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 4
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+    val:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+  _target_: mode.datasets.hulc_data_module.HulcDataModule
+  _recursive_: false
+  root_data_dir: ${root_data_dir}
+  action_space: 7
+  action_max:
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  action_min:
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1
+  shuffle_val: false
+  observation_space:
+    rgb_obs:
+    - rgb_static
+    - rgb_gripper
+    depth_obs: []
+    state_obs:
+    - robot_obs
+    actions:
+    - rel_actions
+    language:
+    - language
+  proprioception_dims:
+    n_state_obs: 8
+    keep_indices:
+    - - 0
+      - 7
+    - - 14
+      - 15
+    robot_orientation_idx:
+    - 3
+    - 6
+    normalize: true
+    normalize_robot_orientation: true
+  datasets:
+    lang_dataset:
+      _target_: mode.datasets.disk_dataset.ExtendedDiskDataset
+      key: lang
+      save_format: npz
+      batch_size: ${batch_size}
+      min_window_size: ${act_seq_len}
+      max_window_size: ${act_seq_len}
+      proprio_state: ${datamodule.proprioception_dims}
+      obs_space: ${datamodule.observation_space}
+      skip_frames: 1
+      pad: false
+      lang_folder: ${lang_folder}
+      aux_lang_loss_window: 8
+      num_workers: ${num_workers}
+      action_seq_len: ${act_seq_len}
+      obs_seq_len: ${obs_seq_len}
+      future_range: 1
+      use_extracted_rel_actions: ${use_extracted_rel_actions}
+model:
+  _target_: mode.models.mode_agent.MoDEAgent
+  _recursive_: false
+  multistep: ${multistep}
+  use_lr_scheduler: true
+  entropy_gamma: 0.01
+  router_z_delta: 0.0
+  use_proprio: false
+  seed: ${seed}
+  sampler_type: ddim
+  num_sampling_steps: 10
+  sigma_data: 0.5
+  sigma_min: 0.001
+  sigma_max: 80
+  noise_scheduler: exponential
+  sigma_sample_density_type: loglogistic
+  ckpt_path: /home/yiming/pretrained_models/MoDE/MoDE_Pretrained
+  start_from_pretrained: true
+  act_window_size: ${act_seq_len}
+  latent_dim: 1024
+  obs_enc_dim: ${obs_dim}
+  cond_dim: 512
+  resnet_type: '50'
+  optimizer:
+    _target_: torch.optim.AdamW
+    transformer_weight_decay: 0.05
+    obs_encoder_weight_decay: 0.05
+    learning_rate: 0.0001
+    betas:
+    - 0.9
+    - 0.95
+  lr_scheduler:
+    lr_scheduler:
+      init_lr: 0.0001
+      init_lr_scale: 0.1
+      final_lr_scale: 1.0e-06
+      total_steps: 45000
+      phase_ratio: (0.02, 0.08, 0.9)
+      lr: 0.0001
+  model:
+    _target_: mode.models.edm_diffusion.score_wrappers.GCDenoiser
+    _recursive_: false
+    sigma_data: ${model.sigma_data}
+    inner_model:
+      _target_: mode.models.networks.modedit.MoDeDiT
+      action_dim: ${datamodule.action_space}
+      goal_dim: ${model.cond_dim}
+      obs_dim: 2048
+      goal_conditioned: true
+      causal: true
+      use_custom_attn_mask: false
+      use_proprio: ${model.use_proprio}
+      state_dim: ${proprio_dims}
+      embed_dim: ${model.latent_dim}
+      n_layers: 10
+      goal_seq_len: 1
+      obs_seq_len: ${obs_seq_len}
+      action_seq_len: ${act_seq_len}
+      embed_pdrob: 0
+      goal_drop: 0.1
+      attn_pdrop: 0.3
+      mlp_pdrop: 0.1
+      n_heads: 8
+      device: ${device}
+      linear_output: true
+      cond_router: true
+      num_experts: 4
+      top_k: 2
+      router_normalize: true
+      use_goal_in_routing: false
+      use_argmax: false
+      use_shared_expert: false
+      use_noise_token_as_input: true
+      init_style: olmoe
+  language_goal:
+    _target_: mode.models.networks.clip_lang_encoder.LangClip
+    _recursive_: false
+    model_name: ${clip_lang_model_name}
+root_data_dir: data/lebai
+lang_folder: lang_clip_resnet50
+vis_clip_model_name: ViT-B/16
+clip_lang_model_name: ViT-B/32
+log_dir: ./logs
+slurm: false
+seed: 421
+device: cuda
+batch_size: 64
+devices: 4
+act_dim: 7
+proprio_dims: 7
+obs_dim: 512
+goal_dim: 512
+obs_seq_len: 1
+act_seq_len: 32
+multistep: 32
+p_last_state: 0
+gen_img_res: 112
+max_epochs: 200
+rollout_lh_skip_epochs: 9
+num_workers: 12
+benchmark_name: lebai_mode
+use_extracted_rel_actions: true
+trainer:
+  devices: ${devices}
+  precision: bf16
+  max_epochs: ${max_epochs}
+  sync_batchnorm: true
+  accelerator: gpu
+  strategy: ddp
+logger:
+  _target_: pytorch_lightning.loggers.WandbLogger
+  save_dir: .
+  name: logger
+  group: mode
+  log_model: false
+  project: ${benchmark_name}
+  entity: yimingwu0
+  id: ???

l10_ds1_only/model_cleaned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a54ed8b02ac954e16a881ed7ab9aca568f551c7fc03e54f178e355c3a74d2e0a
+size 3115403448

l10_ds2_only/config.yaml

@@ -0,0 +1,523 @@
+callbacks:
+  rollout_lh:
+    tasks:
+      _target_: calvin_env.envs.tasks.Tasks
+      tasks:
+        rotate_red_block_right:
+        - rotate_object
+        - block_red
+        - -60
+        rotate_red_block_left:
+        - rotate_object
+        - block_red
+        - 60
+        rotate_blue_block_right:
+        - rotate_object
+        - block_blue
+        - -60
+        rotate_blue_block_left:
+        - rotate_object
+        - block_blue
+        - 60
+        rotate_pink_block_right:
+        - rotate_object
+        - block_pink
+        - -60
+        rotate_pink_block_left:
+        - rotate_object
+        - block_pink
+        - 60
+        push_red_block_right:
+        - push_object
+        - block_red
+        - 0.1
+        - 0
+        push_red_block_left:
+        - push_object
+        - block_red
+        - -0.1
+        - 0
+        push_blue_block_right:
+        - push_object
+        - block_blue
+        - 0.1
+        - 0
+        push_blue_block_left:
+        - push_object
+        - block_blue
+        - -0.1
+        - 0
+        push_pink_block_right:
+        - push_object
+        - block_pink
+        - 0.1
+        - 0
+        push_pink_block_left:
+        - push_object
+        - block_pink
+        - -0.1
+        - 0
+        move_slider_left:
+        - move_door_rel
+        - base__slide
+        - 0.15
+        move_slider_right:
+        - move_door_rel
+        - base__slide
+        - -0.15
+        open_drawer:
+        - move_door_rel
+        - base__drawer
+        - 0.12
+        close_drawer:
+        - move_door_rel
+        - base__drawer
+        - -0.12
+        lift_red_block_table:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - base_link
+        lift_red_block_slider:
+        - lift_object
+        - block_red
+        - 0.03
+        - table
+        - plank_link
+        lift_red_block_drawer:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - drawer_link
+        lift_blue_block_table:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - base_link
+        lift_blue_block_slider:
+        - lift_object
+        - block_blue
+        - 0.03
+        - table
+        - plank_link
+        lift_blue_block_drawer:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - drawer_link
+        lift_pink_block_table:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - base_link
+        lift_pink_block_slider:
+        - lift_object
+        - block_pink
+        - 0.03
+        - table
+        - plank_link
+        lift_pink_block_drawer:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - drawer_link
+        place_in_slider:
+        - place_object
+        - table
+        - plank_link
+        place_in_drawer:
+        - place_object
+        - table
+        - drawer_link
+        stack_block:
+        - stack_objects
+        unstack_block:
+        - unstack_objects
+        turn_on_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 0
+        - 1
+        turn_off_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 1
+        - 0
+        turn_on_led:
+        - toggle_light
+        - led
+        - 0
+        - 1
+        turn_off_led:
+        - toggle_light
+        - led
+        - 1
+        - 0
+        push_into_drawer:
+        - push_object_into
+        - - block_red
+          - block_blue
+          - block_pink
+        - table
+        - base_link
+        - table
+        - drawer_link
+    val_annotations:
+      rotate_red_block_right:
+      - take the red block and rotate it to the right
+      rotate_red_block_left:
+      - take the red block and rotate it to the left
+      rotate_blue_block_right:
+      - take the blue block and rotate it to the right
+      rotate_blue_block_left:
+      - take the blue block and rotate it to the left
+      rotate_pink_block_right:
+      - take the pink block and rotate it to the right
+      rotate_pink_block_left:
+      - take the pink block and rotate it to the left
+      push_red_block_right:
+      - go push the red block right
+      push_red_block_left:
+      - go push the red block left
+      push_blue_block_right:
+      - go push the blue block right
+      push_blue_block_left:
+      - go push the blue block left
+      push_pink_block_right:
+      - go push the pink block right
+      push_pink_block_left:
+      - go push the pink block left
+      move_slider_left:
+      - push the sliding door to the left side
+      move_slider_right:
+      - push the sliding door to the right side
+      open_drawer:
+      - pull the handle to open the drawer
+      close_drawer:
+      - push the handle to close the drawer
+      lift_red_block_table:
+      - grasp and lift the red block
+      lift_blue_block_table:
+      - grasp and lift the blue block
+      lift_pink_block_table:
+      - grasp and lift the pink block
+      lift_red_block_slider:
+      - lift the red block from the sliding cabinet
+      lift_blue_block_slider:
+      - lift the blue block from the sliding cabinet
+      lift_pink_block_slider:
+      - lift the pink block from the sliding cabinet
+      lift_red_block_drawer:
+      - Take the red block from the drawer
+      lift_blue_block_drawer:
+      - Take the blue block from the drawer
+      lift_pink_block_drawer:
+      - Take the pink block from the drawer
+      place_in_slider:
+      - store the grasped block in the sliding cabinet
+      place_in_drawer:
+      - store the grasped block in the drawer
+      push_into_drawer:
+      - slide the block that it falls into the drawer
+      stack_block:
+      - stack the grasped block
+      unstack_block:
+      - remove the stacked block
+      turn_on_lightbulb:
+      - use the switch to turn on the light bulb
+      turn_off_lightbulb:
+      - use the switch to turn off the light bulb
+      turn_on_led:
+      - press the button to turn on the led light
+      turn_off_led:
+      - press the button to turn off the led light
+    _target_: mode.rollout.rollout_long_horizon.RolloutLongHorizon
+    _recursive_: false
+    env_cfg:
+      _target_: mode.wrappers.hulc_wrapper.HulcWrapper
+    skip_epochs: ${rollout_lh_skip_epochs}
+    rollout_freq: 5
+    num_videos: 0
+    num_sequences: 1000
+    replan_freq: 30
+    ep_len: 360
+    empty_cache: false
+    log_video_to_file: false
+    save_dir: ./videos
+    lang_folder: ${lang_folder}
+    debug: false
+  ema:
+    _target_: mode.callbacks.ema.EMA
+    decay: 0.999
+    start_step: 0
+    save_ema_weights_in_callback_state: true
+    evaluate_ema_weights_instead: true
+    power: 0.6666666666666666
+    inv_gamma: 1.0
+    min_value: 0.0
+    max_value: 0.9999
+  checkpoint:
+    _target_: pytorch_lightning.callbacks.ModelCheckpoint
+    verbose: true
+    dirpath: saved_models
+    filename: '{epoch:02d}'
+    every_n_epochs: ${callbacks.rollout_lh.rollout_freq}
+datamodule:
+  transforms:
+    train:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 10
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 4
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+    val:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+  _target_: mode.datasets.hulc_data_module.HulcDataModule
+  _recursive_: false
+  root_data_dir: ${root_data_dir}
+  action_space: 7
+  action_max:
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  action_min:
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1
+  shuffle_val: false
+  observation_space:
+    rgb_obs:
+    - rgb_static
+    - rgb_gripper
+    depth_obs: []
+    state_obs:
+    - robot_obs
+    actions:
+    - rel_actions
+    language:
+    - language
+  proprioception_dims:
+    n_state_obs: 8
+    keep_indices:
+    - - 0
+      - 7
+    - - 14
+      - 15
+    robot_orientation_idx:
+    - 3
+    - 6
+    normalize: true
+    normalize_robot_orientation: true
+  datasets:
+    lang_dataset:
+      _target_: mode.datasets.disk_dataset.ExtendedDiskDataset
+      key: lang
+      save_format: npz
+      batch_size: ${batch_size}
+      min_window_size: ${act_seq_len}
+      max_window_size: ${act_seq_len}
+      proprio_state: ${datamodule.proprioception_dims}
+      obs_space: ${datamodule.observation_space}
+      skip_frames: 1
+      pad: false
+      lang_folder: ${lang_folder}
+      aux_lang_loss_window: 8
+      num_workers: ${num_workers}
+      action_seq_len: ${act_seq_len}
+      obs_seq_len: ${obs_seq_len}
+      future_range: 1
+      use_extracted_rel_actions: ${use_extracted_rel_actions}
+model:
+  _target_: mode.models.mode_agent.MoDEAgent
+  _recursive_: false
+  multistep: ${multistep}
+  use_lr_scheduler: true
+  entropy_gamma: 0.01
+  router_z_delta: 0.0
+  use_proprio: false
+  seed: ${seed}
+  sampler_type: ddim
+  num_sampling_steps: 10
+  sigma_data: 0.5
+  sigma_min: 0.001
+  sigma_max: 80
+  noise_scheduler: exponential
+  sigma_sample_density_type: loglogistic
+  ckpt_path: /home/yiming/pretrained_models/MoDE/MoDE_Pretrained
+  start_from_pretrained: true
+  act_window_size: ${act_seq_len}
+  latent_dim: 1024
+  obs_enc_dim: ${obs_dim}
+  cond_dim: 512
+  resnet_type: '50'
+  optimizer:
+    _target_: torch.optim.AdamW
+    transformer_weight_decay: 0.05
+    obs_encoder_weight_decay: 0.05
+    learning_rate: 0.0001
+    betas:
+    - 0.9
+    - 0.95
+  lr_scheduler:
+    lr_scheduler:
+      init_lr: 0.0001
+      init_lr_scale: 0.1
+      final_lr_scale: 1.0e-06
+      total_steps: 45000
+      phase_ratio: (0.02, 0.08, 0.9)
+      lr: 0.0001
+  model:
+    _target_: mode.models.edm_diffusion.score_wrappers.GCDenoiser
+    _recursive_: false
+    sigma_data: ${model.sigma_data}
+    inner_model:
+      _target_: mode.models.networks.modedit.MoDeDiT
+      action_dim: ${datamodule.action_space}
+      goal_dim: ${model.cond_dim}
+      obs_dim: 2048
+      goal_conditioned: true
+      causal: true
+      use_custom_attn_mask: false
+      use_proprio: ${model.use_proprio}
+      state_dim: ${proprio_dims}
+      embed_dim: ${model.latent_dim}
+      n_layers: 10
+      goal_seq_len: 1
+      obs_seq_len: ${obs_seq_len}
+      action_seq_len: ${act_seq_len}
+      embed_pdrob: 0
+      goal_drop: 0.1
+      attn_pdrop: 0.3
+      mlp_pdrop: 0.1
+      n_heads: 8
+      device: ${device}
+      linear_output: true
+      cond_router: true
+      num_experts: 4
+      top_k: 2
+      router_normalize: true
+      use_goal_in_routing: false
+      use_argmax: false
+      use_shared_expert: false
+      use_noise_token_as_input: true
+      init_style: olmoe
+  language_goal:
+    _target_: mode.models.networks.clip_lang_encoder.LangClip
+    _recursive_: false
+    model_name: ${clip_lang_model_name}
+root_data_dir: data/lebai
+lang_folder: lang_clip_resnet50
+vis_clip_model_name: ViT-B/16
+clip_lang_model_name: ViT-B/32
+log_dir: ./logs
+slurm: false
+seed: 421
+device: cuda
+batch_size: 64
+devices: 4
+act_dim: 7
+proprio_dims: 7
+obs_dim: 512
+goal_dim: 512
+obs_seq_len: 1
+act_seq_len: 32
+multistep: 32
+p_last_state: 0
+gen_img_res: 112
+max_epochs: 200
+rollout_lh_skip_epochs: 9
+num_workers: 12
+benchmark_name: lebai_mode
+use_extracted_rel_actions: true
+trainer:
+  devices: ${devices}
+  precision: bf16
+  max_epochs: ${max_epochs}
+  sync_batchnorm: true
+  accelerator: gpu
+  strategy: ddp
+logger:
+  _target_: pytorch_lightning.loggers.WandbLogger
+  save_dir: .
+  name: logger
+  group: mode
+  log_model: false
+  project: ${benchmark_name}
+  entity: yimingwu0
+  id: ???

l10_ds2_only/model_cleaned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:423414d6b17475736239742ddd7d6c9ecab7df933c9e91a26b01289fa37e7e05
+size 3115403448

l10_ds3_only/config.yaml

@@ -0,0 +1,523 @@
+callbacks:
+  rollout_lh:
+    tasks:
+      _target_: calvin_env.envs.tasks.Tasks
+      tasks:
+        rotate_red_block_right:
+        - rotate_object
+        - block_red
+        - -60
+        rotate_red_block_left:
+        - rotate_object
+        - block_red
+        - 60
+        rotate_blue_block_right:
+        - rotate_object
+        - block_blue
+        - -60
+        rotate_blue_block_left:
+        - rotate_object
+        - block_blue
+        - 60
+        rotate_pink_block_right:
+        - rotate_object
+        - block_pink
+        - -60
+        rotate_pink_block_left:
+        - rotate_object
+        - block_pink
+        - 60
+        push_red_block_right:
+        - push_object
+        - block_red
+        - 0.1
+        - 0
+        push_red_block_left:
+        - push_object
+        - block_red
+        - -0.1
+        - 0
+        push_blue_block_right:
+        - push_object
+        - block_blue
+        - 0.1
+        - 0
+        push_blue_block_left:
+        - push_object
+        - block_blue
+        - -0.1
+        - 0
+        push_pink_block_right:
+        - push_object
+        - block_pink
+        - 0.1
+        - 0
+        push_pink_block_left:
+        - push_object
+        - block_pink
+        - -0.1
+        - 0
+        move_slider_left:
+        - move_door_rel
+        - base__slide
+        - 0.15
+        move_slider_right:
+        - move_door_rel
+        - base__slide
+        - -0.15
+        open_drawer:
+        - move_door_rel
+        - base__drawer
+        - 0.12
+        close_drawer:
+        - move_door_rel
+        - base__drawer
+        - -0.12
+        lift_red_block_table:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - base_link
+        lift_red_block_slider:
+        - lift_object
+        - block_red
+        - 0.03
+        - table
+        - plank_link
+        lift_red_block_drawer:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - drawer_link
+        lift_blue_block_table:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - base_link
+        lift_blue_block_slider:
+        - lift_object
+        - block_blue
+        - 0.03
+        - table
+        - plank_link
+        lift_blue_block_drawer:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - drawer_link
+        lift_pink_block_table:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - base_link
+        lift_pink_block_slider:
+        - lift_object
+        - block_pink
+        - 0.03
+        - table
+        - plank_link
+        lift_pink_block_drawer:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - drawer_link
+        place_in_slider:
+        - place_object
+        - table
+        - plank_link
+        place_in_drawer:
+        - place_object
+        - table
+        - drawer_link
+        stack_block:
+        - stack_objects
+        unstack_block:
+        - unstack_objects
+        turn_on_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 0
+        - 1
+        turn_off_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 1
+        - 0
+        turn_on_led:
+        - toggle_light
+        - led
+        - 0
+        - 1
+        turn_off_led:
+        - toggle_light
+        - led
+        - 1
+        - 0
+        push_into_drawer:
+        - push_object_into
+        - - block_red
+          - block_blue
+          - block_pink
+        - table
+        - base_link
+        - table
+        - drawer_link
+    val_annotations:
+      rotate_red_block_right:
+      - take the red block and rotate it to the right
+      rotate_red_block_left:
+      - take the red block and rotate it to the left
+      rotate_blue_block_right:
+      - take the blue block and rotate it to the right
+      rotate_blue_block_left:
+      - take the blue block and rotate it to the left
+      rotate_pink_block_right:
+      - take the pink block and rotate it to the right
+      rotate_pink_block_left:
+      - take the pink block and rotate it to the left
+      push_red_block_right:
+      - go push the red block right
+      push_red_block_left:
+      - go push the red block left
+      push_blue_block_right:
+      - go push the blue block right
+      push_blue_block_left:
+      - go push the blue block left
+      push_pink_block_right:
+      - go push the pink block right
+      push_pink_block_left:
+      - go push the pink block left
+      move_slider_left:
+      - push the sliding door to the left side
+      move_slider_right:
+      - push the sliding door to the right side
+      open_drawer:
+      - pull the handle to open the drawer
+      close_drawer:
+      - push the handle to close the drawer
+      lift_red_block_table:
+      - grasp and lift the red block
+      lift_blue_block_table:
+      - grasp and lift the blue block
+      lift_pink_block_table:
+      - grasp and lift the pink block
+      lift_red_block_slider:
+      - lift the red block from the sliding cabinet
+      lift_blue_block_slider:
+      - lift the blue block from the sliding cabinet
+      lift_pink_block_slider:
+      - lift the pink block from the sliding cabinet
+      lift_red_block_drawer:
+      - Take the red block from the drawer
+      lift_blue_block_drawer:
+      - Take the blue block from the drawer
+      lift_pink_block_drawer:
+      - Take the pink block from the drawer
+      place_in_slider:
+      - store the grasped block in the sliding cabinet
+      place_in_drawer:
+      - store the grasped block in the drawer
+      push_into_drawer:
+      - slide the block that it falls into the drawer
+      stack_block:
+      - stack the grasped block
+      unstack_block:
+      - remove the stacked block
+      turn_on_lightbulb:
+      - use the switch to turn on the light bulb
+      turn_off_lightbulb:
+      - use the switch to turn off the light bulb
+      turn_on_led:
+      - press the button to turn on the led light
+      turn_off_led:
+      - press the button to turn off the led light
+    _target_: mode.rollout.rollout_long_horizon.RolloutLongHorizon
+    _recursive_: false
+    env_cfg:
+      _target_: mode.wrappers.hulc_wrapper.HulcWrapper
+    skip_epochs: ${rollout_lh_skip_epochs}
+    rollout_freq: 5
+    num_videos: 0
+    num_sequences: 1000
+    replan_freq: 30
+    ep_len: 360
+    empty_cache: false
+    log_video_to_file: false
+    save_dir: ./videos
+    lang_folder: ${lang_folder}
+    debug: false
+  ema:
+    _target_: mode.callbacks.ema.EMA
+    decay: 0.999
+    start_step: 0
+    save_ema_weights_in_callback_state: true
+    evaluate_ema_weights_instead: true
+    power: 0.6666666666666666
+    inv_gamma: 1.0
+    min_value: 0.0
+    max_value: 0.9999
+  checkpoint:
+    _target_: pytorch_lightning.callbacks.ModelCheckpoint
+    verbose: true
+    dirpath: saved_models
+    filename: '{epoch:02d}'
+    every_n_epochs: ${callbacks.rollout_lh.rollout_freq}
+datamodule:
+  transforms:
+    train:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 10
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 4
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+    val:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+  _target_: mode.datasets.hulc_data_module.HulcDataModule
+  _recursive_: false
+  root_data_dir: ${root_data_dir}
+  action_space: 7
+  action_max:
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  action_min:
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1
+  shuffle_val: false
+  observation_space:
+    rgb_obs:
+    - rgb_static
+    - rgb_gripper
+    depth_obs: []
+    state_obs:
+    - robot_obs
+    actions:
+    - rel_actions
+    language:
+    - language
+  proprioception_dims:
+    n_state_obs: 8
+    keep_indices:
+    - - 0
+      - 7
+    - - 14
+      - 15
+    robot_orientation_idx:
+    - 3
+    - 6
+    normalize: true
+    normalize_robot_orientation: true
+  datasets:
+    lang_dataset:
+      _target_: mode.datasets.disk_dataset.ExtendedDiskDataset
+      key: lang
+      save_format: npz
+      batch_size: ${batch_size}
+      min_window_size: ${act_seq_len}
+      max_window_size: ${act_seq_len}
+      proprio_state: ${datamodule.proprioception_dims}
+      obs_space: ${datamodule.observation_space}
+      skip_frames: 1
+      pad: false
+      lang_folder: ${lang_folder}
+      aux_lang_loss_window: 8
+      num_workers: ${num_workers}
+      action_seq_len: ${act_seq_len}
+      obs_seq_len: ${obs_seq_len}
+      future_range: 1
+      use_extracted_rel_actions: ${use_extracted_rel_actions}
+model:
+  _target_: mode.models.mode_agent.MoDEAgent
+  _recursive_: false
+  multistep: ${multistep}
+  use_lr_scheduler: true
+  entropy_gamma: 0.01
+  router_z_delta: 0.0
+  use_proprio: false
+  seed: ${seed}
+  sampler_type: ddim
+  num_sampling_steps: 10
+  sigma_data: 0.5
+  sigma_min: 0.001
+  sigma_max: 80
+  noise_scheduler: exponential
+  sigma_sample_density_type: loglogistic
+  ckpt_path: /home/yiming/pretrained_models/MoDE/MoDE_Pretrained
+  start_from_pretrained: true
+  act_window_size: ${act_seq_len}
+  latent_dim: 1024
+  obs_enc_dim: ${obs_dim}
+  cond_dim: 512
+  resnet_type: '50'
+  optimizer:
+    _target_: torch.optim.AdamW
+    transformer_weight_decay: 0.05
+    obs_encoder_weight_decay: 0.05
+    learning_rate: 0.0001
+    betas:
+    - 0.9
+    - 0.95
+  lr_scheduler:
+    lr_scheduler:
+      init_lr: 0.0001
+      init_lr_scale: 0.1
+      final_lr_scale: 1.0e-06
+      total_steps: 45000
+      phase_ratio: (0.02, 0.08, 0.9)
+      lr: 0.0001
+  model:
+    _target_: mode.models.edm_diffusion.score_wrappers.GCDenoiser
+    _recursive_: false
+    sigma_data: ${model.sigma_data}
+    inner_model:
+      _target_: mode.models.networks.modedit.MoDeDiT
+      action_dim: ${datamodule.action_space}
+      goal_dim: ${model.cond_dim}
+      obs_dim: 2048
+      goal_conditioned: true
+      causal: true
+      use_custom_attn_mask: false
+      use_proprio: ${model.use_proprio}
+      state_dim: ${proprio_dims}
+      embed_dim: ${model.latent_dim}
+      n_layers: 10
+      goal_seq_len: 1
+      obs_seq_len: ${obs_seq_len}
+      action_seq_len: ${act_seq_len}
+      embed_pdrob: 0
+      goal_drop: 0.1
+      attn_pdrop: 0.3
+      mlp_pdrop: 0.1
+      n_heads: 8
+      device: ${device}
+      linear_output: true
+      cond_router: true
+      num_experts: 4
+      top_k: 2
+      router_normalize: true
+      use_goal_in_routing: false
+      use_argmax: false
+      use_shared_expert: false
+      use_noise_token_as_input: true
+      init_style: olmoe
+  language_goal:
+    _target_: mode.models.networks.clip_lang_encoder.LangClip
+    _recursive_: false
+    model_name: ${clip_lang_model_name}
+root_data_dir: data/lebai
+lang_folder: lang_clip_resnet50
+vis_clip_model_name: ViT-B/16
+clip_lang_model_name: ViT-B/32
+log_dir: ./logs
+slurm: false
+seed: 421
+device: cuda
+batch_size: 64
+devices: 4
+act_dim: 7
+proprio_dims: 7
+obs_dim: 512
+goal_dim: 512
+obs_seq_len: 1
+act_seq_len: 32
+multistep: 32
+p_last_state: 0
+gen_img_res: 112
+max_epochs: 200
+rollout_lh_skip_epochs: 9
+num_workers: 12
+benchmark_name: lebai_mode
+use_extracted_rel_actions: true
+trainer:
+  devices: ${devices}
+  precision: bf16
+  max_epochs: ${max_epochs}
+  sync_batchnorm: true
+  accelerator: gpu
+  strategy: ddp
+logger:
+  _target_: pytorch_lightning.loggers.WandbLogger
+  save_dir: .
+  name: logger
+  group: mode
+  log_model: false
+  project: ${benchmark_name}
+  entity: yimingwu0
+  id: ???

l10_ds3_only/model_cleaned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e94672c75395ac5ac3a1cf7ffdf0ffcd8fdde763ce58523d629ea11c657c59c
+size 3115403448

l10_ds4_only/config.yaml

@@ -0,0 +1,523 @@
+callbacks:
+  rollout_lh:
+    tasks:
+      _target_: calvin_env.envs.tasks.Tasks
+      tasks:
+        rotate_red_block_right:
+        - rotate_object
+        - block_red
+        - -60
+        rotate_red_block_left:
+        - rotate_object
+        - block_red
+        - 60
+        rotate_blue_block_right:
+        - rotate_object
+        - block_blue
+        - -60
+        rotate_blue_block_left:
+        - rotate_object
+        - block_blue
+        - 60
+        rotate_pink_block_right:
+        - rotate_object
+        - block_pink
+        - -60
+        rotate_pink_block_left:
+        - rotate_object
+        - block_pink
+        - 60
+        push_red_block_right:
+        - push_object
+        - block_red
+        - 0.1
+        - 0
+        push_red_block_left:
+        - push_object
+        - block_red
+        - -0.1
+        - 0
+        push_blue_block_right:
+        - push_object
+        - block_blue
+        - 0.1
+        - 0
+        push_blue_block_left:
+        - push_object
+        - block_blue
+        - -0.1
+        - 0
+        push_pink_block_right:
+        - push_object
+        - block_pink
+        - 0.1
+        - 0
+        push_pink_block_left:
+        - push_object
+        - block_pink
+        - -0.1
+        - 0
+        move_slider_left:
+        - move_door_rel
+        - base__slide
+        - 0.15
+        move_slider_right:
+        - move_door_rel
+        - base__slide
+        - -0.15
+        open_drawer:
+        - move_door_rel
+        - base__drawer
+        - 0.12
+        close_drawer:
+        - move_door_rel
+        - base__drawer
+        - -0.12
+        lift_red_block_table:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - base_link
+        lift_red_block_slider:
+        - lift_object
+        - block_red
+        - 0.03
+        - table
+        - plank_link
+        lift_red_block_drawer:
+        - lift_object
+        - block_red
+        - 0.05
+        - table
+        - drawer_link
+        lift_blue_block_table:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - base_link
+        lift_blue_block_slider:
+        - lift_object
+        - block_blue
+        - 0.03
+        - table
+        - plank_link
+        lift_blue_block_drawer:
+        - lift_object
+        - block_blue
+        - 0.05
+        - table
+        - drawer_link
+        lift_pink_block_table:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - base_link
+        lift_pink_block_slider:
+        - lift_object
+        - block_pink
+        - 0.03
+        - table
+        - plank_link
+        lift_pink_block_drawer:
+        - lift_object
+        - block_pink
+        - 0.05
+        - table
+        - drawer_link
+        place_in_slider:
+        - place_object
+        - table
+        - plank_link
+        place_in_drawer:
+        - place_object
+        - table
+        - drawer_link
+        stack_block:
+        - stack_objects
+        unstack_block:
+        - unstack_objects
+        turn_on_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 0
+        - 1
+        turn_off_lightbulb:
+        - toggle_light
+        - lightbulb
+        - 1
+        - 0
+        turn_on_led:
+        - toggle_light
+        - led
+        - 0
+        - 1
+        turn_off_led:
+        - toggle_light
+        - led
+        - 1
+        - 0
+        push_into_drawer:
+        - push_object_into
+        - - block_red
+          - block_blue
+          - block_pink
+        - table
+        - base_link
+        - table
+        - drawer_link
+    val_annotations:
+      rotate_red_block_right:
+      - take the red block and rotate it to the right
+      rotate_red_block_left:
+      - take the red block and rotate it to the left
+      rotate_blue_block_right:
+      - take the blue block and rotate it to the right
+      rotate_blue_block_left:
+      - take the blue block and rotate it to the left
+      rotate_pink_block_right:
+      - take the pink block and rotate it to the right
+      rotate_pink_block_left:
+      - take the pink block and rotate it to the left
+      push_red_block_right:
+      - go push the red block right
+      push_red_block_left:
+      - go push the red block left
+      push_blue_block_right:
+      - go push the blue block right
+      push_blue_block_left:
+      - go push the blue block left
+      push_pink_block_right:
+      - go push the pink block right
+      push_pink_block_left:
+      - go push the pink block left
+      move_slider_left:
+      - push the sliding door to the left side
+      move_slider_right:
+      - push the sliding door to the right side
+      open_drawer:
+      - pull the handle to open the drawer
+      close_drawer:
+      - push the handle to close the drawer
+      lift_red_block_table:
+      - grasp and lift the red block
+      lift_blue_block_table:
+      - grasp and lift the blue block
+      lift_pink_block_table:
+      - grasp and lift the pink block
+      lift_red_block_slider:
+      - lift the red block from the sliding cabinet
+      lift_blue_block_slider:
+      - lift the blue block from the sliding cabinet
+      lift_pink_block_slider:
+      - lift the pink block from the sliding cabinet
+      lift_red_block_drawer:
+      - Take the red block from the drawer
+      lift_blue_block_drawer:
+      - Take the blue block from the drawer
+      lift_pink_block_drawer:
+      - Take the pink block from the drawer
+      place_in_slider:
+      - store the grasped block in the sliding cabinet
+      place_in_drawer:
+      - store the grasped block in the drawer
+      push_into_drawer:
+      - slide the block that it falls into the drawer
+      stack_block:
+      - stack the grasped block
+      unstack_block:
+      - remove the stacked block
+      turn_on_lightbulb:
+      - use the switch to turn on the light bulb
+      turn_off_lightbulb:
+      - use the switch to turn off the light bulb
+      turn_on_led:
+      - press the button to turn on the led light
+      turn_off_led:
+      - press the button to turn off the led light
+    _target_: mode.rollout.rollout_long_horizon.RolloutLongHorizon
+    _recursive_: false
+    env_cfg:
+      _target_: mode.wrappers.hulc_wrapper.HulcWrapper
+    skip_epochs: ${rollout_lh_skip_epochs}
+    rollout_freq: 5
+    num_videos: 0
+    num_sequences: 1000
+    replan_freq: 30
+    ep_len: 360
+    empty_cache: false
+    log_video_to_file: false
+    save_dir: ./videos
+    lang_folder: ${lang_folder}
+    debug: false
+  ema:
+    _target_: mode.callbacks.ema.EMA
+    decay: 0.999
+    start_step: 0
+    save_ema_weights_in_callback_state: true
+    evaluate_ema_weights_instead: true
+    power: 0.6666666666666666
+    inv_gamma: 1.0
+    min_value: 0.0
+    max_value: 0.9999
+  checkpoint:
+    _target_: pytorch_lightning.callbacks.ModelCheckpoint
+    verbose: true
+    dirpath: saved_models
+    filename: '{epoch:02d}'
+    every_n_epochs: ${callbacks.rollout_lh.rollout_freq}
+datamodule:
+  transforms:
+    train:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 10
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.RandomShiftsAug
+        pad: 4
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+    val:
+      rgb_static:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      rgb_gripper:
+      - _target_: torchvision.transforms.Resize
+        size: 224
+        antialias: true
+      - _target_: mode.utils.transforms.ScaleImageTensor
+      - _target_: torchvision.transforms.Normalize
+        mean:
+        - 0.48145466
+        - 0.4578275
+        - 0.40821073
+        std:
+        - 0.26862954
+        - 0.26130258
+        - 0.27577711
+      robot_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+      scene_obs:
+      - _target_: mode.utils.transforms.NormalizeVector
+  _target_: mode.datasets.hulc_data_module.HulcDataModule
+  _recursive_: false
+  root_data_dir: ${root_data_dir}
+  action_space: 7
+  action_max:
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  - 1.0
+  action_min:
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1.0
+  - -1
+  shuffle_val: false
+  observation_space:
+    rgb_obs:
+    - rgb_static
+    - rgb_gripper
+    depth_obs: []
+    state_obs:
+    - robot_obs
+    actions:
+    - rel_actions
+    language:
+    - language
+  proprioception_dims:
+    n_state_obs: 8
+    keep_indices:
+    - - 0
+      - 7
+    - - 14
+      - 15
+    robot_orientation_idx:
+    - 3
+    - 6
+    normalize: true
+    normalize_robot_orientation: true
+  datasets:
+    lang_dataset:
+      _target_: mode.datasets.disk_dataset.ExtendedDiskDataset
+      key: lang
+      save_format: npz
+      batch_size: ${batch_size}
+      min_window_size: ${act_seq_len}
+      max_window_size: ${act_seq_len}
+      proprio_state: ${datamodule.proprioception_dims}
+      obs_space: ${datamodule.observation_space}
+      skip_frames: 1
+      pad: false
+      lang_folder: ${lang_folder}
+      aux_lang_loss_window: 8
+      num_workers: ${num_workers}
+      action_seq_len: ${act_seq_len}
+      obs_seq_len: ${obs_seq_len}
+      future_range: 1
+      use_extracted_rel_actions: ${use_extracted_rel_actions}
+model:
+  _target_: mode.models.mode_agent.MoDEAgent
+  _recursive_: false
+  multistep: ${multistep}
+  use_lr_scheduler: true
+  entropy_gamma: 0.01
+  router_z_delta: 0.0
+  use_proprio: false
+  seed: ${seed}
+  sampler_type: ddim
+  num_sampling_steps: 10
+  sigma_data: 0.5
+  sigma_min: 0.001
+  sigma_max: 80
+  noise_scheduler: exponential
+  sigma_sample_density_type: loglogistic
+  ckpt_path: /home/yiming/pretrained_models/MoDE/MoDE_Pretrained
+  start_from_pretrained: true
+  act_window_size: ${act_seq_len}
+  latent_dim: 1024
+  obs_enc_dim: ${obs_dim}
+  cond_dim: 512
+  resnet_type: '50'
+  optimizer:
+    _target_: torch.optim.AdamW
+    transformer_weight_decay: 0.05
+    obs_encoder_weight_decay: 0.05
+    learning_rate: 0.0001
+    betas:
+    - 0.9
+    - 0.95
+  lr_scheduler:
+    lr_scheduler:
+      init_lr: 0.0001
+      init_lr_scale: 0.1
+      final_lr_scale: 1.0e-06
+      total_steps: 45000
+      phase_ratio: (0.02, 0.08, 0.9)
+      lr: 0.0001
+  model:
+    _target_: mode.models.edm_diffusion.score_wrappers.GCDenoiser
+    _recursive_: false
+    sigma_data: ${model.sigma_data}
+    inner_model:
+      _target_: mode.models.networks.modedit.MoDeDiT
+      action_dim: ${datamodule.action_space}
+      goal_dim: ${model.cond_dim}
+      obs_dim: 2048
+      goal_conditioned: true
+      causal: true
+      use_custom_attn_mask: false
+      use_proprio: ${model.use_proprio}
+      state_dim: ${proprio_dims}
+      embed_dim: ${model.latent_dim}
+      n_layers: 10
+      goal_seq_len: 1
+      obs_seq_len: ${obs_seq_len}
+      action_seq_len: ${act_seq_len}
+      embed_pdrob: 0
+      goal_drop: 0.1
+      attn_pdrop: 0.3
+      mlp_pdrop: 0.1
+      n_heads: 8
+      device: ${device}
+      linear_output: true
+      cond_router: true
+      num_experts: 4
+      top_k: 2
+      router_normalize: true
+      use_goal_in_routing: false
+      use_argmax: false
+      use_shared_expert: false
+      use_noise_token_as_input: true
+      init_style: olmoe
+  language_goal:
+    _target_: mode.models.networks.clip_lang_encoder.LangClip
+    _recursive_: false
+    model_name: ${clip_lang_model_name}
+root_data_dir: data/lebai
+lang_folder: lang_clip_resnet50
+vis_clip_model_name: ViT-B/16
+clip_lang_model_name: ViT-B/32
+log_dir: ./logs
+slurm: false
+seed: 421
+device: cuda
+batch_size: 64
+devices: 4
+act_dim: 7
+proprio_dims: 7
+obs_dim: 512
+goal_dim: 512
+obs_seq_len: 1
+act_seq_len: 32
+multistep: 32
+p_last_state: 0
+gen_img_res: 112
+max_epochs: 200
+rollout_lh_skip_epochs: 9
+num_workers: 12
+benchmark_name: lebai_mode
+use_extracted_rel_actions: true
+trainer:
+  devices: ${devices}
+  precision: bf16
+  max_epochs: ${max_epochs}
+  sync_batchnorm: true
+  accelerator: gpu
+  strategy: ddp
+logger:
+  _target_: pytorch_lightning.loggers.WandbLogger
+  save_dir: .
+  name: logger
+  group: mode
+  log_model: false
+  project: ${benchmark_name}
+  entity: yimingwu0
+  id: ???

l10_ds4_only/model_cleaned.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ced8bd34e8b00519126bbc65294c011373003cb19f70953346009e34fdc2c14
+size 3115403448