Add UniLACT model weights of 3 stages for calvin

unilact_finetuned_on_calvin/config.yaml

@@ -0,0 +1,37 @@
+_target_: unilact.src.models.unilact.UniLACT
+model_lang:
+  _target_: transformers.T5EncoderModel.from_pretrained
+  pretrained_model_name_or_path: t5-base
+model_vision:
+  _target_: unilact.src.models.mae_model.MaeEncoder
+  use_obs_feature: true
+  pretrained_model_name_or_path: facebook/vit-mae-large
+model_causal_transformer:
+  _target_: unilact.src.models.trajectory_gpt2.GPT2Model
+  config:
+    _target_: unilact.src.models.trajectory_gpt2.GPT2Config
+    vocab_size: 1
+    n_embd: 768
+    n_layer: 12
+    n_head: 12
+    activation_function: relu
+    dropout: 0.1
+    n_positions: 1024
+act_dim: 7
+hidden_size: 768
+sequence_length: 2
+chunk_size: 5
+per_latent_motion_len: 8
+latent_motion_codebook_size: 128
+latent_motion_pred: true
+act_pred: true
+img_feat_dim: 1024
+patch_feat_dim: 1024
+lang_feat_dim: 768
+mask_latent_motion_probability: 0.5
+freeze_lang: true
+freeze_vision: true
+use_latent_motion_pos_embedding: true
+pred_tokens_modality: unified
+output_modality_tokens: same-modal
+

unilact_finetuned_on_calvin/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c7319ea0a4ab81e93467da59003b8926884180d5f936eb86fe2c7b467463944
+size 364206482

unilact_pretrained_on_calvin/config.yaml

@@ -0,0 +1,36 @@
+_target_: unilact.src.models.unilact.UniLACT
+model_lang:
+  _target_: transformers.T5EncoderModel.from_pretrained
+  pretrained_model_name_or_path: t5-base
+model_vision:
+  _target_: unilact.src.models.mae_model.MaeEncoder
+  use_obs_feature: true
+  pretrained_model_name_or_path: facebook/vit-mae-large
+model_causal_transformer:
+  _target_: unilact.src.models.trajectory_gpt2.GPT2Model
+  config:
+    _target_: unilact.src.models.trajectory_gpt2.GPT2Config
+    vocab_size: 1
+    n_embd: 768
+    n_layer: 12
+    n_head: 12
+    activation_function: relu
+    dropout: 0.1
+    n_positions: 1024
+act_dim: 7
+hidden_size: 768
+sequence_length: 2
+chunk_size: 5
+per_latent_motion_len: 8
+latent_motion_codebook_size: 128
+latent_motion_pred: true
+act_pred: true
+img_feat_dim: 1024
+patch_feat_dim: 1024
+lang_feat_dim: 768
+mask_latent_motion_probability: 0.5
+freeze_lang: true
+freeze_vision: true
+use_latent_motion_pos_embedding: true
+pred_tokens_modality: unified
+output_modality_tokens: cross-modal

unilact_pretrained_on_calvin/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28af0ea019db6ad36e6c109582afd97f95f0b14f819b097182ea1431a1e0e064
+size 364172042

unilarn_trained_on_calvin/config.yaml

@@ -0,0 +1,105 @@
+_target_: unilarn.src.models.unilarn.UniLARN
+codebook_dim: 32
+commit_loss_w: 1.0
+recon_loss_w: 1.0
+recon_depth_loss_w: 1.0
+perceptual_loss_w: 1.0
+image_encoder:
+  _target_: transformers.ViTMAEModel.from_pretrained
+  pretrained_model_name_or_path: facebook/vit-mae-large
+m_former:
+  _target_: unilarn.src.models.m_former.MFormer
+  add_pooling_layer: false
+  config:
+    _target_: transformers.ViTConfig
+    query_num: 8
+    input_hidden_size: 1024
+    num_patches: 197
+    attention_probs_dropout_prob: 0.0
+    hidden_act: gelu
+    hidden_dropout_prob: 0.0
+    hidden_size: 768
+    initializer_range: 0.02
+    intermediate_size: 3072
+    layer_norm_eps: 1.0e-12
+    model_type: vit
+    num_attention_heads: 12
+    num_hidden_layers: 4
+    qkv_bias: true
+m_former_depth:
+  _target_: unilarn.src.models.m_former.MFormer
+  add_pooling_layer: false
+  config:
+    _target_: transformers.ViTConfig
+    query_num: 8
+    input_hidden_size: 1024
+    num_patches: 197
+    attention_probs_dropout_prob: 0.0
+    hidden_act: gelu
+    hidden_dropout_prob: 0.0
+    hidden_size: 768
+    initializer_range: 0.02
+    intermediate_size: 3072
+    layer_norm_eps: 1.0e-12
+    model_type: vit
+    num_attention_heads: 12
+    num_hidden_layers: 4
+    qkv_bias: true
+vector_quantizer:
+  _target_: unilarn.src.models.vector_quantizer.VectorQuantizer2
+  n_e: 128
+  e_dim: 32
+  beta: 0.25
+  remap: null
+  sane_index_shape: true
+  legacy: false
+vector_quantizer_uni:
+  _target_: unilarn.src.models.vector_quantizer.VectorQuantizer2
+  n_e: 128
+  e_dim: 32
+  beta: 0.25
+  remap: null
+  sane_index_shape: true
+  legacy: false
+decoder:
+  _target_: unilarn.src.models.latent_action_decoder.LatentActionDecoder
+  config:
+    _target_: transformers.ViTConfig
+    query_num: 8
+    attention_probs_dropout_prob: 0.0
+    hidden_act: gelu
+    hidden_dropout_prob: 0.0
+    hidden_size: 768
+    image_size: 224
+    initializer_range: 0.02
+    intermediate_size: 3072
+    layer_norm_eps: 1.0e-12
+    model_type: vit
+    num_attention_heads: 12
+    num_channels: 3
+    num_hidden_layers: 12
+    patch_size: 16
+    qkv_bias: true
+    encoder_stride: 16
+    num_patches: 196
+depth_decoder:
+  _target_: unilarn.src.models.latent_action_decoder.LatentActionDecoder
+  config:
+    _target_: transformers.ViTConfig
+    query_num: 8
+    attention_probs_dropout_prob: 0.0
+    hidden_act: gelu
+    hidden_dropout_prob: 0.0
+    hidden_size: 768
+    image_size: 224
+    initializer_range: 0.02
+    intermediate_size: 3072
+    layer_norm_eps: 1.0e-12
+    model_type: vit
+    num_attention_heads: 12
+    num_channels: 3
+    num_hidden_layers: 12
+    patch_size: 16
+    qkv_bias: true
+    encoder_stride: 16
+    num_patches: 196

unilarn_trained_on_calvin/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5c64ce3b2833687157a598d37e362aa80a578445ee14d69ad827b6a7524c294
+size 967328706