Upload folder using huggingface_hub

checkpoint-25000/config.json

@@ -0,0 +1,70 @@
+{
+  "action_horizon": 50,
+  "add_pos_embed": true,
+  "apply_sincos_state_encoding": true,
+  "architectures": [
+    "Gr00tN1d6"
+  ],
+  "attn_dropout": 0.2,
+  "attn_implementation": null,
+  "backbone_embedding_dim": 2048,
+  "backbone_model_type": "eagle",
+  "backbone_trainable_params_fp32": true,
+  "collator_overwrite_image_inputs": false,
+  "color_jitter_params": {
+    "brightness": 0.1,
+    "contrast": 0.1,
+    "hue": 0.1,
+    "saturation": 0.1
+  },
+  "crop_fraction": 0.95,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "eagle_collator": true,
+  "formalize_language": true,
+  "gemma_collator": false,
+  "hidden_size": 1024,
+  "image_crop_size": null,
+  "image_target_size": null,
+  "input_embedding_dim": 1536,
+  "load_bf16": true,
+  "max_action_dim": 128,
+  "max_num_embodiments": 32,
+  "max_seq_len": 1024,
+  "max_state_dim": 128,
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "model_type": "Gr00tN1d6",
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_inference_timesteps": 4,
+  "num_timestep_buckets": 1000,
+  "random_rotation_angle": null,
+  "reproject_vision": false,
+  "select_layer": 16,
+  "shortest_image_edge": 256,
+  "state_dropout_prob": 0.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "tune_diffusion_model": true,
+  "tune_llm": false,
+  "tune_projector": true,
+  "tune_top_llm_layers": 4,
+  "tune_visual": false,
+  "tune_vlln": true,
+  "use_albumentations_transforms": true,
+  "use_alternate_vl_dit": true,
+  "use_flash_attention": true,
+  "use_relative_action": true,
+  "use_vlln": true
+}

checkpoint-25000/embodiment_id.json

@@ -0,0 +1,11 @@
+{
+  "robocasa_panda_omron": 13,
+  "gr1": 20,
+  "behavior_r1_pro": 24,
+  "unitree_g1": 8,
+  "oxe_google": 0,
+  "oxe_widowx": 1,
+  "libero_panda": 2,
+  "oxe_droid": 16,
+  "new_embodiment": 10
+}

checkpoint-25000/experiment_cfg/conf.yaml

@@ -0,0 +1,234 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d6
+  model_dtype: bfloat16
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  backbone_model_type: eagle
+  model_revision: null
+  tune_top_llm_layers: 4
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 16
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  collator_overwrite_image_inputs: false
+  eagle_collator: true
+  backbone_trainable_params_fp32: true
+  image_crop_size: null
+  image_target_size: null
+  shortest_image_edge: 256
+  crop_fraction: 0.95
+  random_rotation_angle: null
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    saturation: 0.5
+    hue: 0.08
+  use_albumentations_transforms: true
+  extra_augmentation_config: null
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_relative_action: true
+  max_state_dim: 29
+  max_action_dim: 29
+  action_horizon: 16
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 32
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.0
+  state_additive_noise_scale: 0.0
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - gubam/upper_body_dataset
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - front
+        - left_wrist
+        - right_wrist
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - arm
+        - left_hand
+        - right_hand
+        sin_cos_embedding_keys:
+        - arm
+        - left_hand
+        - right_hand
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        - 16
+        - 17
+        - 18
+        - 19
+        - 20
+        - 21
+        - 22
+        - 23
+        - 24
+        - 25
+        - 26
+        - 27
+        - 28
+        - 29
+        modality_keys:
+        - arm
+        - left_hand
+        - right_hand
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: fork
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: torchcodec
+training:
+  output_dir: /tmp/dual_arm_finetune
+  experiment_name: null
+  max_steps: 50000
+  global_batch_size: 64
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 25000
+  save_total_limit: 5
+  save_vl_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 4
+  remove_unused_columns: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 50000
+save_steps: 25000

checkpoint-25000/experiment_cfg/config.yaml

@@ -0,0 +1,268 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - gubam/upper_body_dataset
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: &id002 !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: &id003 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: *id002
+          state_key: null
+          type: *id003
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: *id002
+          state_key: null
+          type: *id003
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        - 16
+        - 17
+        - 18
+        - 19
+        - 20
+        - 21
+        - 22
+        - 23
+        - 24
+        - 25
+        - 26
+        - 27
+        - 28
+        - 29
+        mean_std_embedding_keys: null
+        modality_keys:
+        - arm
+        - left_hand
+        - right_hand
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - arm
+        - left_hand
+        - right_hand
+        sin_cos_embedding_keys:
+        - arm
+        - left_hand
+        - right_hand
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - front
+        - left_wrist
+        - right_wrist
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: fork
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  random_chop: 0.0
+  seed: 42
+  shard_size: 1024
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: torchcodec
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
+  _attn_implementation_autoset: false
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  add_cross_attention: false
+  architectures: null
+  backbone_model_type: eagle
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    hue: 0.08
+    saturation: 0.5
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 32
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  eagle_collator: true
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  extra_augmentation_config: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_attentions: false
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.0
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torch_dtype: null
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 4
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 64
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 50000
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: /tmp/dual_arm_finetune
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_steps: 25000
+  save_total_limit: 5
+  save_vl_model: false
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

checkpoint-25000/experiment_cfg/dataset_statistics.json

@@ -0,0 +1,813 @@
+{
+  "new_embodiment": {
+    "state": {
+      "arm": {
+        "min": [
+          -0.0006981316837482154,
+          -0.0024434609804302454,
+          -0.6288421154022217,
+          -0.44854962825775146,
+          -0.8210029006004333,
+          -0.4991641640663147,
+          -0.09669124335050583,
+          -0.6223844289779663,
+          -0.9733701348304749,
+          -0.5129522681236267,
+          -0.47246062755584717,
+          -0.47822022438049316,
+          -0.8032005429267883,
+          -0.7274532318115234,
+          -1.6929693222045898,
+          -1.0246827602386475,
+          -0.9484119415283203,
+          -1.373574137687683,
+          -0.4761258065700531,
+          -0.47368237376213074
+        ],
+        "max": [
+          0.0008726646192371845,
+          0.0006981316837482154,
+          0.7929030656814575,
+          0.3269001543521881,
+          0.9124581217765808,
+          0.8902924656867981,
+          1.63694429397583,
+          0.9662142992019653,
+          0.45553094148635864,
+          1.445656180381775,
+          0.47455501556396484,
+          0.4740314185619354,
+          0.829380452632904,
+          0.8866272568702698,
+          0.016406094655394554,
+          0.6817256212234497,
+          1.0342820882797241,
+          1.0231120586395264,
+          0.47333329916000366,
+          0.47682395577430725
+        ],
+        "mean": [
+          -8.076138594063484e-07,
+          3.0537906135411266e-06,
+          0.004025201443343724,
+          -0.106839506396076,
+          0.012814986898180149,
+          0.15896875719206852,
+          0.8534196868931058,
+          0.06762093321189754,
+          -0.3314590204826914,
+          0.2632455375793913,
+          0.020509420259097345,
+          -0.14299405403307847,
+          -0.0037216641098350048,
+          -0.1443416739101096,
+          -0.9090582326583341,
+          -0.09771556978633221,
+          0.36533875231778595,
+          -0.23848672985559247,
+          -0.16839635328002986,
+          0.15968955264953888
+        ],
+        "std": [
+          6.68305329907084e-05,
+          4.278830258104584e-05,
+          0.27632803159057195,
+          0.1031391849687569,
+          0.21593103530511093,
+          0.1426352780284767,
+          0.17564987399430124,
+          0.15910994986378815,
+          0.18504214171125488,
+          0.30606451766910636,
+          0.1554592124135248,
+          0.1859134005343795,
+          0.23041502031790836,
+          0.17109378851688062,
+          0.20105957075942452,
+          0.18465144901511993,
+          0.20598322974819286,
+          0.37752503742465104,
+          0.16176032456796732,
+          0.209982125025823
+        ],
+        "q01": [
+          -0.0001761727144185956,
+          -6.812984705377427e-05,
+          -0.34380334710819843,
+          -0.29353822405261265,
+          -0.49171140765602955,
+          -0.11508113798319357,
+          0.5739165571398043,
+          -0.22209883528953628,
+          -0.6382970698194925,
+          -0.05048877088467583,
+          -0.23948595344457352,
+          -0.41341574397836467,
+          -0.33359782573626867,
+          -0.44144052869891365,
+          -1.2649022016051807,
+          -0.5335335171843635,
+          -0.06932329291282342,
+          -0.8823546013980487,
+          -0.42073663519315196,
+          -0.2189616747621502
+        ],
+        "q99": [
+          0.00017598653218951375,
+          0.00015066502308079557,
+          0.3422460626808077,
+          0.056641715341579135,
+          0.3719052899183461,
+          0.4368372476473694,
+          1.153966983550112,
+          0.40466381992586953,
+          0.01223556216846413,
+          0.7870821352051268,
+          0.2707992157297519,
+          0.16422090043397727,
+          0.5579769587902337,
+          0.24133049901128434,
+          -0.5478116844161816,
+          0.21787840519530197,
+          0.7216934483725967,
+          0.15448134992693266,
+          0.14274281318251142,
+          0.4380456959063586
+        ]
+      },
+      "left_hand": {
+        "min": [
+          -0.860447347164154,
+          0.7120943069458008,
+          -0.891863226890564,
+          -1.027998924255371,
+          -0.1727875918149948,
+          0.012217304669320583,
+          -0.15533430874347687,
+          0.024434609338641167,
+          -0.15882496535778046,
+          -0.04014257341623306,
+          -0.16406095027923584,
+          0.01745329238474369,
+          -0.4084070324897766,
+          0.0,
+          -0.1500983089208603,
+          0.010471975430846214,
+          -0.06457718461751938,
+          -0.595157265663147,
+          0.008726646192371845,
+          0.13788101077079773
+        ],
+        "max": [
+          0.04014257341623306,
+          1.83085036277771,
+          -0.3543018400669098,
+          0.12566371262073517,
+          0.5567600131034851,
+          1.5062191486358643,
+          1.4521139860153198,
+          1.5446163415908813,
+          0.4921828508377075,
+          1.2967796325683594,
+          1.4974925518035889,
+          1.586504340171814,
+          0.22689279913902283,
+          1.3561208248138428,
+          1.4451326131820679,
+          1.576032280921936,
+          0.04363323003053665,
+          -0.04014257341623306,
+          1.577777624130249,
+          1.530653715133667
+        ],
+        "mean": [
+          -0.12916538234236632,
+          1.376141627845568,
+          -0.5974346987153767,
+          -0.04435670103580047,
+          0.3636791117819629,
+          0.5902737730081836,
+          0.033071112851852894,
+          0.533722619804661,
+          0.23935603508223233,
+          0.26795747388310004,
+          0.05880051231688046,
+          0.4106006244211684,
+          -0.07247953411303715,
+          0.32536608217859136,
+          0.021850320537598826,
+          0.48054924670620924,
+          0.004586551841409656,
+          -0.2872830319123344,
+          0.5324850869755637,
+          0.22302741732397693
+        ],
+        "std": [
+          0.1820096848837831,
+          0.18821082103179884,
+          0.0383618674714932,
+          0.08580352115504818,
+          0.10266044655069544,
+          0.27960173403679445,
+          0.1448563942110651,
+          0.25444040043363414,
+          0.10732763838871918,
+          0.18261099083302232,
+          0.17370513521519468,
+          0.2977325844852106,
+          0.0849824454436069,
+          0.22911564056001565,
+          0.08484813391378729,
+          0.34918716865543786,
+          0.015126139619470597,
+          0.07222469670516816,
+          0.31120811629260187,
+          0.04733751780159246
+        ],
+        "q01": [
+          -0.6203178651596793,
+          1.1651488640586132,
+          -0.663073371187542,
+          -0.29948327686771475,
+          0.24480558767964009,
+          0.27953559750985485,
+          -0.019133575525358288,
+          0.29321194070801376,
+          0.08405627958517456,
+          0.08698530973701081,
+          0.008375168461385862,
+          0.14015780065990607,
+          -0.20499186387054075,
+          0.10807116195011923,
+          -0.03593066740546433,
+          0.17861930482664712,
+          -0.0004471278064463094,
+          -0.3945708347189374,
+          0.24163943781341096,
+          0.2088678994773458
+        ],
+        "q99": [
+          -0.028091832495861904,
+          1.6592481415891205,
+          -0.5521809752762913,
+          -0.004371203922586993,
+          0.42714076269815554,
+          1.1774596617898176,
+          0.3954404905500605,
+          1.1347713118060374,
+          0.35540928239208186,
+          0.7117942088312333,
+          0.559904130222523,
+          1.1689669955521202,
+          0.03642433268748423,
+          0.8575968036182136,
+          0.2410983404943445,
+          1.3105606222443886,
+          0.014554555685431566,
+          -0.20818513323287732,
+          1.1365133019580054,
+          0.30657573611436517
+        ]
+      },
+      "right_hand": {
+        "min": [
+          -0.07853981852531433,
+          -1.8116517066955566,
+          0.23038345575332642,
+          -0.08552113175392151,
+          -0.5183627605438232,
+          0.04886921867728233,
+          -0.09599310904741287,
+          0.024434609338641167,
+          -0.5899212956428528,
+          0.03490658476948738,
+          -0.1186823919415474,
+          0.024434609338641167,
+          -0.3176499307155609,
+          0.02967059798538685,
+          -0.13439035415649414,
+          0.02617993950843811,
+          -0.05061454698443413,
+          -0.09773843735456467,
+          -0.024434609338641167,
+          0.10471975803375244
+        ],
+        "max": [
+          0.942477822303772,
+          -0.5672320127487183,
+          0.8726646304130554,
+          1.146681308746338,
+          0.0,
+          1.4014993906021118,
+          1.1013027429580688,
+          1.5585789680480957,
+          -0.06632251292467117,
+          1.242674469947815,
+          1.5603243112564087,
+          1.5446163415908813,
+          0.21991148591041565,
+          1.3159782886505127,
+          1.4241886138916016,
+          1.6144295930862427,
+          0.03141592815518379,
+          0.5759586691856384,
+          1.6109389066696167,
+          1.5446163415908813
+        ],
+        "mean": [
+          0.08732205124532119,
+          -1.3515142157311884,
+          0.5860485981188814,
+          0.04222940470260995,
+          -0.39225604013486204,
+          0.6816943204695467,
+          -0.0023892602428072745,
+          0.603416635597419,
+          -0.3610242297308094,
+          0.432317803054292,
+          0.04194121809078737,
+          0.4891484012380831,
+          -0.127616240047779,
+          0.45217231200710073,
+          0.013140720890466806,
+          0.4938726423320787,
+          -0.014045313832414617,
+          0.15257342129823098,
+          0.6583795979024015,
+          0.2257495313911657
+        ],
+        "std": [
+          0.1341312685567186,
+          0.1682050627347457,
+          0.04645222806632484,
+          0.08697156707075793,
+          0.031928398427410444,
+          0.20537231601266742,
+          0.05530703896109472,
+          0.25054327807997007,
+          0.0720470225370238,
+          0.1736117631333828,
+          0.1204903079020455,
+          0.24655711144811424,
+          0.09497679476716676,
+          0.20499188829579684,
+          0.06714406151269925,
+          0.29066084684137816,
+          0.009477716402105218,
+          0.08723463811303951,
+          0.33394118029044084,
+          0.05208327357938751
+        ],
+        "q01": [
+          0.024307096659506287,
+          -1.6502210851484855,
+          0.45876371711080155,
+          -0.0035277651640500955,
+          -0.4210949303662783,
+          0.37951074431192705,
+          -0.035692522713118,
+          0.3506586314254852,
+          -0.4516104227248592,
+          0.1631463266031545,
+          0.005253287352257153,
+          0.20386044314240565,
+          -0.24894458779100756,
+          0.15880754921289522,
+          -0.04149282519228719,
+          0.20258045443353387,
+          -0.03787891490303466,
+          0.06077176788457635,
+          0.3162963808837116,
+          0.20361250086550506
+        ],
+        "q99": [
+          0.4870936258809745,
+          -1.1706684983420805,
+          0.6307717980752034,
+          0.2800037986993521,
+          -0.3476623605953785,
+          1.1034270282608116,
+          0.11763696399923158,
+          1.2489453907495742,
+          -0.2544253695010915,
+          0.7977079944176352,
+          0.3566995199885687,
+          1.1253466726580794,
+          0.005086851467515925,
+          0.8733681377683882,
+          0.15090692671068542,
+          1.2984744094470615,
+          -0.009331353694428703,
+          0.2587334180468299,
+          1.3355227665003735,
+          0.3072850506793767
+        ]
+      }
+    },
+    "action": {
+      "arm": {
+        "min": [
+          -0.0006981316837482154,
+          -0.0024434609804302454,
+          -0.7850000262260437,
+          -0.6247848272323608,
+          -0.842519998550415,
+          -0.4991641640663147,
+          -0.0979316234588623,
+          -0.6220138072967529,
+          -0.973471462726593,
+          -0.5152764916419983,
+          -0.49396848678588867,
+          -0.4999072253704071,
+          -0.8028976917266846,
+          -0.7391713857650757,
+          -1.715409755706787,
+          -1.0240248441696167,
+          -0.9464260339736938,
+          -1.3702261447906494,
+          -0.4958377480506897,
+          -0.4955917298793793
+        ],
+        "max": [
+          0.0008726646192371845,
+          0.0,
+          0.7850000262260437,
+          0.29835808277130127,
+          0.9119161367416382,
+          0.890291154384613,
+          1.644073724746704,
+          0.9660546183586121,
+          0.45562389492988586,
+          1.4430302381515503,
+          0.4950156807899475,
+          0.4944920837879181,
+          0.8447622060775757,
+          0.8822340369224548,
+          0.015839435160160065,
+          0.6863510012626648,
+          1.0327516794204712,
+          1.0249639749526978,
+          0.49615368247032166,
+          0.4964403212070465
+        ],
+        "mean": [
+          -7.565100935672921e-07,
+          -2.734424040564876e-06,
+          0.0015625113303062233,
+          -0.10770792895105542,
+          0.012797537757365607,
+          0.15919185826406693,
+          0.853652544767167,
+          0.06761602762960633,
+          -0.3316022136769494,
+          0.26350797662337844,
+          0.02061022363664793,
+          -0.14381767943396778,
+          -0.0035584753122770074,
+          -0.14450589130256247,
+          -0.9093130011371935,
+          -0.0978520963516777,
+          0.36546998229542166,
+          -0.23875873233105468,
+          -0.16929286973811716,
+          0.1614393169093547
+        ],
+        "std": [
+          6.6392213185449e-05,
+          2.7477303136889642e-05,
+          0.3236236855006889,
+          0.10498868324818088,
+          0.21592920231663346,
+          0.14242412024634826,
+          0.17537817653960341,
+          0.1589983448423053,
+          0.1847030522707091,
+          0.3060052894278398,
+          0.15545421217912833,
+          0.18732387928520802,
+          0.23097992612678261,
+          0.17079917723668858,
+          0.20074672089177023,
+          0.18450774051586757,
+          0.20560809014181267,
+          0.3775058422894344,
+          0.16285387660315898,
+          0.21299280533759732
+        ],
+        "q01": [
+          -0.0001749494229231163,
+          -7.102986238156782e-05,
+          -0.4576713352188768,
+          -0.30153348134798175,
+          -0.4925865405961751,
+          -0.11370973105044871,
+          0.575421578469159,
+          -0.2209467398567723,
+          -0.6372816928992551,
+          -0.050352534999616734,
+          -0.23822213308286633,
+          -0.4218674707222603,
+          -0.33289762926446964,
+          -0.4403506284535931,
+          -1.2638898242176249,
+          -0.5333060165767463,
+          -0.06842082932288861,
+          -0.8808569036987537,
+          -0.43036151802551864,
+          -0.22260962031806678
+        ],
+        "q99": [
+          0.00017484051565752354,
+          -3.4853625995277704e-10,
+          0.4395555218474595,
+          0.05241236958600787,
+          0.3711243260578636,
+          0.43583597900427307,
+          1.1524254208812423,
+          0.4036845933510512,
+          0.010783664846300164,
+          0.7864205056015764,
+          0.27070234831315787,
+          0.1641843369874286,
+          0.5632010381667683,
+          0.23991706572733013,
+          -0.548851543258573,
+          0.21723416177116134,
+          0.7202919613199235,
+          0.15411611751854068,
+          0.142010680673695,
+          0.45282141791847796
+        ]
+      },
+      "left_hand": {
+        "min": [
+          -0.880666971206665,
+          0.6509807705879211,
+          -0.9295765161514282,
+          -1.0574641227722168,
+          -0.6108652353286743,
+          0.0,
+          0.0,
+          0.0,
+          -0.17246204614639282,
+          0.0,
+          0.0,
+          0.0,
+          -0.41887903213500977,
+          0.0,
+          0.0,
+          0.0,
+          -0.0,
+          -0.6108652353286743,
+          0.0,
+          0.18421649932861328
+        ],
+        "max": [
+          0.0,
+          1.8240469694137573,
+          -0.4169214367866516,
+          0.0,
+          0.41887903213500977,
+          1.5967257022857666,
+          1.5707963705062866,
+          1.5707963705062866,
+          0.6108652353286743,
+          1.3334362506866455,
+          1.5707963705062866,
+          1.5707963705062866,
+          0.2761457860469818,
+          1.474050521850586,
+          1.5047622919082642,
+          1.5707963705062866,
+          -0.0,
+          -0.013026786036789417,
+          1.5707963705062866,
+          1.5707963705062866
+        ],
+        "mean": [
+          -0.11197184536473949,
+          1.3519521447728844,
+          -0.6145068955473336,
+          -0.04490551318122404,
+          0.3812392963674508,
+          0.5928953152018989,
+          0.028781154436057742,
+          0.533942906643585,
+          0.2586826909204934,
+          0.24856345721893203,
+          0.04199277436774596,
+          0.40243144011328036,
+          -0.05372014356756498,
+          0.32290915338039355,
+          0.028910841144483782,
+          0.4829249856261449,
+          0.0,
+          -0.2736987368992572,
+          0.5229504504255549,
+          0.2163480956489075
+        ],
+        "std": [
+          0.18078946912307625,
+          0.19270362935123467,
+          0.059722650226918514,
+          0.13456131846867891,
+          0.12495907901258241,
+          0.30308983822349966,
+          0.15583292538180302,
+          0.2616750261097083,
+          0.1082498098717154,
+          0.21063265328615333,
+          0.20388137333506792,
+          0.3069878052972663,
+          0.08657551602472234,
+          0.28322161741911617,
+          0.11428067004348502,
+          0.3659270477492105,
+          0.0,
+          0.08003530419489042,
+          0.3308618144426684,
+          0.047378584156049966
+        ],
+        "q01": [
+          -0.5874023409507655,
+          1.1259081133842275,
+          -0.7722857806858863,
+          -0.46699307730378525,
+          0.2166453155186744,
+          0.2500263838689038,
+          -6.69347151562168e-11,
+          0.2810358129914362,
+          0.08265342079552167,
+          0.03212275062917445,
+          -7.422571247960103e-11,
+          0.11523256281603016,
+          -0.20857053595062272,
+          0.06834987345413719,
+          0.0032137934620088096,
+          0.16083063598248018,
+          3.9995784462121275e-16,
+          -0.41066407554491824,
+          0.21280464706121974,
+          0.20159717992290016
+        ],
+        "q99": [
+          -0.004032147496030359,
+          1.6196350682050786,
+          -0.5529140781257509,
+          -0.00022676876993767606,
+          0.4188780675074272,
+          1.2361410219097821,
+          0.4266783342163138,
+          1.1679802915746502,
+          0.3961736551386673,
+          0.7744282902748555,
+          0.6498963336012552,
+          1.195352623563556,
+          0.06817064976497338,
+          1.0339126384354067,
+          0.3498183122489737,
+          1.3578909040686071,
+          3.959582661750007e-14,
+          -0.18808869776308895,
+          1.2038159253048966,
+          0.30760064823760574
+        ]
+      },
+      "right_hand": {
+        "min": [
+          0.0,
+          -1.7954039573669434,
+          0.4210455119609833,
+          0.0,
+          -0.41887903213500977,
+          0.0,
+          0.0,
+          0.0,
+          -0.6108652353286743,
+          0.0,
+          0.0,
+          0.0,
+          -0.4958767592906952,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          -0.12086009234189987,
+          0.0,
+          0.19107787311077118
+        ],
+        "max": [
+          0.8792693018913269,
+          -0.5100259184837341,
+          1.5707963705062866,
+          1.4147123098373413,
+          0.28721120953559875,
+          1.5164011716842651,
+          1.1974519491195679,
+          1.5707963705062866,
+          -0.04774348810315132,
+          1.4103626012802124,
+          1.5707963705062866,
+          1.5707963705062866,
+          0.24290113151073456,
+          1.5230220556259155,
+          1.5707963705062866,
+          1.5707963705062866,
+          0.0,
+          0.5971487760543823,
+          1.5707963705062866,
+          1.5707963705062866
+        ],
+        "mean": [
+          0.06154122902723653,
+          -1.3294183619600155,
+          0.6250821797211654,
+          0.05554929311166913,
+          -0.41492612799882095,
+          0.6856338848469389,
+          0.006940385029546583,
+          0.6091391652412862,
+          -0.37595000803726775,
+          0.422836852505782,
+          0.025116036945811486,
+          0.4918787185539051,
+          -0.14475854791832374,
+          0.46284594344482655,
+          0.027444823626726584,
+          0.5093871628244654,
+          0.0,
+          0.1401173905144233,
+          0.6521685371675148,
+          0.2238743152068344
+        ],
+        "std": [
+          0.13674527190111965,
+          0.17102490658313696,
+          0.05741908683253429,
+          0.14382058266604184,
+          0.02688551022749764,
+          0.2281908323797236,
+          0.05783917003853192,
+          0.26035589698369127,
+          0.07511741685767037,
+          0.18406048799000968,
+          0.14373912752045906,
+          0.2479658881484305,
+          0.09645055710684194,
+          0.26237633130190713,
+          0.10678798651992859,
+          0.3103495174470505,
+          0.0,
+          0.08664956776847747,
+          0.3581149787301811,
+          0.05026933816117329
+        ],
+        "q01": [
+          0.0009769272041044426,
+          -1.6190670990502298,
+          0.5518429740664715,
+          0.0013591393093312651,
+          -0.41887903213500977,
+          0.3491621230894666,
+          -4.421230354850494e-11,
+          0.3471016557019131,
+          -0.48352879670754245,
+          0.11780533016338945,
+          -7.137328803777223e-11,
+          0.19241114988176986,
+          -0.29701005092982646,
+          0.1185908435022493,
+          0.0034413322477957684,
+          0.19206370624490998,
+          3.9995784462121275e-16,
+          0.03772304600026963,
+          0.2884605865309452,
+          0.21015920588283557
+        ],
+        "q99": [
+          0.47407716229998736,
+          -1.1360530624353573,
+          0.7761147162999562,
+          0.5139581911771537,
+          -0.36667831242864685,
+          1.2000692036298772,
+          0.14459797121303056,
+          1.2871256631356942,
+          -0.25034097642726927,
+          0.8421005002108355,
+          0.4358143292303531,
+          1.1489288891981229,
+          0.006749831496495902,
+          1.1029886603257073,
+          0.340557722147255,
+          1.399224397253747,
+          3.959582661750007e-14,
+          0.2650572054718265,
+          1.4308893815770474,
+          0.31706126604163526
+        ]
+      }
+    },
+    "relative_action": {}
+  }
+}

checkpoint-25000/experiment_cfg/final_model_config.json

@@ -0,0 +1,54 @@
+{
+  "model_type": "Gr00tN1d6",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "backbone_model_type": "eagle",
+  "model_revision": null,
+  "tune_top_llm_layers": 4,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": true,
+  "collator_overwrite_image_inputs": false,
+  "eagle_collator": true,
+  "backbone_trainable_params_fp32": true,
+  "extra_augmentation_config": null,
+  "apply_sincos_state_encoding": true,
+  "use_relative_action": true,
+  "max_state_dim": 128,
+  "max_action_dim": 128,
+  "action_horizon": 50,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.0,
+  "state_additive_noise_scale": 0.0,
+  "max_num_embodiments": 32
+}

checkpoint-25000/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ec81200d3254ebc88c95e59f11c6e19aea1ed7d05d1bed9c5fd0c30c4a4b051
+size 4990120184

checkpoint-25000/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0abec41214a57f6d4541f14a26da82413d52751f8839db303bb1cc883f32ac59
+size 4823190320

checkpoint-25000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42ccb90fb9e56f5c705ce277a5211c335b1fdb27444ca8d34516f7258de6cea1
+size 12960193762

checkpoint-25000/processor_config.json

@@ -0,0 +1,482 @@
+{
+  "processor_class": "Gr00tN1d6Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "behavior_r1_pro": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "observation.images.rgb.head_256_256",
+            "observation.images.rgb.left_wrist_256_256",
+            "observation.images.rgb.right_wrist_256_256"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "robot_pos",
+            "robot_ori_cos",
+            "robot_ori_sin",
+            "robot_2d_ori",
+            "robot_2d_ori_cos",
+            "robot_2d_ori_sin",
+            "robot_lin_vel",
+            "robot_ang_vel",
+            "arm_left_qpos",
+            "arm_left_qpos_sin",
+            "arm_left_qpos_cos",
+            "eef_left_pos",
+            "eef_left_quat",
+            "gripper_left_qpos",
+            "arm_right_qpos",
+            "arm_right_qpos_sin",
+            "arm_right_qpos_cos",
+            "eef_right_pos",
+            "eef_right_quat",
+            "gripper_right_qpos",
+            "trunk_qpos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31
+          ],
+          "modality_keys": [
+            "base",
+            "torso",
+            "left_arm",
+            "left_gripper",
+            "right_arm",
+            "right_gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "trunk_qpos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_left_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_right_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "gr1": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "ego_view_bg_crop_pad_res256_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "robocasa_panda_omron": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "res256_image_side_0",
+            "res256_image_side_1",
+            "res256_image_wrist_0"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "end_effector_position_relative",
+            "end_effector_rotation_relative",
+            "gripper_qpos",
+            "base_position",
+            "base_rotation"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "end_effector_position",
+            "end_effector_rotation",
+            "gripper_close",
+            "base_motion",
+            "control_mode"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.action.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "front",
+            "left_wrist",
+            "right_wrist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "arm",
+            "left_hand",
+            "right_hand"
+          ],
+          "sin_cos_embedding_keys": [
+            "arm",
+            "left_hand",
+            "right_hand"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29
+          ],
+          "modality_keys": [
+            "arm",
+            "left_hand",
+            "right_hand"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": null,
+    "image_target_size": null,
+    "use_albumentations": true,
+    "random_rotation_angle": null,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "model_name": "nvidia/Eagle-Block2A-2B-v2",
+    "model_type": "eagle",
+    "formalize_language": true,
+    "max_state_dim": 128,
+    "max_action_dim": 128,
+    "max_action_horizon": 50,
+    "use_percentiles": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": true,
+    "use_relative_action": true
+  }
+}

checkpoint-25000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f3382948b12d9872782a99c90040245832772d775c4710d481910b48b4a7ecb
+size 14645

checkpoint-25000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f675ed237e65de27ab43c95dbf5b8adbc50df3fb5101341c5e4c48249e0a9af
+size 1465

checkpoint-25000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eaf5a805ee93e1a7a31c36e844e8c8d1a26ed080f42b6a25c957500318121651
+size 5713

checkpoint-25000/wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d6", "run_id": "dual_arm_finetune"}

experiment_cfg/conf.yaml

@@ -0,0 +1,234 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d6
+  model_dtype: bfloat16
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  backbone_model_type: eagle
+  model_revision: null
+  tune_top_llm_layers: 4
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 16
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  collator_overwrite_image_inputs: false
+  eagle_collator: true
+  backbone_trainable_params_fp32: true
+  image_crop_size: null
+  image_target_size: null
+  shortest_image_edge: 256
+  crop_fraction: 0.95
+  random_rotation_angle: null
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    saturation: 0.5
+    hue: 0.08
+  use_albumentations_transforms: true
+  extra_augmentation_config: null
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_relative_action: true
+  max_state_dim: 29
+  max_action_dim: 29
+  action_horizon: 16
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 32
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.0
+  state_additive_noise_scale: 0.0
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - gubam/upper_body_dataset
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - front
+        - left_wrist
+        - right_wrist
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - arm
+        - left_hand
+        - right_hand
+        sin_cos_embedding_keys:
+        - arm
+        - left_hand
+        - right_hand
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        - 16
+        - 17
+        - 18
+        - 19
+        - 20
+        - 21
+        - 22
+        - 23
+        - 24
+        - 25
+        - 26
+        - 27
+        - 28
+        - 29
+        modality_keys:
+        - arm
+        - left_hand
+        - right_hand
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: fork
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: torchcodec
+training:
+  output_dir: /tmp/dual_arm_finetune
+  experiment_name: null
+  max_steps: 50000
+  global_batch_size: 64
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 25000
+  save_total_limit: 5
+  save_vl_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 4
+  remove_unused_columns: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 50000
+save_steps: 25000

experiment_cfg/config.yaml

@@ -0,0 +1,268 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - gubam/upper_body_dataset
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: &id002 !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: &id003 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: *id002
+          state_key: null
+          type: *id003
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: *id002
+          state_key: null
+          type: *id003
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        - 16
+        - 17
+        - 18
+        - 19
+        - 20
+        - 21
+        - 22
+        - 23
+        - 24
+        - 25
+        - 26
+        - 27
+        - 28
+        - 29
+        mean_std_embedding_keys: null
+        modality_keys:
+        - arm
+        - left_hand
+        - right_hand
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - arm
+        - left_hand
+        - right_hand
+        sin_cos_embedding_keys:
+        - arm
+        - left_hand
+        - right_hand
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - front
+        - left_wrist
+        - right_wrist
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: fork
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  random_chop: 0.0
+  seed: 42
+  shard_size: 1024
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: torchcodec
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
+  _attn_implementation_autoset: false
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  add_cross_attention: false
+  architectures: null
+  backbone_model_type: eagle
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    hue: 0.08
+    saturation: 0.5
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 32
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  eagle_collator: true
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  extra_augmentation_config: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_attentions: false
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.0
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torch_dtype: null
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 4
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 64
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 50000
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: /tmp/dual_arm_finetune
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_steps: 25000
+  save_total_limit: 5
+  save_vl_model: false
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

experiment_cfg/dataset_statistics.json

@@ -0,0 +1,813 @@
+{
+  "new_embodiment": {
+    "state": {
+      "arm": {
+        "min": [
+          -0.0006981316837482154,
+          -0.0024434609804302454,
+          -0.6288421154022217,
+          -0.44854962825775146,
+          -0.8210029006004333,
+          -0.4991641640663147,
+          -0.09669124335050583,
+          -0.6223844289779663,
+          -0.9733701348304749,
+          -0.5129522681236267,
+          -0.47246062755584717,
+          -0.47822022438049316,
+          -0.8032005429267883,
+          -0.7274532318115234,
+          -1.6929693222045898,
+          -1.0246827602386475,
+          -0.9484119415283203,
+          -1.373574137687683,
+          -0.4761258065700531,
+          -0.47368237376213074
+        ],
+        "max": [
+          0.0008726646192371845,
+          0.0006981316837482154,
+          0.7929030656814575,
+          0.3269001543521881,
+          0.9124581217765808,
+          0.8902924656867981,
+          1.63694429397583,
+          0.9662142992019653,
+          0.45553094148635864,
+          1.445656180381775,
+          0.47455501556396484,
+          0.4740314185619354,
+          0.829380452632904,
+          0.8866272568702698,
+          0.016406094655394554,
+          0.6817256212234497,
+          1.0342820882797241,
+          1.0231120586395264,
+          0.47333329916000366,
+          0.47682395577430725
+        ],
+        "mean": [
+          -8.076138594063484e-07,
+          3.0537906135411266e-06,
+          0.004025201443343724,
+          -0.106839506396076,
+          0.012814986898180149,
+          0.15896875719206852,
+          0.8534196868931058,
+          0.06762093321189754,
+          -0.3314590204826914,
+          0.2632455375793913,
+          0.020509420259097345,
+          -0.14299405403307847,
+          -0.0037216641098350048,
+          -0.1443416739101096,
+          -0.9090582326583341,
+          -0.09771556978633221,
+          0.36533875231778595,
+          -0.23848672985559247,
+          -0.16839635328002986,
+          0.15968955264953888
+        ],
+        "std": [
+          6.68305329907084e-05,
+          4.278830258104584e-05,
+          0.27632803159057195,
+          0.1031391849687569,
+          0.21593103530511093,
+          0.1426352780284767,
+          0.17564987399430124,
+          0.15910994986378815,
+          0.18504214171125488,
+          0.30606451766910636,
+          0.1554592124135248,
+          0.1859134005343795,
+          0.23041502031790836,
+          0.17109378851688062,
+          0.20105957075942452,
+          0.18465144901511993,
+          0.20598322974819286,
+          0.37752503742465104,
+          0.16176032456796732,
+          0.209982125025823
+        ],
+        "q01": [
+          -0.0001761727144185956,
+          -6.812984705377427e-05,
+          -0.34380334710819843,
+          -0.29353822405261265,
+          -0.49171140765602955,
+          -0.11508113798319357,
+          0.5739165571398043,
+          -0.22209883528953628,
+          -0.6382970698194925,
+          -0.05048877088467583,
+          -0.23948595344457352,
+          -0.41341574397836467,
+          -0.33359782573626867,
+          -0.44144052869891365,
+          -1.2649022016051807,
+          -0.5335335171843635,
+          -0.06932329291282342,
+          -0.8823546013980487,
+          -0.42073663519315196,
+          -0.2189616747621502
+        ],
+        "q99": [
+          0.00017598653218951375,
+          0.00015066502308079557,
+          0.3422460626808077,
+          0.056641715341579135,
+          0.3719052899183461,
+          0.4368372476473694,
+          1.153966983550112,
+          0.40466381992586953,
+          0.01223556216846413,
+          0.7870821352051268,
+          0.2707992157297519,
+          0.16422090043397727,
+          0.5579769587902337,
+          0.24133049901128434,
+          -0.5478116844161816,
+          0.21787840519530197,
+          0.7216934483725967,
+          0.15448134992693266,
+          0.14274281318251142,
+          0.4380456959063586
+        ]
+      },
+      "left_hand": {
+        "min": [
+          -0.860447347164154,
+          0.7120943069458008,
+          -0.891863226890564,
+          -1.027998924255371,
+          -0.1727875918149948,
+          0.012217304669320583,
+          -0.15533430874347687,
+          0.024434609338641167,
+          -0.15882496535778046,
+          -0.04014257341623306,
+          -0.16406095027923584,
+          0.01745329238474369,
+          -0.4084070324897766,
+          0.0,
+          -0.1500983089208603,
+          0.010471975430846214,
+          -0.06457718461751938,
+          -0.595157265663147,
+          0.008726646192371845,
+          0.13788101077079773
+        ],
+        "max": [
+          0.04014257341623306,
+          1.83085036277771,
+          -0.3543018400669098,
+          0.12566371262073517,
+          0.5567600131034851,
+          1.5062191486358643,
+          1.4521139860153198,
+          1.5446163415908813,
+          0.4921828508377075,
+          1.2967796325683594,
+          1.4974925518035889,
+          1.586504340171814,
+          0.22689279913902283,
+          1.3561208248138428,
+          1.4451326131820679,
+          1.576032280921936,
+          0.04363323003053665,
+          -0.04014257341623306,
+          1.577777624130249,
+          1.530653715133667
+        ],
+        "mean": [
+          -0.12916538234236632,
+          1.376141627845568,
+          -0.5974346987153767,
+          -0.04435670103580047,
+          0.3636791117819629,
+          0.5902737730081836,
+          0.033071112851852894,
+          0.533722619804661,
+          0.23935603508223233,
+          0.26795747388310004,
+          0.05880051231688046,
+          0.4106006244211684,
+          -0.07247953411303715,
+          0.32536608217859136,
+          0.021850320537598826,
+          0.48054924670620924,
+          0.004586551841409656,
+          -0.2872830319123344,
+          0.5324850869755637,
+          0.22302741732397693
+        ],
+        "std": [
+          0.1820096848837831,
+          0.18821082103179884,
+          0.0383618674714932,
+          0.08580352115504818,
+          0.10266044655069544,
+          0.27960173403679445,
+          0.1448563942110651,
+          0.25444040043363414,
+          0.10732763838871918,
+          0.18261099083302232,
+          0.17370513521519468,
+          0.2977325844852106,
+          0.0849824454436069,
+          0.22911564056001565,
+          0.08484813391378729,
+          0.34918716865543786,
+          0.015126139619470597,
+          0.07222469670516816,
+          0.31120811629260187,
+          0.04733751780159246
+        ],
+        "q01": [
+          -0.6203178651596793,
+          1.1651488640586132,
+          -0.663073371187542,
+          -0.29948327686771475,
+          0.24480558767964009,
+          0.27953559750985485,
+          -0.019133575525358288,
+          0.29321194070801376,
+          0.08405627958517456,
+          0.08698530973701081,
+          0.008375168461385862,
+          0.14015780065990607,
+          -0.20499186387054075,
+          0.10807116195011923,
+          -0.03593066740546433,
+          0.17861930482664712,
+          -0.0004471278064463094,
+          -0.3945708347189374,
+          0.24163943781341096,
+          0.2088678994773458
+        ],
+        "q99": [
+          -0.028091832495861904,
+          1.6592481415891205,
+          -0.5521809752762913,
+          -0.004371203922586993,
+          0.42714076269815554,
+          1.1774596617898176,
+          0.3954404905500605,
+          1.1347713118060374,
+          0.35540928239208186,
+          0.7117942088312333,
+          0.559904130222523,
+          1.1689669955521202,
+          0.03642433268748423,
+          0.8575968036182136,
+          0.2410983404943445,
+          1.3105606222443886,
+          0.014554555685431566,
+          -0.20818513323287732,
+          1.1365133019580054,
+          0.30657573611436517
+        ]
+      },
+      "right_hand": {
+        "min": [
+          -0.07853981852531433,
+          -1.8116517066955566,
+          0.23038345575332642,
+          -0.08552113175392151,
+          -0.5183627605438232,
+          0.04886921867728233,
+          -0.09599310904741287,
+          0.024434609338641167,
+          -0.5899212956428528,
+          0.03490658476948738,
+          -0.1186823919415474,
+          0.024434609338641167,
+          -0.3176499307155609,
+          0.02967059798538685,
+          -0.13439035415649414,
+          0.02617993950843811,
+          -0.05061454698443413,
+          -0.09773843735456467,
+          -0.024434609338641167,
+          0.10471975803375244
+        ],
+        "max": [
+          0.942477822303772,
+          -0.5672320127487183,
+          0.8726646304130554,
+          1.146681308746338,
+          0.0,
+          1.4014993906021118,
+          1.1013027429580688,
+          1.5585789680480957,
+          -0.06632251292467117,
+          1.242674469947815,
+          1.5603243112564087,
+          1.5446163415908813,
+          0.21991148591041565,
+          1.3159782886505127,
+          1.4241886138916016,
+          1.6144295930862427,
+          0.03141592815518379,
+          0.5759586691856384,
+          1.6109389066696167,
+          1.5446163415908813
+        ],
+        "mean": [
+          0.08732205124532119,
+          -1.3515142157311884,
+          0.5860485981188814,
+          0.04222940470260995,
+          -0.39225604013486204,
+          0.6816943204695467,
+          -0.0023892602428072745,
+          0.603416635597419,
+          -0.3610242297308094,
+          0.432317803054292,
+          0.04194121809078737,
+          0.4891484012380831,
+          -0.127616240047779,
+          0.45217231200710073,
+          0.013140720890466806,
+          0.4938726423320787,
+          -0.014045313832414617,
+          0.15257342129823098,
+          0.6583795979024015,
+          0.2257495313911657
+        ],
+        "std": [
+          0.1341312685567186,
+          0.1682050627347457,
+          0.04645222806632484,
+          0.08697156707075793,
+          0.031928398427410444,
+          0.20537231601266742,
+          0.05530703896109472,
+          0.25054327807997007,
+          0.0720470225370238,
+          0.1736117631333828,
+          0.1204903079020455,
+          0.24655711144811424,
+          0.09497679476716676,
+          0.20499188829579684,
+          0.06714406151269925,
+          0.29066084684137816,
+          0.009477716402105218,
+          0.08723463811303951,
+          0.33394118029044084,
+          0.05208327357938751
+        ],
+        "q01": [
+          0.024307096659506287,
+          -1.6502210851484855,
+          0.45876371711080155,
+          -0.0035277651640500955,
+          -0.4210949303662783,
+          0.37951074431192705,
+          -0.035692522713118,
+          0.3506586314254852,
+          -0.4516104227248592,
+          0.1631463266031545,
+          0.005253287352257153,
+          0.20386044314240565,
+          -0.24894458779100756,
+          0.15880754921289522,
+          -0.04149282519228719,
+          0.20258045443353387,
+          -0.03787891490303466,
+          0.06077176788457635,
+          0.3162963808837116,
+          0.20361250086550506
+        ],
+        "q99": [
+          0.4870936258809745,
+          -1.1706684983420805,
+          0.6307717980752034,
+          0.2800037986993521,
+          -0.3476623605953785,
+          1.1034270282608116,
+          0.11763696399923158,
+          1.2489453907495742,
+          -0.2544253695010915,
+          0.7977079944176352,
+          0.3566995199885687,
+          1.1253466726580794,
+          0.005086851467515925,
+          0.8733681377683882,
+          0.15090692671068542,
+          1.2984744094470615,
+          -0.009331353694428703,
+          0.2587334180468299,
+          1.3355227665003735,
+          0.3072850506793767
+        ]
+      }
+    },
+    "action": {
+      "arm": {
+        "min": [
+          -0.0006981316837482154,
+          -0.0024434609804302454,
+          -0.7850000262260437,
+          -0.6247848272323608,
+          -0.842519998550415,
+          -0.4991641640663147,
+          -0.0979316234588623,
+          -0.6220138072967529,
+          -0.973471462726593,
+          -0.5152764916419983,
+          -0.49396848678588867,
+          -0.4999072253704071,
+          -0.8028976917266846,
+          -0.7391713857650757,
+          -1.715409755706787,
+          -1.0240248441696167,
+          -0.9464260339736938,
+          -1.3702261447906494,
+          -0.4958377480506897,
+          -0.4955917298793793
+        ],
+        "max": [
+          0.0008726646192371845,
+          0.0,
+          0.7850000262260437,
+          0.29835808277130127,
+          0.9119161367416382,
+          0.890291154384613,
+          1.644073724746704,
+          0.9660546183586121,
+          0.45562389492988586,
+          1.4430302381515503,
+          0.4950156807899475,
+          0.4944920837879181,
+          0.8447622060775757,
+          0.8822340369224548,
+          0.015839435160160065,
+          0.6863510012626648,
+          1.0327516794204712,
+          1.0249639749526978,
+          0.49615368247032166,
+          0.4964403212070465
+        ],
+        "mean": [
+          -7.565100935672921e-07,
+          -2.734424040564876e-06,
+          0.0015625113303062233,
+          -0.10770792895105542,
+          0.012797537757365607,
+          0.15919185826406693,
+          0.853652544767167,
+          0.06761602762960633,
+          -0.3316022136769494,
+          0.26350797662337844,
+          0.02061022363664793,
+          -0.14381767943396778,
+          -0.0035584753122770074,
+          -0.14450589130256247,
+          -0.9093130011371935,
+          -0.0978520963516777,
+          0.36546998229542166,
+          -0.23875873233105468,
+          -0.16929286973811716,
+          0.1614393169093547
+        ],
+        "std": [
+          6.6392213185449e-05,
+          2.7477303136889642e-05,
+          0.3236236855006889,
+          0.10498868324818088,
+          0.21592920231663346,
+          0.14242412024634826,
+          0.17537817653960341,
+          0.1589983448423053,
+          0.1847030522707091,
+          0.3060052894278398,
+          0.15545421217912833,
+          0.18732387928520802,
+          0.23097992612678261,
+          0.17079917723668858,
+          0.20074672089177023,
+          0.18450774051586757,
+          0.20560809014181267,
+          0.3775058422894344,
+          0.16285387660315898,
+          0.21299280533759732
+        ],
+        "q01": [
+          -0.0001749494229231163,
+          -7.102986238156782e-05,
+          -0.4576713352188768,
+          -0.30153348134798175,
+          -0.4925865405961751,
+          -0.11370973105044871,
+          0.575421578469159,
+          -0.2209467398567723,
+          -0.6372816928992551,
+          -0.050352534999616734,
+          -0.23822213308286633,
+          -0.4218674707222603,
+          -0.33289762926446964,
+          -0.4403506284535931,
+          -1.2638898242176249,
+          -0.5333060165767463,
+          -0.06842082932288861,
+          -0.8808569036987537,
+          -0.43036151802551864,
+          -0.22260962031806678
+        ],
+        "q99": [
+          0.00017484051565752354,
+          -3.4853625995277704e-10,
+          0.4395555218474595,
+          0.05241236958600787,
+          0.3711243260578636,
+          0.43583597900427307,
+          1.1524254208812423,
+          0.4036845933510512,
+          0.010783664846300164,
+          0.7864205056015764,
+          0.27070234831315787,
+          0.1641843369874286,
+          0.5632010381667683,
+          0.23991706572733013,
+          -0.548851543258573,
+          0.21723416177116134,
+          0.7202919613199235,
+          0.15411611751854068,
+          0.142010680673695,
+          0.45282141791847796
+        ]
+      },
+      "left_hand": {
+        "min": [
+          -0.880666971206665,
+          0.6509807705879211,
+          -0.9295765161514282,
+          -1.0574641227722168,
+          -0.6108652353286743,
+          0.0,
+          0.0,
+          0.0,
+          -0.17246204614639282,
+          0.0,
+          0.0,
+          0.0,
+          -0.41887903213500977,
+          0.0,
+          0.0,
+          0.0,
+          -0.0,
+          -0.6108652353286743,
+          0.0,
+          0.18421649932861328
+        ],
+        "max": [
+          0.0,
+          1.8240469694137573,
+          -0.4169214367866516,
+          0.0,
+          0.41887903213500977,
+          1.5967257022857666,
+          1.5707963705062866,
+          1.5707963705062866,
+          0.6108652353286743,
+          1.3334362506866455,
+          1.5707963705062866,
+          1.5707963705062866,
+          0.2761457860469818,
+          1.474050521850586,
+          1.5047622919082642,
+          1.5707963705062866,
+          -0.0,
+          -0.013026786036789417,
+          1.5707963705062866,
+          1.5707963705062866
+        ],
+        "mean": [
+          -0.11197184536473949,
+          1.3519521447728844,
+          -0.6145068955473336,
+          -0.04490551318122404,
+          0.3812392963674508,
+          0.5928953152018989,
+          0.028781154436057742,
+          0.533942906643585,
+          0.2586826909204934,
+          0.24856345721893203,
+          0.04199277436774596,
+          0.40243144011328036,
+          -0.05372014356756498,
+          0.32290915338039355,
+          0.028910841144483782,
+          0.4829249856261449,
+          0.0,
+          -0.2736987368992572,
+          0.5229504504255549,
+          0.2163480956489075
+        ],
+        "std": [
+          0.18078946912307625,
+          0.19270362935123467,
+          0.059722650226918514,
+          0.13456131846867891,
+          0.12495907901258241,
+          0.30308983822349966,
+          0.15583292538180302,
+          0.2616750261097083,
+          0.1082498098717154,
+          0.21063265328615333,
+          0.20388137333506792,
+          0.3069878052972663,
+          0.08657551602472234,
+          0.28322161741911617,
+          0.11428067004348502,
+          0.3659270477492105,
+          0.0,
+          0.08003530419489042,
+          0.3308618144426684,
+          0.047378584156049966
+        ],
+        "q01": [
+          -0.5874023409507655,
+          1.1259081133842275,
+          -0.7722857806858863,
+          -0.46699307730378525,
+          0.2166453155186744,
+          0.2500263838689038,
+          -6.69347151562168e-11,
+          0.2810358129914362,
+          0.08265342079552167,
+          0.03212275062917445,
+          -7.422571247960103e-11,
+          0.11523256281603016,
+          -0.20857053595062272,
+          0.06834987345413719,
+          0.0032137934620088096,
+          0.16083063598248018,
+          3.9995784462121275e-16,
+          -0.41066407554491824,
+          0.21280464706121974,
+          0.20159717992290016
+        ],
+        "q99": [
+          -0.004032147496030359,
+          1.6196350682050786,
+          -0.5529140781257509,
+          -0.00022676876993767606,
+          0.4188780675074272,
+          1.2361410219097821,
+          0.4266783342163138,
+          1.1679802915746502,
+          0.3961736551386673,
+          0.7744282902748555,
+          0.6498963336012552,
+          1.195352623563556,
+          0.06817064976497338,
+          1.0339126384354067,
+          0.3498183122489737,
+          1.3578909040686071,
+          3.959582661750007e-14,
+          -0.18808869776308895,
+          1.2038159253048966,
+          0.30760064823760574
+        ]
+      },
+      "right_hand": {
+        "min": [
+          0.0,
+          -1.7954039573669434,
+          0.4210455119609833,
+          0.0,
+          -0.41887903213500977,
+          0.0,
+          0.0,
+          0.0,
+          -0.6108652353286743,
+          0.0,
+          0.0,
+          0.0,
+          -0.4958767592906952,
+          0.0,
+          0.0,
+          0.0,
+          0.0,
+          -0.12086009234189987,
+          0.0,
+          0.19107787311077118
+        ],
+        "max": [
+          0.8792693018913269,
+          -0.5100259184837341,
+          1.5707963705062866,
+          1.4147123098373413,
+          0.28721120953559875,
+          1.5164011716842651,
+          1.1974519491195679,
+          1.5707963705062866,
+          -0.04774348810315132,
+          1.4103626012802124,
+          1.5707963705062866,
+          1.5707963705062866,
+          0.24290113151073456,
+          1.5230220556259155,
+          1.5707963705062866,
+          1.5707963705062866,
+          0.0,
+          0.5971487760543823,
+          1.5707963705062866,
+          1.5707963705062866
+        ],
+        "mean": [
+          0.06154122902723653,
+          -1.3294183619600155,
+          0.6250821797211654,
+          0.05554929311166913,
+          -0.41492612799882095,
+          0.6856338848469389,
+          0.006940385029546583,
+          0.6091391652412862,
+          -0.37595000803726775,
+          0.422836852505782,
+          0.025116036945811486,
+          0.4918787185539051,
+          -0.14475854791832374,
+          0.46284594344482655,
+          0.027444823626726584,
+          0.5093871628244654,
+          0.0,
+          0.1401173905144233,
+          0.6521685371675148,
+          0.2238743152068344
+        ],
+        "std": [
+          0.13674527190111965,
+          0.17102490658313696,
+          0.05741908683253429,
+          0.14382058266604184,
+          0.02688551022749764,
+          0.2281908323797236,
+          0.05783917003853192,
+          0.26035589698369127,
+          0.07511741685767037,
+          0.18406048799000968,
+          0.14373912752045906,
+          0.2479658881484305,
+          0.09645055710684194,
+          0.26237633130190713,
+          0.10678798651992859,
+          0.3103495174470505,
+          0.0,
+          0.08664956776847747,
+          0.3581149787301811,
+          0.05026933816117329
+        ],
+        "q01": [
+          0.0009769272041044426,
+          -1.6190670990502298,
+          0.5518429740664715,
+          0.0013591393093312651,
+          -0.41887903213500977,
+          0.3491621230894666,
+          -4.421230354850494e-11,
+          0.3471016557019131,
+          -0.48352879670754245,
+          0.11780533016338945,
+          -7.137328803777223e-11,
+          0.19241114988176986,
+          -0.29701005092982646,
+          0.1185908435022493,
+          0.0034413322477957684,
+          0.19206370624490998,
+          3.9995784462121275e-16,
+          0.03772304600026963,
+          0.2884605865309452,
+          0.21015920588283557
+        ],
+        "q99": [
+          0.47407716229998736,
+          -1.1360530624353573,
+          0.7761147162999562,
+          0.5139581911771537,
+          -0.36667831242864685,
+          1.2000692036298772,
+          0.14459797121303056,
+          1.2871256631356942,
+          -0.25034097642726927,
+          0.8421005002108355,
+          0.4358143292303531,
+          1.1489288891981229,
+          0.006749831496495902,
+          1.1029886603257073,
+          0.340557722147255,
+          1.399224397253747,
+          3.959582661750007e-14,
+          0.2650572054718265,
+          1.4308893815770474,
+          0.31706126604163526
+        ]
+      }
+    },
+    "relative_action": {}
+  }
+}

experiment_cfg/final_model_config.json

@@ -0,0 +1,54 @@
+{
+  "model_type": "Gr00tN1d6",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "backbone_model_type": "eagle",
+  "model_revision": null,
+  "tune_top_llm_layers": 4,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": true,
+  "collator_overwrite_image_inputs": false,
+  "eagle_collator": true,
+  "backbone_trainable_params_fp32": true,
+  "extra_augmentation_config": null,
+  "apply_sincos_state_encoding": true,
+  "use_relative_action": true,
+  "max_state_dim": 128,
+  "max_action_dim": 128,
+  "action_horizon": 50,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.0,
+  "state_additive_noise_scale": 0.0,
+  "max_num_embodiments": 32
+}

processor/embodiment_id.json

@@ -0,0 +1,11 @@
+{
+  "robocasa_panda_omron": 13,
+  "gr1": 20,
+  "behavior_r1_pro": 24,
+  "unitree_g1": 8,
+  "oxe_google": 0,
+  "oxe_widowx": 1,
+  "libero_panda": 2,
+  "oxe_droid": 16,
+  "new_embodiment": 10
+}

processor/processor_config.json

@@ -0,0 +1,482 @@
+{
+  "processor_class": "Gr00tN1d6Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "behavior_r1_pro": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "observation.images.rgb.head_256_256",
+            "observation.images.rgb.left_wrist_256_256",
+            "observation.images.rgb.right_wrist_256_256"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "robot_pos",
+            "robot_ori_cos",
+            "robot_ori_sin",
+            "robot_2d_ori",
+            "robot_2d_ori_cos",
+            "robot_2d_ori_sin",
+            "robot_lin_vel",
+            "robot_ang_vel",
+            "arm_left_qpos",
+            "arm_left_qpos_sin",
+            "arm_left_qpos_cos",
+            "eef_left_pos",
+            "eef_left_quat",
+            "gripper_left_qpos",
+            "arm_right_qpos",
+            "arm_right_qpos_sin",
+            "arm_right_qpos_cos",
+            "eef_right_pos",
+            "eef_right_quat",
+            "gripper_right_qpos",
+            "trunk_qpos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31
+          ],
+          "modality_keys": [
+            "base",
+            "torso",
+            "left_arm",
+            "left_gripper",
+            "right_arm",
+            "right_gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "trunk_qpos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_left_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_right_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "gr1": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "ego_view_bg_crop_pad_res256_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "robocasa_panda_omron": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "res256_image_side_0",
+            "res256_image_side_1",
+            "res256_image_wrist_0"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "end_effector_position_relative",
+            "end_effector_rotation_relative",
+            "gripper_qpos",
+            "base_position",
+            "base_rotation"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "end_effector_position",
+            "end_effector_rotation",
+            "gripper_close",
+            "base_motion",
+            "control_mode"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.action.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "front",
+            "left_wrist",
+            "right_wrist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "arm",
+            "left_hand",
+            "right_hand"
+          ],
+          "sin_cos_embedding_keys": [
+            "arm",
+            "left_hand",
+            "right_hand"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29
+          ],
+          "modality_keys": [
+            "arm",
+            "left_hand",
+            "right_hand"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": null,
+    "image_target_size": null,
+    "use_albumentations": true,
+    "random_rotation_angle": null,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "model_name": "nvidia/Eagle-Block2A-2B-v2",
+    "model_type": "eagle",
+    "formalize_language": true,
+    "max_state_dim": 128,
+    "max_action_dim": 128,
+    "max_action_horizon": 50,
+    "use_percentiles": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": true,
+    "use_relative_action": true
+  }
+}

wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d6", "run_id": "dual_arm_finetune"}