Upload folder using huggingface_hub

config.json

@@ -0,0 +1,70 @@
+{
+  "action_horizon": 50,
+  "add_pos_embed": true,
+  "apply_sincos_state_encoding": true,
+  "architectures": [
+    "Gr00tN1d6"
+  ],
+  "attn_dropout": 0.2,
+  "attn_implementation": null,
+  "backbone_embedding_dim": 2048,
+  "backbone_model_type": "eagle",
+  "backbone_trainable_params_fp32": true,
+  "collator_overwrite_image_inputs": false,
+  "color_jitter_params": {
+    "brightness": 0.1,
+    "contrast": 0.1,
+    "hue": 0.1,
+    "saturation": 0.1
+  },
+  "crop_fraction": 0.95,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "eagle_collator": true,
+  "formalize_language": true,
+  "gemma_collator": false,
+  "hidden_size": 1024,
+  "image_crop_size": null,
+  "image_target_size": null,
+  "input_embedding_dim": 1536,
+  "load_bf16": true,
+  "max_action_dim": 128,
+  "max_num_embodiments": 32,
+  "max_seq_len": 1024,
+  "max_state_dim": 128,
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "model_type": "Gr00tN1d6",
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_inference_timesteps": 4,
+  "num_timestep_buckets": 1000,
+  "random_rotation_angle": null,
+  "reproject_vision": false,
+  "select_layer": 16,
+  "shortest_image_edge": 256,
+  "state_dropout_prob": 0.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "tune_diffusion_model": true,
+  "tune_llm": false,
+  "tune_projector": true,
+  "tune_top_llm_layers": 4,
+  "tune_visual": false,
+  "tune_vlln": true,
+  "use_albumentations_transforms": true,
+  "use_alternate_vl_dit": true,
+  "use_flash_attention": true,
+  "use_relative_action": true,
+  "use_vlln": true
+}

embodiment_id.json

@@ -0,0 +1,11 @@
+{
+  "robocasa_panda_omron": 13,
+  "gr1": 20,
+  "behavior_r1_pro": 24,
+  "unitree_g1": 8,
+  "oxe_google": 0,
+  "oxe_widowx": 1,
+  "libero_panda": 2,
+  "oxe_droid": 16,
+  "new_embodiment": 10
+}

experiment_cfg/conf.yaml

@@ -0,0 +1,211 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d6
+  model_dtype: bfloat16
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  backbone_model_type: eagle
+  model_revision: null
+  tune_top_llm_layers: 4
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 16
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  collator_overwrite_image_inputs: false
+  eagle_collator: true
+  backbone_trainable_params_fp32: true
+  image_crop_size: null
+  image_target_size: null
+  shortest_image_edge: 256
+  crop_fraction: 0.95
+  random_rotation_angle: null
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    saturation: 0.5
+    hue: 0.08
+  use_albumentations_transforms: true
+  extra_augmentation_config: null
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_relative_action: true
+  max_state_dim: 29
+  max_action_dim: 29
+  action_horizon: 16
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 32
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.0
+  state_additive_noise_scale: 0.0
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - ./demo_data/ps2/viis0002/cluttered_picking_v2
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - front
+        - wrist_color
+        - wrist_depth
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: RELATIVE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: fork
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: torchcodec
+training:
+  output_dir: ./output
+  experiment_name: null
+  max_steps: 10000
+  global_batch_size: 16
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 3000
+  save_total_limit: 5
+  save_vl_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 2
+  remove_unused_columns: false
+  use_wandb: false
+  wandb_project: finetune-gr00t-n1d6
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 10000
+save_steps: 3000

experiment_cfg/config.yaml

@@ -0,0 +1,245 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - ./demo_data/ps2/viis0002/cluttered_picking_v2
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - relative
+          state_key: null
+          type: &id002 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: *id002
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - front
+        - wrist_color
+        - wrist_depth
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: fork
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  random_chop: 0.0
+  seed: 42
+  shard_size: 1024
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: torchcodec
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
+  _attn_implementation_autoset: false
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  add_cross_attention: false
+  architectures: null
+  backbone_model_type: eagle
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    hue: 0.08
+    saturation: 0.5
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 32
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  eagle_collator: true
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  extra_augmentation_config: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_attentions: false
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.0
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torch_dtype: null
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 2
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 16
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 10000
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: ./output
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_steps: 3000
+  save_total_limit: 5
+  save_vl_model: false
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: false
+  wandb_project: finetune-gr00t-n1d6
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

experiment_cfg/dataset_statistics.json

@@ -0,0 +1,824 @@
+{
+  "new_embodiment": {
+    "state": {
+      "single_arm": {
+        "min": [
+          -43.319149017333984,
+          -99.40652465820312,
+          -39.10285568237305,
+          48.36065673828125,
+          -34.70085525512695
+        ],
+        "max": [
+          11.829787254333496,
+          40.398475646972656,
+          100.0,
+          100.0,
+          56.43467712402344
+        ],
+        "mean": [
+          -4.42374567107736,
+          -13.337984391845804,
+          15.277576091885644,
+          94.22222804451272,
+          9.506204733949865
+        ],
+        "std": [
+          12.136476122153015,
+          37.019769025893595,
+          34.34312925794012,
+          9.17392603160315,
+          16.675806387908786
+        ],
+        "q01": [
+          -32.64885159730033,
+          -98.19004714888533,
+          -20.175488649367793,
+          67.80894963783983,
+          -5.611908867094556
+        ],
+        "q99": [
+          8.506614531308287,
+          31.698417280947474,
+          95.9570446389378,
+          99.69303209181219,
+          40.82514578839471
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.5750798583030701
+        ],
+        "max": [
+          73.9297103881836
+        ],
+        "mean": [
+          10.041797340809085
+        ],
+        "std": [
+          12.109399021808883
+        ],
+        "q01": [
+          1.1988521671247627
+        ],
+        "q99": [
+          48.28906892196614
+        ]
+      }
+    },
+    "action": {
+      "single_arm": {
+        "min": [
+          -43.884891510009766,
+          -99.5762710571289,
+          -43.259193420410156,
+          46.74245834350586,
+          -34.94505310058594
+        ],
+        "max": [
+          12.53313159942627,
+          41.949153900146484,
+          99.63685607910156,
+          100.0,
+          56.82539749145508
+        ],
+        "mean": [
+          -4.319766296502379,
+          -15.558851080192529,
+          13.403112002696794,
+          94.04837653407854,
+          9.478646227815892
+        ],
+        "std": [
+          12.365528345704007,
+          36.579233916866265,
+          34.877090505999554,
+          9.286612361449192,
+          16.768166099670406
+        ],
+        "q01": [
+          -33.13864023520728,
+          -98.97692010724991,
+          -23.120881738340405,
+          66.65992739925638,
+          -5.897602745703706
+        ],
+        "q99": [
+          9.039396071548122,
+          31.64287464467848,
+          93.34089969984919,
+          99.88225409045437,
+          41.069476918322614
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.0
+        ],
+        "max": [
+          74.22003173828125
+        ],
+        "mean": [
+          7.485323198921303
+        ],
+        "std": [
+          12.4770328877653
+        ],
+        "q01": [
+          0.7537590801103762
+        ],
+        "q99": [
+          48.88889882187005
+        ]
+      }
+    },
+    "relative_action": {
+      "single_arm": {
+        "min": [
+          [
+            -14.407760620117188,
+            -27.423320770263672,
+            -21.879257202148438,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -14.407760620117188,
+            -27.423320770263672,
+            -22.60553741455078,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -14.407760620117188,
+            -27.423320770263672,
+            -25.817413330078125,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -15.768430709838867,
+            -27.423320770263672,
+            -29.261001586914062,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -17.888839721679688,
+            -27.423320770263672,
+            -32.529273986816406,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -19.93351936340332,
+            -27.423320770263672,
+            -35.610374450683594,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -21.82674217224121,
+            -27.68240737915039,
+            -38.79345703125,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -23.699081420898438,
+            -29.25457000732422,
+            -41.87455749511719,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -25.51657485961914,
+            -31.56486701965332,
+            -44.69236373901367,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -27.258338928222656,
+            -34.022499084472656,
+            -47.77346420288086,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -28.772918701171875,
+            -36.31063461303711,
+            -50.587806701660156,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -30.060306549072266,
+            -38.68351364135742,
+            -53.2205810546875,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -31.120513916015625,
+            -40.971649169921875,
+            -55.757965087890625,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -32.14179229736328,
+            -43.17504119873047,
+            -58.114105224609375,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -33.37945556640625,
+            -45.708072662353516,
+            -60.20216751098633,
+            -48.30026626586914,
+            -45.86080551147461
+          ],
+          [
+            -34.48583984375,
+            -48.25044631958008,
+            -62.19582748413086,
+            -48.30026626586914,
+            -45.86080551147461
+          ]
+        ],
+        "max": [
+          [
+            9.253393173217773,
+            18.55738067626953,
+            13.711788177490234,
+            11.358840942382812,
+            11.868131637573242
+          ],
+          [
+            11.551265716552734,
+            20.422576904296875,
+            13.711788177490234,
+            14.330146789550781,
+            14.163614273071289
+          ],
+          [
+            13.747403144836426,
+            22.11749267578125,
+            16.409061431884766,
+            16.994083404541016,
+            16.361417770385742
+          ],
+          [
+            15.792083740234375,
+            24.909435272216797,
+            19.24298095703125,
+            19.530155181884766,
+            18.168498992919922
+          ],
+          [
+            17.912492752075195,
+            28.13056182861328,
+            22.329681396484375,
+            22.153682708740234,
+            19.78022003173828
+          ],
+          [
+            19.7299861907959,
+            31.18191146850586,
+            25.320167541503906,
+            24.689754486083984,
+            21.098901748657227
+          ],
+          [
+            21.517221450805664,
+            34.232757568359375,
+            28.40127182006836,
+            27.05988311767578,
+            23.10134506225586
+          ],
+          [
+            23.25898551940918,
+            36.85987854003906,
+            31.215618133544922,
+            29.407485961914062,
+            25.10378646850586
+          ],
+          [
+            24.96111297607422,
+            39.402252197265625,
+            33.934242248535156,
+            31.68120574951172,
+            27.008548736572266
+          ],
+          [
+            26.32281494140625,
+            41.860923767089844,
+            36.5670166015625,
+            33.69257354736328,
+            28.913311004638672
+          ],
+          [
+            27.610206604003906,
+            44.233802795410156,
+            39.19501876831055,
+            35.63929748535156,
+            30.818073272705078
+          ],
+          [
+            28.746139526367188,
+            46.68543243408203,
+            41.737003326416016,
+            37.38831329345703,
+            32.625152587890625
+          ],
+          [
+            29.80634307861328,
+            49.737571716308594,
+            44.274383544921875,
+            38.92519760131836,
+            34.334556579589844
+          ],
+          [
+            30.57229995727539,
+            52.535362243652344,
+            46.634803771972656,
+            40.14950942993164,
+            36.141639709472656
+          ],
+          [
+            31.500513076782227,
+            55.078819274902344,
+            48.98831558227539,
+            40.76426315307617,
+            37.80220031738281
+          ],
+          [
+            33.01509094238281,
+            57.621192932128906,
+            51.16715621948242,
+            41.1302375793457,
+            39.120880126953125
+          ]
+        ],
+        "mean": [
+          [
+            0.10775035619735718,
+            -2.2124874591827393,
+            -1.8821313381195068,
+            -0.15602795779705048,
+            -0.028898637741804123
+          ],
+          [
+            0.10764574259519577,
+            -2.2118215560913086,
+            -1.8829621076583862,
+            -0.15521754324436188,
+            -0.029046669602394104
+          ],
+          [
+            0.10753412544727325,
+            -2.2111780643463135,
+            -1.88377046585083,
+            -0.15439072251319885,
+            -0.02919066697359085
+          ],
+          [
+            0.10741761326789856,
+            -2.2105801105499268,
+            -1.8845477104187012,
+            -0.15354374051094055,
+            -0.029330182820558548
+          ],
+          [
+            0.10729756951332092,
+            -2.210028886795044,
+            -1.8853235244750977,
+            -0.15268175303936005,
+            -0.02946470119059086
+          ],
+          [
+            0.10717550665140152,
+            -2.2094948291778564,
+            -1.8860677480697632,
+            -0.15180876851081848,
+            -0.029594941064715385
+          ],
+          [
+            0.10705345124006271,
+            -2.208998918533325,
+            -1.8868061304092407,
+            -0.15092876553535461,
+            -0.02971998043358326
+          ],
+          [
+            0.1069316565990448,
+            -2.2085275650024414,
+            -1.8875151872634888,
+            -0.15004493296146393,
+            -0.029840001836419106
+          ],
+          [
+            0.10681024193763733,
+            -2.20807147026062,
+            -1.8882157802581787,
+            -0.14915572106838226,
+            -0.029957549646496773
+          ],
+          [
+            0.106687992811203,
+            -2.207639217376709,
+            -1.88888680934906,
+            -0.14825686812400818,
+            -0.03007364645600319
+          ],
+          [
+            0.10656628757715225,
+            -2.2072384357452393,
+            -1.8895448446273804,
+            -0.14735420048236847,
+            -0.030191101133823395
+          ],
+          [
+            0.1064440980553627,
+            -2.2068586349487305,
+            -1.8901573419570923,
+            -0.14644339680671692,
+            -0.030311299487948418
+          ],
+          [
+            0.10632333904504776,
+            -2.206537961959839,
+            -1.890722393989563,
+            -0.14552569389343262,
+            -0.030435657128691673
+          ],
+          [
+            0.10620231181383133,
+            -2.206232786178589,
+            -1.891249656677246,
+            -0.14459942281246185,
+            -0.0305632296949625
+          ],
+          [
+            0.1060793474316597,
+            -2.2059710025787354,
+            -1.8917396068572998,
+            -0.1436663717031479,
+            -0.03069279156625271
+          ],
+          [
+            0.10595506429672241,
+            -2.2057464122772217,
+            -1.8921877145767212,
+            -0.14272987842559814,
+            -0.03082442656159401
+          ]
+        ],
+        "std": [
+          [
+            1.2841439247131348,
+            2.8950278759002686,
+            2.4275267124176025,
+            1.0856059789657593,
+            0.987791121006012
+          ],
+          [
+            1.5570536851882935,
+            3.247357130050659,
+            2.7970223426818848,
+            1.234719157218933,
+            1.189427137374878
+          ],
+          [
+            1.832870602607727,
+            3.618945837020874,
+            3.1819772720336914,
+            1.3905600309371948,
+            1.39454185962677
+          ],
+          [
+            2.108621120452881,
+            4.002004623413086,
+            3.5744118690490723,
+            1.5488932132720947,
+            1.6003150939941406
+          ],
+          [
+            2.3826022148132324,
+            4.391429901123047,
+            3.969658851623535,
+            1.7073702812194824,
+            1.8050740957260132
+          ],
+          [
+            2.6539173126220703,
+            4.783688545227051,
+            4.3641886711120605,
+            1.86428701877594,
+            2.008115530014038
+          ],
+          [
+            2.921947717666626,
+            5.176029682159424,
+            4.756381988525391,
+            2.018906354904175,
+            2.2087574005126953
+          ],
+          [
+            3.18644642829895,
+            5.567348003387451,
+            5.144635200500488,
+            2.1705174446105957,
+            2.4067444801330566
+          ],
+          [
+            3.447101593017578,
+            5.956623554229736,
+            5.527939796447754,
+            2.3186304569244385,
+            2.601766586303711
+          ],
+          [
+            3.7035326957702637,
+            6.342449188232422,
+            5.905921936035156,
+            2.462908983230591,
+            2.7937722206115723
+          ],
+          [
+            3.9558348655700684,
+            6.724341869354248,
+            6.27820348739624,
+            2.6032063961029053,
+            2.9825356006622314
+          ],
+          [
+            4.203965187072754,
+            7.10208797454834,
+            6.644443511962891,
+            2.739359140396118,
+            3.16814923286438
+          ],
+          [
+            4.447895050048828,
+            7.474869251251221,
+            7.0046796798706055,
+            2.8713440895080566,
+            3.350452184677124
+          ],
+          [
+            4.687520503997803,
+            7.842729091644287,
+            7.358129501342773,
+            2.9990336894989014,
+            3.5294036865234375
+          ],
+          [
+            4.922867774963379,
+            8.205199241638184,
+            7.705315589904785,
+            3.122511863708496,
+            3.7049813270568848
+          ],
+          [
+            5.1538190841674805,
+            8.56218433380127,
+            8.046268463134766,
+            3.241928815841675,
+            3.8773155212402344
+          ]
+        ],
+        "q01": [
+          [
+            -3.857243537902832,
+            -8.598920822143555,
+            -9.160676002502441,
+            -3.4342920780181885,
+            -3.467641830444336
+          ],
+          [
+            -4.825390338897705,
+            -9.875429153442383,
+            -10.873571395874023,
+            -3.8172378540039062,
+            -4.34676456451416
+          ],
+          [
+            -5.829190731048584,
+            -11.199197769165039,
+            -12.570626258850098,
+            -4.2969207763671875,
+            -5.274724960327148
+          ],
+          [
+            -6.83367919921875,
+            -12.496345520019531,
+            -14.261612892150879,
+            -4.763179779052734,
+            -6.153846740722656
+          ],
+          [
+            -7.82426118850708,
+            -13.83658504486084,
+            -15.928194999694824,
+            -5.32944393157959,
+            -7.032968521118164
+          ],
+          [
+            -8.813674926757812,
+            -15.110017776489258,
+            -17.43770980834961,
+            -5.864686489105225,
+            -7.9199042320251465
+          ],
+          [
+            -9.763386726379395,
+            -16.384273529052734,
+            -19.013256072998047,
+            -6.480079650878906,
+            -8.791208267211914
+          ],
+          [
+            -10.717988014221191,
+            -17.67698097229004,
+            -20.471559524536133,
+            -6.955631256103516,
+            -9.678147315979004
+          ],
+          [
+            -11.672321319580078,
+            -18.927724838256836,
+            -21.935583114624023,
+            -7.477760314941406,
+            -10.598289489746094
+          ],
+          [
+            -12.580903053283691,
+            -20.128698348999023,
+            -23.29421615600586,
+            -7.980550289154053,
+            -11.428571701049805
+          ],
+          [
+            -13.46851921081543,
+            -21.378007888793945,
+            -24.701528549194336,
+            -8.435519218444824,
+            -12.258852005004883
+          ],
+          [
+            -14.374682426452637,
+            -22.51150894165039,
+            -26.01465606689453,
+            -8.79336929321289,
+            -13.089134216308594
+          ],
+          [
+            -15.234027862548828,
+            -23.693721771240234,
+            -27.294567108154297,
+            -9.178253173828125,
+            -13.968252182006836
+          ],
+          [
+            -16.0954532623291,
+            -24.78091812133789,
+            -28.555892944335938,
+            -9.539568901062012,
+            -14.700855255126953
+          ],
+          [
+            -16.930395126342773,
+            -25.81134796142578,
+            -29.762718200683594,
+            -9.925280570983887,
+            -15.531136512756348
+          ],
+          [
+            -17.743947982788086,
+            -26.893335342407227,
+            -30.908802032470703,
+            -10.261674880981445,
+            -16.263734817504883
+          ]
+        ],
+        "q99": [
+          [
+            4.942091941833496,
+            5.228148460388184,
+            5.199999809265137,
+            2.6471364498138428,
+            3.956043243408203
+          ],
+          [
+            6.152984142303467,
+            6.845273971557617,
+            6.727862358093262,
+            3.2062454223632812,
+            4.981685638427734
+          ],
+          [
+            7.376312255859375,
+            8.517343521118164,
+            8.359054565429688,
+            3.559464693069458,
+            6.007326126098633
+          ],
+          [
+            8.570808410644531,
+            10.04684829711914,
+            9.90521240234375,
+            4.104945182800293,
+            6.984127044677734
+          ],
+          [
+            9.74888801574707,
+            11.609418869018555,
+            11.436421394348145,
+            4.578891754150391,
+            7.960927963256836
+          ],
+          [
+            10.930181503295898,
+            13.18136215209961,
+            12.887182235717773,
+            5.075687408447266,
+            8.888888359069824
+          ],
+          [
+            12.05977725982666,
+            14.643962860107422,
+            14.382278442382812,
+            5.598910331726074,
+            9.86568832397461
+          ],
+          [
+            13.165159225463867,
+            16.154401779174805,
+            15.85221004486084,
+            6.150146484375,
+            10.695971488952637
+          ],
+          [
+            14.26892375946045,
+            17.632768630981445,
+            17.293310165405273,
+            6.720368385314941,
+            11.623930931091309
+          ],
+          [
+            15.29382038116455,
+            19.11349868774414,
+            18.627479553222656,
+            7.233709335327148,
+            12.551891326904297
+          ],
+          [
+            16.346874237060547,
+            20.474987030029297,
+            20.002891540527344,
+            7.730985164642334,
+            13.38980484008789
+          ],
+          [
+            17.317106246948242,
+            21.834218978881836,
+            21.317062377929688,
+            8.229083061218262,
+            14.407814025878906
+          ],
+          [
+            18.268630981445312,
+            23.183210372924805,
+            22.662113189697266,
+            8.661979675292969,
+            15.14041519165039
+          ],
+          [
+            19.180822372436523,
+            24.46126365661621,
+            23.940603256225586,
+            9.03399658203125,
+            16.068376541137695
+          ],
+          [
+            20.067045211791992,
+            25.73427963256836,
+            25.14236831665039,
+            9.510910987854004,
+            16.857450485229492
+          ],
+          [
+            20.93355941772461,
+            26.99875831604004,
+            26.298900604248047,
+            9.919087409973145,
+            17.631256103515625
+          ]
+        ]
+      }
+    }
+  }
+}

experiment_cfg/final_model_config.json

@@ -0,0 +1,54 @@
+{
+  "model_type": "Gr00tN1d6",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "backbone_model_type": "eagle",
+  "model_revision": null,
+  "tune_top_llm_layers": 4,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": true,
+  "collator_overwrite_image_inputs": false,
+  "eagle_collator": true,
+  "backbone_trainable_params_fp32": true,
+  "extra_augmentation_config": null,
+  "apply_sincos_state_encoding": true,
+  "use_relative_action": true,
+  "max_state_dim": 128,
+  "max_action_dim": 128,
+  "action_horizon": 50,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.0,
+  "state_additive_noise_scale": 0.0,
+  "max_num_embodiments": 32
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5036baa2d7b7203a5addf27f8bbc7496cc66c7e7d1b039a0b8a3a3fa4bbde0c8
+size 4990571768

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a02caa3d238bd6633c5471c8067b2c4662ceda07c6f39382de32b3e98dd7be7
+size 4823190320

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b34af109dce6ad956deef4c3046870f05026d71e3c35b2a9e783c12e6f16f691
+size 12960193762

processor_config.json

@@ -0,0 +1,456 @@
+{
+  "processor_class": "Gr00tN1d6Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "behavior_r1_pro": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "observation.images.rgb.head_256_256",
+            "observation.images.rgb.left_wrist_256_256",
+            "observation.images.rgb.right_wrist_256_256"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "robot_pos",
+            "robot_ori_cos",
+            "robot_ori_sin",
+            "robot_2d_ori",
+            "robot_2d_ori_cos",
+            "robot_2d_ori_sin",
+            "robot_lin_vel",
+            "robot_ang_vel",
+            "arm_left_qpos",
+            "arm_left_qpos_sin",
+            "arm_left_qpos_cos",
+            "eef_left_pos",
+            "eef_left_quat",
+            "gripper_left_qpos",
+            "arm_right_qpos",
+            "arm_right_qpos_sin",
+            "arm_right_qpos_cos",
+            "eef_right_pos",
+            "eef_right_quat",
+            "gripper_right_qpos",
+            "trunk_qpos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31
+          ],
+          "modality_keys": [
+            "base",
+            "torso",
+            "left_arm",
+            "left_gripper",
+            "right_arm",
+            "right_gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "trunk_qpos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_left_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_right_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "gr1": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "ego_view_bg_crop_pad_res256_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "robocasa_panda_omron": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "res256_image_side_0",
+            "res256_image_side_1",
+            "res256_image_wrist_0"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "end_effector_position_relative",
+            "end_effector_rotation_relative",
+            "gripper_qpos",
+            "base_position",
+            "base_rotation"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "end_effector_position",
+            "end_effector_rotation",
+            "gripper_close",
+            "base_motion",
+            "control_mode"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.action.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "front",
+            "wrist_color",
+            "wrist_depth"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": null,
+    "image_target_size": null,
+    "use_albumentations": true,
+    "random_rotation_angle": null,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "model_name": "nvidia/Eagle-Block2A-2B-v2",
+    "model_type": "eagle",
+    "formalize_language": true,
+    "max_state_dim": 128,
+    "max_action_dim": 128,
+    "max_action_horizon": 50,
+    "use_percentiles": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": true,
+    "use_relative_action": true
+  }
+}

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28cf164b86b85a4945d7d11885be0be695fac1038673c932c331c8879eb9961f
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d02be6d8bda4ea9c67040ed89f878acdc986bd4df3fbc60440a9d3eacca02d63
+size 1465

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:091dae5f9ca2bd94099e5f39389c742416696e9002ee009015d98b4611bf79e1
+size 5713

wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d6", "run_id": "output"}