Upload folder using huggingface_hub

ckpt/checkpoint-10000/config.json

@@ -0,0 +1,70 @@
+{
+  "action_horizon": 50,
+  "add_pos_embed": true,
+  "apply_sincos_state_encoding": true,
+  "architectures": [
+    "Gr00tN1d6"
+  ],
+  "attn_dropout": 0.2,
+  "attn_implementation": null,
+  "backbone_embedding_dim": 2048,
+  "backbone_model_type": "eagle",
+  "backbone_trainable_params_fp32": true,
+  "collator_overwrite_image_inputs": false,
+  "color_jitter_params": {
+    "brightness": 0.1,
+    "contrast": 0.1,
+    "hue": 0.1,
+    "saturation": 0.1
+  },
+  "crop_fraction": 0.95,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "eagle_collator": true,
+  "formalize_language": true,
+  "gemma_collator": false,
+  "hidden_size": 1024,
+  "image_crop_size": null,
+  "image_target_size": null,
+  "input_embedding_dim": 1536,
+  "load_bf16": true,
+  "max_action_dim": 128,
+  "max_num_embodiments": 32,
+  "max_seq_len": 1024,
+  "max_state_dim": 128,
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "model_type": "Gr00tN1d6",
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_inference_timesteps": 4,
+  "num_timestep_buckets": 1000,
+  "random_rotation_angle": null,
+  "reproject_vision": false,
+  "select_layer": 16,
+  "shortest_image_edge": 256,
+  "state_dropout_prob": 0.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "tune_diffusion_model": true,
+  "tune_llm": false,
+  "tune_projector": true,
+  "tune_top_llm_layers": 4,
+  "tune_visual": false,
+  "tune_vlln": true,
+  "use_albumentations_transforms": true,
+  "use_alternate_vl_dit": true,
+  "use_flash_attention": true,
+  "use_relative_action": true,
+  "use_vlln": true
+}

ckpt/checkpoint-10000/embodiment_id.json

@@ -0,0 +1,10 @@
+{
+  "robocasa_panda_omron": 13,
+  "gr1": 20,
+  "behavior_r1_pro": 24,
+  "unitree_g1": 8,
+  "oxe_google": 0,
+  "oxe_widowx": 1,
+  "libero_panda": 2,
+  "new_embodiment": 10
+}

ckpt/checkpoint-10000/experiment_cfg/conf.yaml

@@ -0,0 +1,209 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d6
+  model_dtype: bfloat16
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  backbone_model_type: eagle
+  model_revision: null
+  tune_top_llm_layers: 4
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 16
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  collator_overwrite_image_inputs: false
+  eagle_collator: true
+  backbone_trainable_params_fp32: true
+  image_crop_size: null
+  image_target_size: null
+  shortest_image_edge: 256
+  crop_fraction: 0.95
+  random_rotation_angle: null
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    saturation: 0.5
+    hue: 0.08
+  use_albumentations_transforms: true
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_relative_action: true
+  max_state_dim: 29
+  max_action_dim: 29
+  action_horizon: 16
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 32
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.0
+  state_additive_noise_scale: 0.0
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - dataset/so101_pick_orange_v2.1
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: RELATIVE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: fork
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: torchcodec
+training:
+  output_dir: outputs/so101_finetune
+  experiment_name: null
+  max_steps: 10000
+  global_batch_size: 32
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 1000
+  save_total_limit: 5
+  save_vl_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 4
+  remove_unused_columns: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 10000
+save_steps: 1000

ckpt/checkpoint-10000/experiment_cfg/config.yaml

@@ -0,0 +1,243 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - dataset/so101_pick_orange_v2.1
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - relative
+          state_key: null
+          type: &id002 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: *id002
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: fork
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  random_chop: 0.0
+  seed: 42
+  shard_size: 1024
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: torchcodec
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
+  _attn_implementation_autoset: false
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  add_cross_attention: false
+  architectures: null
+  backbone_model_type: eagle
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    hue: 0.08
+    saturation: 0.5
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 32
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  eagle_collator: true
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_attentions: false
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.0
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torch_dtype: null
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 4
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 32
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 10000
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: outputs/so101_finetune
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_steps: 1000
+  save_total_limit: 5
+  save_vl_model: false
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

ckpt/checkpoint-10000/experiment_cfg/dataset_statistics.json

@@ -0,0 +1,824 @@
+{
+  "new_embodiment": {
+    "state": {
+      "single_arm": {
+        "min": [
+          -66.94554138183594,
+          -99.34537506103516,
+          -100.07406616210938,
+          -7.2442169189453125,
+          -14.350746154785156
+        ],
+        "max": [
+          54.9786376953125,
+          94.00718688964844,
+          99.91923522949219,
+          100.17535400390625,
+          49.98741149902344
+        ],
+        "mean": [
+          9.436819110182034,
+          7.574628396223122,
+          -10.053358760235177,
+          80.99322336603086,
+          8.907621910152459
+        ],
+        "std": [
+          18.523341824919854,
+          44.35623147233547,
+          52.39530042055574,
+          17.853640756898756,
+          11.958544505091808
+        ],
+        "q01": [
+          -16.689096150230384,
+          -94.91365668057918,
+          -83.75251391588807,
+          38.96477862518415,
+          -7.5895381550527485
+        ],
+        "q99": [
+          44.3971940337351,
+          57.803459759429344,
+          93.71004261030197,
+          99.54684530336125,
+          33.206195503441855
+        ]
+      },
+      "gripper": {
+        "min": [
+          1.4235210418701172
+        ],
+        "max": [
+          72.82665252685547
+        ],
+        "mean": [
+          35.93042549630967
+        ],
+        "std": [
+          19.82074107207529
+        ],
+        "q01": [
+          4.166254402088548
+        ],
+        "q99": [
+          61.39789539026209
+        ]
+      }
+    },
+    "action": {
+      "single_arm": {
+        "min": [
+          -67.02870178222656,
+          -99.99999237060547,
+          -99.54914093017578,
+          -7.626762390136719,
+          -15.4171142578125
+        ],
+        "max": [
+          54.38323974609375,
+          90.63290405273438,
+          99.09828186035156,
+          100.0,
+          50.42237854003906
+        ],
+        "mean": [
+          9.34462854049006,
+          6.218694137043469,
+          -11.439668769206587,
+          81.78630601437781,
+          8.801531099191099
+        ],
+        "std": [
+          18.455675361161806,
+          44.32732024212432,
+          52.883627011344565,
+          18.074943936960118,
+          11.953933302294447
+        ],
+        "q01": [
+          -16.572277213108652,
+          -96.09065517543287,
+          -84.84769216215412,
+          39.35092962124239,
+          -7.671825312079834
+        ],
+        "q99": [
+          43.40559771349356,
+          55.26623004959493,
+          92.96575841515937,
+          99.27690853548235,
+          33.246040566403735
+        ]
+      },
+      "gripper": {
+        "min": [
+          1.483216643333435
+        ],
+        "max": [
+          72.98985290527344
+        ],
+        "mean": [
+          32.06279950114537
+        ],
+        "std": [
+          22.009280451587454
+        ],
+        "q01": [
+          4.0092054223149365
+        ],
+        "q99": [
+          61.458876840633906
+        ]
+      }
+    },
+    "relative_action": {
+      "single_arm": {
+        "min": [
+          [
+            -12.078048706054688,
+            -51.95113754272461,
+            -41.35333251953125,
+            -20.325958251953125,
+            -13.003585815429688
+          ],
+          [
+            -14.987480163574219,
+            -51.95113754272461,
+            -41.353355407714844,
+            -28.125457763671875,
+            -18.019424438476562
+          ],
+          [
+            -18.291839599609375,
+            -51.95113754272461,
+            -41.361572265625,
+            -35.89482116699219,
+            -22.878860473632812
+          ],
+          [
+            -21.74469757080078,
+            -51.95113754272461,
+            -46.38298034667969,
+            -43.38505554199219,
+            -27.603317260742188
+          ],
+          [
+            -24.92536163330078,
+            -51.95113754272461,
+            -52.875328063964844,
+            -50.71356201171875,
+            -31.510398864746094
+          ],
+          [
+            -27.711326599121094,
+            -51.95113754272461,
+            -58.254844665527344,
+            -57.86737060546875,
+            -33.94642639160156
+          ],
+          [
+            -30.726837158203125,
+            -54.53240966796875,
+            -62.85357666015625,
+            -63.66200256347656,
+            -36.22076416015625
+          ],
+          [
+            -33.83001708984375,
+            -59.21240997314453,
+            -67.64421844482422,
+            -67.32625579833984,
+            -38.53020477294922
+          ],
+          [
+            -36.622886657714844,
+            -63.709739685058594,
+            -73.31746673583984,
+            -69.5547103881836,
+            -40.86298370361328
+          ],
+          [
+            -39.131980895996094,
+            -68.01353454589844,
+            -78.90808868408203,
+            -71.34422302246094,
+            -43.1685791015625
+          ],
+          [
+            -41.22662353515625,
+            -72.27005767822266,
+            -83.95648956298828,
+            -71.68507385253906,
+            -45.630462646484375
+          ],
+          [
+            -43.09895324707031,
+            -76.34874725341797,
+            -88.23937225341797,
+            -72.02021789550781,
+            -48.347808837890625
+          ],
+          [
+            -44.80570983886719,
+            -80.23060607910156,
+            -91.84622192382812,
+            -72.1054458618164,
+            -51.21324157714844
+          ],
+          [
+            -46.06871032714844,
+            -83.98172760009766,
+            -95.32868957519531,
+            -72.1054458618164,
+            -53.853118896484375
+          ],
+          [
+            -47.23240661621094,
+            -87.46543884277344,
+            -98.75520324707031,
+            -72.1054458618164,
+            -56.05763244628906
+          ],
+          [
+            -48.67430114746094,
+            -90.8409652709961,
+            -101.38887023925781,
+            -72.1054458618164,
+            -57.85150146484375
+          ]
+        ],
+        "max": [
+          [
+            18.512191772460938,
+            20.69375228881836,
+            42.411834716796875,
+            23.563674926757812,
+            13.185585021972656
+          ],
+          [
+            22.481399536132812,
+            28.41082000732422,
+            42.411834716796875,
+            27.518478393554688,
+            17.75921630859375
+          ],
+          [
+            25.739761352539062,
+            35.58381652832031,
+            46.46379852294922,
+            34.67655944824219,
+            22.29987335205078
+          ],
+          [
+            28.566734313964844,
+            42.625579833984375,
+            55.480934143066406,
+            40.45378112792969,
+            26.15636444091797
+          ],
+          [
+            30.84162139892578,
+            48.86858367919922,
+            64.38365173339844,
+            45.651885986328125,
+            29.79944610595703
+          ],
+          [
+            34.05448913574219,
+            54.42564392089844,
+            72.3538589477539,
+            49.64585876464844,
+            32.81971740722656
+          ],
+          [
+            37.10010528564453,
+            61.04212951660156,
+            80.28893280029297,
+            52.62840270996094,
+            35.40287780761719
+          ],
+          [
+            39.84075927734375,
+            67.21363830566406,
+            87.31271362304688,
+            55.53631591796875,
+            38.09008026123047
+          ],
+          [
+            42.86634826660156,
+            73.03642272949219,
+            94.16574096679688,
+            57.916900634765625,
+            39.504486083984375
+          ],
+          [
+            45.81437683105469,
+            78.54348754882812,
+            100.11762237548828,
+            60.059600830078125,
+            40.71885681152344
+          ],
+          [
+            48.55254364013672,
+            83.35438537597656,
+            105.4377212524414,
+            62.40226745605469,
+            41.093238830566406
+          ],
+          [
+            50.975975036621094,
+            88.08012390136719,
+            110.27528381347656,
+            64.56655883789062,
+            41.30443572998047
+          ],
+          [
+            53.30335235595703,
+            92.29953002929688,
+            115.33406066894531,
+            65.78926086425781,
+            41.34910583496094
+          ],
+          [
+            55.47357177734375,
+            96.1929702758789,
+            121.76927947998047,
+            69.31730651855469,
+            41.388336181640625
+          ],
+          [
+            57.270782470703125,
+            99.4841079711914,
+            127.72057342529297,
+            72.86603546142578,
+            41.913490295410156
+          ],
+          [
+            58.745025634765625,
+            102.42430114746094,
+            132.68002319335938,
+            76.27465057373047,
+            42.577056884765625
+          ]
+        ],
+        "mean": [
+          [
+            -0.08928748220205307,
+            -1.341666340827942,
+            -1.4313260316848755,
+            0.8176242113113403,
+            -0.10854622721672058
+          ],
+          [
+            -0.08544569462537766,
+            -1.3248456716537476,
+            -1.450048804283142,
+            0.8183177709579468,
+            -0.10887768864631653
+          ],
+          [
+            -0.08182403445243835,
+            -1.309598445892334,
+            -1.4664539098739624,
+            0.8186652660369873,
+            -0.10912427306175232
+          ],
+          [
+            -0.07855380326509476,
+            -1.2955514192581177,
+            -1.4810765981674194,
+            0.8189600110054016,
+            -0.10939010232686996
+          ],
+          [
+            -0.07568887621164322,
+            -1.282740592956543,
+            -1.4939128160476685,
+            0.819760262966156,
+            -0.10975522547960281
+          ],
+          [
+            -0.07324869185686111,
+            -1.270890474319458,
+            -1.5052545070648193,
+            0.821160614490509,
+            -0.1100916713476181
+          ],
+          [
+            -0.07121772319078445,
+            -1.2599191665649414,
+            -1.5153403282165527,
+            0.8231523036956787,
+            -0.1104360818862915
+          ],
+          [
+            -0.06971976906061172,
+            -1.2498693466186523,
+            -1.5240850448608398,
+            0.8258273005485535,
+            -0.11075393855571747
+          ],
+          [
+            -0.0686890035867691,
+            -1.2407728433609009,
+            -1.5315450429916382,
+            0.8293973803520203,
+            -0.11109816282987595
+          ],
+          [
+            -0.06810573488473892,
+            -1.2328988313674927,
+            -1.537511944770813,
+            0.8338295221328735,
+            -0.11150559782981873
+          ],
+          [
+            -0.06795518100261688,
+            -1.2263479232788086,
+            -1.5418497323989868,
+            0.8392298817634583,
+            -0.11192262917757034
+          ],
+          [
+            -0.06824091076850891,
+            -1.2213470935821533,
+            -1.5445207357406616,
+            0.84555584192276,
+            -0.11240900307893753
+          ],
+          [
+            -0.06895144283771515,
+            -1.2181369066238403,
+            -1.5453885793685913,
+            0.8529182076454163,
+            -0.11297398805618286
+          ],
+          [
+            -0.06986649334430695,
+            -1.2168906927108765,
+            -1.544404149055481,
+            0.8613249063491821,
+            -0.113494373857975
+          ],
+          [
+            -0.07098635286092758,
+            -1.2179030179977417,
+            -1.5415356159210205,
+            0.8707390427589417,
+            -0.11393582075834274
+          ],
+          [
+            -0.07230301201343536,
+            -1.2215701341629028,
+            -1.5368025302886963,
+            0.881089448928833,
+            -0.11435369402170181
+          ]
+        ],
+        "std": [
+          [
+            2.871591091156006,
+            5.391886234283447,
+            6.014742374420166,
+            3.662677764892578,
+            1.5400844812393188
+          ],
+          [
+            3.6962430477142334,
+            6.426257610321045,
+            7.588341236114502,
+            4.433483123779297,
+            2.0808327198028564
+          ],
+          [
+            4.523628234863281,
+            7.537558078765869,
+            9.208881378173828,
+            5.248630523681641,
+            2.599971294403076
+          ],
+          [
+            5.338505744934082,
+            8.676809310913086,
+            10.831770896911621,
+            6.0670928955078125,
+            3.0954203605651855
+          ],
+          [
+            6.133913040161133,
+            9.817375183105469,
+            12.435976028442383,
+            6.87053918838501,
+            3.5675745010375977
+          ],
+          [
+            6.90657377243042,
+            10.943366050720215,
+            14.010025024414062,
+            7.649028778076172,
+            4.017260551452637
+          ],
+          [
+            7.654917240142822,
+            12.04592227935791,
+            15.547393798828125,
+            8.397272109985352,
+            4.445760250091553
+          ],
+          [
+            8.378345489501953,
+            13.1199369430542,
+            17.044599533081055,
+            9.112852096557617,
+            4.8546142578125
+          ],
+          [
+            9.07689380645752,
+            14.162562370300293,
+            18.499372482299805,
+            9.794965744018555,
+            5.2454915046691895
+          ],
+          [
+            9.750933647155762,
+            15.172690391540527,
+            19.911304473876953,
+            10.443316459655762,
+            5.619942665100098
+          ],
+          [
+            10.401156425476074,
+            16.14958953857422,
+            21.27997589111328,
+            11.058207511901855,
+            5.979310512542725
+          ],
+          [
+            11.028536796569824,
+            17.093509674072266,
+            22.60550880432129,
+            11.64011001586914,
+            6.324570655822754
+          ],
+          [
+            11.634090423583984,
+            18.0046443939209,
+            23.8878116607666,
+            12.189626693725586,
+            6.656616687774658
+          ],
+          [
+            12.21898078918457,
+            18.88372802734375,
+            25.12725830078125,
+            12.70767879486084,
+            6.976064205169678
+          ],
+          [
+            12.784246444702148,
+            19.73196792602539,
+            26.324321746826172,
+            13.195191383361816,
+            7.283435821533203
+          ],
+          [
+            13.33083724975586,
+            20.551090240478516,
+            27.480131149291992,
+            13.652670860290527,
+            7.579307556152344
+          ]
+        ],
+        "q01": [
+          [
+            -7.962890148162842,
+            -14.315224647521973,
+            -17.707164764404297,
+            -9.451419830322266,
+            -5.023085117340088
+          ],
+          [
+            -9.997512817382812,
+            -17.7712459564209,
+            -22.663740158081055,
+            -12.528553009033203,
+            -6.693915843963623
+          ],
+          [
+            -11.868306159973145,
+            -21.55695343017578,
+            -26.183889389038086,
+            -15.697895050048828,
+            -8.342229843139648
+          ],
+          [
+            -14.148200035095215,
+            -25.02656364440918,
+            -30.18362045288086,
+            -18.600130081176758,
+            -9.885397911071777
+          ],
+          [
+            -16.415008544921875,
+            -28.84429168701172,
+            -33.98527145385742,
+            -21.490951538085938,
+            -11.297945976257324
+          ],
+          [
+            -18.51193618774414,
+            -32.46988296508789,
+            -37.53544235229492,
+            -24.31591796875,
+            -12.669763565063477
+          ],
+          [
+            -20.730897903442383,
+            -35.97110366821289,
+            -41.142967224121094,
+            -26.682071685791016,
+            -13.93807315826416
+          ],
+          [
+            -22.598093032836914,
+            -39.98816680908203,
+            -44.34919738769531,
+            -29.085233688354492,
+            -15.14836311340332
+          ],
+          [
+            -24.54084014892578,
+            -43.015499114990234,
+            -47.237667083740234,
+            -31.39093017578125,
+            -16.20530128479004
+          ],
+          [
+            -26.530101776123047,
+            -46.71792221069336,
+            -50.609710693359375,
+            -33.21760940551758,
+            -17.194290161132812
+          ],
+          [
+            -28.196693420410156,
+            -50.25079345703125,
+            -53.16810989379883,
+            -35.350162506103516,
+            -18.18351173400879
+          ],
+          [
+            -29.735382080078125,
+            -52.41567611694336,
+            -55.63277053833008,
+            -36.781761169433594,
+            -19.38352394104004
+          ],
+          [
+            -31.10464859008789,
+            -55.84165573120117,
+            -57.993980407714844,
+            -38.734779357910156,
+            -20.228023529052734
+          ],
+          [
+            -32.75123977661133,
+            -59.13310241699219,
+            -60.02174758911133,
+            -40.260066986083984,
+            -21.13140296936035
+          ],
+          [
+            -34.169673919677734,
+            -62.30284881591797,
+            -62.26761245727539,
+            -42.105831146240234,
+            -21.92721939086914
+          ],
+          [
+            -35.710262298583984,
+            -65.6181869506836,
+            -64.38263702392578,
+            -43.33319091796875,
+            -22.71847152709961
+          ]
+        ],
+        "q99": [
+          [
+            9.732619285583496,
+            13.937226295471191,
+            17.37236213684082,
+            10.840984344482422,
+            5.523990154266357
+          ],
+          [
+            12.623029708862305,
+            17.370437622070312,
+            23.666194915771484,
+            13.33433723449707,
+            7.570279598236084
+          ],
+          [
+            15.670611381530762,
+            20.512556076049805,
+            29.823646545410156,
+            15.939998626708984,
+            9.560501098632812
+          ],
+          [
+            18.423181533813477,
+            24.460220336914062,
+            35.89162826538086,
+            18.452611923217773,
+            11.480408668518066
+          ],
+          [
+            21.29176902770996,
+            28.305740356445312,
+            41.52001190185547,
+            20.555370330810547,
+            13.225448608398438
+          ],
+          [
+            23.652997970581055,
+            32.20855712890625,
+            47.252647399902344,
+            22.819040298461914,
+            14.876164436340332
+          ],
+          [
+            26.150699615478516,
+            36.052101135253906,
+            53.16713333129883,
+            24.967885971069336,
+            16.451913833618164
+          ],
+          [
+            28.00914192199707,
+            39.1674919128418,
+            58.627197265625,
+            27.356786727905273,
+            18.208866119384766
+          ],
+          [
+            29.93129539489746,
+            42.67388153076172,
+            64.21937561035156,
+            29.275638580322266,
+            19.7204647064209
+          ],
+          [
+            32.04494094848633,
+            45.604549407958984,
+            69.7145004272461,
+            31.53823471069336,
+            21.124452590942383
+          ],
+          [
+            33.8387451171875,
+            48.230594635009766,
+            74.79926300048828,
+            33.60222625732422,
+            22.14889907836914
+          ],
+          [
+            35.145259857177734,
+            51.10918045043945,
+            79.912841796875,
+            35.561344146728516,
+            23.181623458862305
+          ],
+          [
+            36.698463439941406,
+            53.3221435546875,
+            84.57654571533203,
+            36.795257568359375,
+            24.493980407714844
+          ],
+          [
+            37.947391510009766,
+            55.879974365234375,
+            89.63903045654297,
+            38.53077697753906,
+            25.64667510986328
+          ],
+          [
+            39.297698974609375,
+            57.505733489990234,
+            94.03701782226562,
+            40.18893814086914,
+            26.982807159423828
+          ],
+          [
+            40.42583465576172,
+            59.16443634033203,
+            98.92611694335938,
+            41.37394714355469,
+            27.82160758972168
+          ]
+        ]
+      }
+    }
+  }
+}

ckpt/checkpoint-10000/experiment_cfg/final_model_config.json

@@ -0,0 +1,53 @@
+{
+  "model_type": "Gr00tN1d6",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "backbone_model_type": "eagle",
+  "model_revision": null,
+  "tune_top_llm_layers": 4,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": true,
+  "collator_overwrite_image_inputs": false,
+  "eagle_collator": true,
+  "backbone_trainable_params_fp32": true,
+  "apply_sincos_state_encoding": true,
+  "use_relative_action": true,
+  "max_state_dim": 128,
+  "max_action_dim": 128,
+  "action_horizon": 50,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.0,
+  "state_additive_noise_scale": 0.0,
+  "max_num_embodiments": 32
+}

ckpt/checkpoint-10000/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:448c4cd0dfedf8cbf58af31eb51f0744acc7cb7ca14cce693e0a4391a3b76d28
+size 4990120184

ckpt/checkpoint-10000/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16631189cc64c704275f0e14d15f883fbdfe27fd863cea1e3454b11d842e1b9e
+size 4823190320

ckpt/checkpoint-10000/processor_config.json

@@ -0,0 +1,455 @@
+{
+  "processor_class": "Gr00tN1d6Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "behavior_r1_pro": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "observation.images.rgb.head_256_256",
+            "observation.images.rgb.left_wrist_256_256",
+            "observation.images.rgb.right_wrist_256_256"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "robot_pos",
+            "robot_ori_cos",
+            "robot_ori_sin",
+            "robot_2d_ori",
+            "robot_2d_ori_cos",
+            "robot_2d_ori_sin",
+            "robot_lin_vel",
+            "robot_ang_vel",
+            "arm_left_qpos",
+            "arm_left_qpos_sin",
+            "arm_left_qpos_cos",
+            "eef_left_pos",
+            "eef_left_quat",
+            "gripper_left_qpos",
+            "arm_right_qpos",
+            "arm_right_qpos_sin",
+            "arm_right_qpos_cos",
+            "eef_right_pos",
+            "eef_right_quat",
+            "gripper_right_qpos",
+            "trunk_qpos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31
+          ],
+          "modality_keys": [
+            "base",
+            "torso",
+            "left_arm",
+            "left_gripper",
+            "right_arm",
+            "right_gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "trunk_qpos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_left_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_right_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "gr1": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "ego_view_bg_crop_pad_res256_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "robocasa_panda_omron": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "res256_image_side_0",
+            "res256_image_side_1",
+            "res256_image_wrist_0"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "end_effector_position_relative",
+            "end_effector_rotation_relative",
+            "gripper_qpos",
+            "base_position",
+            "base_rotation"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "end_effector_position",
+            "end_effector_rotation",
+            "gripper_close",
+            "base_motion",
+            "control_mode"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.action.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "front",
+            "wrist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": null,
+    "image_target_size": null,
+    "use_albumentations": true,
+    "random_rotation_angle": null,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "model_name": "nvidia/Eagle-Block2A-2B-v2",
+    "model_type": "eagle",
+    "formalize_language": true,
+    "max_state_dim": 128,
+    "max_action_dim": 128,
+    "max_action_horizon": 50,
+    "use_percentiles": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": true,
+    "use_relative_action": true
+  }
+}

ckpt/checkpoint-10000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f23d9dbb04f183208d38f860d86935f3415285bb2ac58730eb13ce55056f3803
+size 5713

ckpt/checkpoint-10000/wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d6", "run_id": "so101_finetune"}

configs_for_gr00t/modality.json

@@ -0,0 +1,36 @@
+{
+    "state": {
+        "single_arm": {
+            "start": 0,
+            "end": 5
+        },
+        "gripper": {
+            "start": 5,
+            "end": 6
+        }
+    },
+    "action": {
+        "single_arm": {
+            "start": 0,
+            "end": 5
+        },
+        "gripper": {
+            "start": 5,
+            "end": 6
+        }
+    },
+    "video": {
+        "front": {
+            "original_key": "observation.images.front"
+        },
+        "wrist": {
+            "original_key": "observation.images.wrist"
+        }
+    },
+    "annotation": {
+        "human.task_description": {
+            "original_key": "task_index"
+        }
+    }
+}
+

configs_for_gr00t/so101_config.py

@@ -0,0 +1,67 @@
+from gr00t.configs.data.embodiment_configs import register_modality_config
+from gr00t.data.embodiment_tags import EmbodimentTag
+from gr00t.data.types import (
+    ActionConfig,
+    ActionFormat,
+    ActionRepresentation,
+    ActionType,
+    ModalityConfig,
+)
+
+
+so101_config = {
+    "video": ModalityConfig(
+        delta_indices=[0],
+        modality_keys=["front", "wrist"],
+    ),
+    "state": ModalityConfig(
+        delta_indices=[0],
+        modality_keys=[
+            "single_arm",
+            "gripper",
+        ],
+    ),
+    "action": ModalityConfig(
+        delta_indices=[
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+        ],
+        modality_keys=[
+            "single_arm",
+            "gripper",
+        ],
+        action_configs=[
+            ActionConfig(
+                rep=ActionRepresentation.RELATIVE,
+                type=ActionType.NON_EEF,
+                format=ActionFormat.DEFAULT,
+            ),
+            ActionConfig(
+                rep=ActionRepresentation.ABSOLUTE,
+                type=ActionType.NON_EEF,
+                format=ActionFormat.DEFAULT,
+            ),
+        ],
+    ),
+    "language": ModalityConfig(
+        delta_indices=[0],
+        modality_keys=["annotation.human.task_description"],
+    ),
+}
+
+register_modality_config(so101_config, embodiment_tag=EmbodimentTag.NEW_EMBODIMENT)
+

scripts/finetune.sh

@@ -0,0 +1,25 @@
+set -x -e
+
+export NUM_GPUS=1
+
+# torchrun --nproc_per_node=$NUM_GPUS --master_port=29500 \
+CUDA_VISIBLE_DEVICES=0 python \
+    gr00t/experiment/launch_finetune.py \
+    --base_model_path nvidia/GR00T-N1.6-3B \
+    --dataset_path dataset/so101_pick_orange_v2.1 \
+    --modality_config_path configs/so101_config.py \
+    --embodiment_tag NEW_EMBODIMENT \
+    --num_gpus $NUM_GPUS \
+    --output_dir outputs/so101_finetune \
+    --save_steps 1000 \
+    --save_total_limit 5 \
+    --max_steps 10000 \
+    --warmup_ratio 0.05 \
+    --weight_decay 1e-5 \
+    --learning_rate 1e-4 \
+    --global_batch_size 32 \
+    --color_jitter_params brightness 0.3 contrast 0.4 saturation 0.5 hue 0.08 \
+    --use_wandb \
+    --dataloader_num_workers 4
+
+

scripts/gr00t_server.sh

@@ -0,0 +1,8 @@
+python gr00t/eval/run_gr00t_server.py \
+    --embodiment-tag NEW_EMBODIMENT \
+    --model-path ckpt/checkpoint-10000 \
+    --modality-config-path configs/modality.json \
+    --device cuda:0 \
+    --host 0.0.0.0 \
+    --port 5555 \
+    --strict

scripts/leisaac_client.sh

@@ -0,0 +1,11 @@
+python scripts/evaluation/policy_inference.py \
+    --task=LeIsaac-SO101-PickOrange-v0 \
+    --eval_rounds=10 \
+    --policy_type=gr00tn1.6 \
+    --policy_host=localhost \
+    --policy_port=5555 \
+    --policy_timeout_ms=5000 \
+    --policy_action_horizon=16 \
+    --policy_language_instruction="Pick up the orange and place it on the plate" \
+    --device=cuda \
+    --enable_cameras