Add files using upload-large-folder tool

config.json

@@ -0,0 +1,88 @@
+{
+  "action_horizon": 40,
+  "add_pos_embed": true,
+  "apply_sincos_state_encoding": false,
+  "architectures": [
+    "Gr00tN1d7"
+  ],
+  "attn_dropout": 0.2,
+  "attn_implementation": null,
+  "backbone_embedding_dim": 2048,
+  "backbone_trainable_params_fp32": true,
+  "color_jitter_params": {
+    "brightness": 0.3,
+    "contrast": 0.4,
+    "hue": 0.08,
+    "saturation": 0.5
+  },
+  "crop_fraction": 0.95,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "dtype": "float32",
+  "exclude_state": false,
+  "formalize_language": true,
+  "hidden_size": 1024,
+  "image_crop_size": [
+    230,
+    230
+  ],
+  "image_target_size": [
+    256,
+    256
+  ],
+  "letter_box_transform": false,
+  "load_bf16": false,
+  "max_action_dim": 132,
+  "max_num_embodiments": 32,
+  "max_seq_len": 1024,
+  "max_state_dim": 132,
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Cosmos-Reason2-2B",
+  "model_type": "Gr00tN1d7",
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_inference_timesteps": 4,
+  "num_timestep_buckets": 1000,
+  "random_history_crop": true,
+  "random_rotation_angle": 0,
+  "reproject_vision": false,
+  "rtc_ramp_rate": 6.0,
+  "select_layer": 16,
+  "shortest_image_edge": 256,
+  "state_dropout_prob": 0.2,
+  "state_gaussian_noise_std": 0.0,
+  "transformers_version": "4.57.3",
+  "tune_diffusion_model": true,
+  "tune_linear": true,
+  "tune_llm": false,
+  "tune_projector": true,
+  "tune_top_llm_layers": 0,
+  "tune_visual": false,
+  "tune_vlln": true,
+  "use_albumentations": true,
+  "use_alternate_vl_dit": true,
+  "use_flash_attention": true,
+  "use_future_tokens": false,
+  "use_mean_std": false,
+  "use_percentiles": true,
+  "use_vl_self_attention": true,
+  "use_vlln": true,
+  "vl_self_attention_cfg": {
+    "attention_head_dim": 64,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "num_attention_heads": 32,
+    "num_layers": 4,
+    "positional_embeddings": null
+  }
+}

embodiment_id.json

@@ -0,0 +1,56 @@
+{
+  "robocasa_panda_omron": 13,
+  "oxe_droid": 17,
+  "oxe_fractal": 18,
+  "oxe_language_table": 19,
+  "oxe_bridge": 20,
+  "unknown": 22,
+  "gr1_unified": 20,
+  "agibot": 26,
+  "sim_behavior_r1_pro": 23,
+  "xdof": 24,
+  "xdof_oss_data": 25,
+  "unitree_g1_full_body_with_waist_height_nav_cmd": 25,
+  "real_r1_pro_sharpa": 27,
+  "real_r1_pro_sharpa_add_view": 27,
+  "real_r1_pro_sharpa_relative_arm_joint": 26,
+  "real_r1_pro_sharpa_delta_eef": 26,
+  "real_r1_pro_sharpa_absolute_eef": 26,
+  "real_r1_pro_sharpa_meanstd": 26,
+  "real_r1_pro_sharpa_relative_eef": 26,
+  "real_r1_pro_sharpa_relative_eef_add_view": 26,
+  "real_r1_pro_sharpa_relative_eef_relative_hand": 26,
+  "real_r1_pro_sharpa_relative_eef_human": 26,
+  "real_r1_pro_sharpa_relative_eef_human_add_view": 26,
+  "real_r1_pro_sharpa_relative_eef_human_relative_hand": 26,
+  "real_r1_pro_sharpa_relative_eef_egodex": 26,
+  "real_r1_pro_sharpa_relative_eef_egodex_relative_hand": 26,
+  "real_r1_pro_sharpa_relative_eef_egodex_wrist_only": 26,
+  "real_r1_pro_sharpa_relative_eef_maxinsights": 26,
+  "real_r1_pro_sharpa_relative_eef_maxinsights_relative_hand": 26,
+  "real_r1_pro_sharpa_relative_eef_mecka": 26,
+  "real_r1_pro_sharpa_relative_eef_mecka_relative_hand": 26,
+  "real_g1_relative_eef_absolute_joints": 25,
+  "real_g1_relative_eef_absolute_joints_wrist_cam": 25,
+  "real_g1_relative_eef_relative_joints": 25,
+  "real_r1_pro_sharpa_relative_eef_relative_hand_relative_joint": 26,
+  "real_r1_pro_sharpa_relative_joint": 29,
+  "oxe_droid_relative_eef_relative_joint": 24,
+  "oxe_droid_relative_eef_relative_joint_swapped": 24,
+  "oxe_droid_relative_eef_relative_joint_upweight_z": 24,
+  "oxe_droid_relative_eef_relative_joint_upweight_z_swapped": 24,
+  "oxe_droid_relative_eef_relative_joint_3view": 24,
+  "oxe_droid_relative_eef_relative_joint_3view_swapped": 24,
+  "oxe_droid_relative_eef": 24,
+  "oxe_droid_joint_position_relative": 24,
+  "xdof_relative_eef_relative_joint": 27,
+  "xdof_relative_eef_relative_joint_subtask": 27,
+  "xdof_relative_eef": 27,
+  "xdof_relative_joint": 28,
+  "simpler_env_google": 0,
+  "simpler_env_widowx": 1,
+  "libero_sim": 2,
+  "droid_sim": 3,
+  "unitree_g1_sonic": 11,
+  "new_embodiment": 10
+}

experiment_cfg/conf.yaml

@@ -0,0 +1,218 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d7
+  model_dtype: bfloat16
+  model_name: nvidia/Cosmos-Reason2-2B
+  backbone_model_type: qwen
+  model_revision: null
+  tune_top_llm_layers: 0
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 12
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  backbone_trainable_params_fp32: true
+  image_crop_size:
+  - 230
+  - 230
+  image_target_size:
+  - 256
+  - 256
+  shortest_image_edge: null
+  crop_fraction: null
+  random_rotation_angle: null
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    saturation: 0.5
+    hue: 0.08
+  use_albumentations_transforms: true
+  extra_augmentation_config: null
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_percentiles: true
+  use_relative_action: true
+  max_state_dim: 132
+  max_action_dim: 132
+  action_horizon: 40
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  state_history_length: 1
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 16
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.2
+  exclude_state: false
+  use_mean_std: false
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - /workspace/Isaac-GR00T/examples/so101/merged_v2_out/thinhpt/merged_data_29042026
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - top
+        - side
+        - wrist
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: RELATIVE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: true
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: fork
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: torchcodec
+training:
+  output_dir: /workspace/Isaac-GR00T/outputs/so101_from_a100
+  experiment_name: null
+  max_steps: 1
+  global_batch_size: 1
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.7-3B
+  skip_weight_loading: false
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 1
+  save_total_limit: 5
+  save_vl_model: false
+  save_only_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 4
+  remove_unused_columns: false
+  use_wandb: false
+  wandb_project: finetune-gr00t-n1d7
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 1
+save_steps: 1

experiment_cfg/config.yaml

@@ -0,0 +1,244 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - /workspace/Isaac-GR00T/examples/so101/merged_v2_out/thinhpt/merged_data_29042026
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - relative
+          state_key: null
+          type: &id002 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: *id002
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - top
+        - side
+        - wrist
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: fork
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: true
+  random_chop: 0.0
+  seed: 42
+  shard_size: 1024
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: torchcodec
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d7.Gr00tN1d7Config
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  _output_attentions: false
+  add_cross_attention: false
+  architectures: null
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    hue: 0.08
+    saturation: 0.5
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 16
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  dtype: null
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  extra_augmentation_config: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Cosmos-Reason2-2B
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.2
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 4
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 1
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 1
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: /workspace/Isaac-GR00T/outputs/so101_from_a100
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_only_model: false
+  save_steps: 1
+  save_total_limit: 5
+  save_vl_model: false
+  skip_weight_loading: false
+  start_from_checkpoint: nvidia/GR00T-N1.7-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: false
+  wandb_project: finetune-gr00t-n1d7
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

experiment_cfg/dataset_statistics.json

@@ -0,0 +1,824 @@
+{
+  "new_embodiment": {
+    "state": {
+      "single_arm": {
+        "min": [
+          -54.74613571166992,
+          -98.57022857666016,
+          -43.726234436035156,
+          46.123172760009766,
+          -21.846965789794922
+        ],
+        "max": [
+          33.480499267578125,
+          44.49116897583008,
+          99.6197738647461,
+          99.37970733642578,
+          44.96042251586914
+        ],
+        "mean": [
+          -3.71223117675045,
+          -28.96169491491071,
+          38.268331990254325,
+          76.11713071680639,
+          8.719410478347504
+        ],
+        "std": [
+          19.7703417672378,
+          45.76361889878161,
+          36.95550118330955,
+          10.436981253480331,
+          10.28527453206456
+        ],
+        "q01": [
+          -28.35573358165663,
+          -98.37625679512502,
+          -6.215841300198603,
+          61.27545041751672,
+          -3.5717462805583824
+        ],
+        "q99": [
+          22.835918021244282,
+          16.923880809559762,
+          94.53338718633756,
+          94.08840435465513,
+          19.448960641935898
+        ]
+      },
+      "gripper": {
+        "min": [
+          1.6655563116073608
+        ],
+        "max": [
+          44.7701530456543
+        ],
+        "mean": [
+          12.234805392212133
+        ],
+        "std": [
+          10.744181979537759
+        ],
+        "q01": [
+          2.819680079656035
+        ],
+        "q99": [
+          29.035170235970288
+        ]
+      }
+    },
+    "action": {
+      "single_arm": {
+        "min": [
+          -54.81263732910156,
+          -100.0,
+          -47.463768005371094,
+          45.159915924072266,
+          -22.161027908325195
+        ],
+        "max": [
+          34.09257888793945,
+          43.78151321411133,
+          99.18478393554688,
+          100.0,
+          45.239967346191406
+        ],
+        "mean": [
+          -3.654850216570309,
+          -29.786135515368066,
+          36.804539250043725,
+          76.00833611622012,
+          8.795426704788387
+        ],
+        "std": [
+          19.86646705175531,
+          45.46621788789847,
+          37.49960349292722,
+          10.664087697927368,
+          10.308307671275646
+        ],
+        "q01": [
+          -28.57498577516432,
+          -99.09725515198915,
+          -9.289173182009739,
+          60.45941761745114,
+          -3.646041259007378
+        ],
+        "q99": [
+          23.24454320656618,
+          16.751953349857775,
+          94.22454394741354,
+          94.70154876475645,
+          19.739092304095912
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.0
+        ],
+        "max": [
+          45.26813888549805
+        ],
+        "mean": [
+          11.791910935375741
+        ],
+        "std": [
+          11.282201040172414
+        ],
+        "q01": [
+          1.7659824175548617
+        ],
+        "q99": [
+          29.41554971485232
+        ]
+      }
+    },
+    "relative_action": {
+      "single_arm": {
+        "min": [
+          [
+            -11.495086669921875,
+            -26.346839904785156,
+            -37.27923583984375,
+            -18.473644256591797,
+            -6.521447658538818
+          ],
+          [
+            -14.360626220703125,
+            -32.31824493408203,
+            -44.78874206542969,
+            -22.993900299072266,
+            -8.119771003723145
+          ],
+          [
+            -17.15679931640625,
+            -37.86912536621094,
+            -51.7279052734375,
+            -27.247333526611328,
+            -9.588430404663086
+          ],
+          [
+            -19.658639907836914,
+            -43.504112243652344,
+            -58.52138137817383,
+            -31.146312713623047,
+            -10.847281455993652
+          ],
+          [
+            -21.789426803588867,
+            -48.88678741455078,
+            -65.27043151855469,
+            -34.89897537231445,
+            -12.061001777648926
+          ],
+          [
+            -23.629013061523438,
+            -54.09234619140625,
+            -71.73430633544922,
+            -38.620731353759766,
+            -13.162496566772461
+          ],
+          [
+            -25.321434020996094,
+            -59.47502136230469,
+            -77.62784576416016,
+            -41.81081008911133,
+            -14.21790599822998
+          ],
+          [
+            -26.866689682006836,
+            -64.76913452148438,
+            -83.5213851928711,
+            -44.7958869934082,
+            -15.214496612548828
+          ],
+          [
+            -28.338359832763672,
+            -69.81539154052734,
+            -89.12974548339844,
+            -47.09982681274414,
+            -16.148725509643555
+          ],
+          [
+            -29.70535659790039,
+            -74.60933685302734,
+            -94.35787963867188,
+            -49.40259552001953,
+            -17.457027435302734
+          ],
+          [
+            -31.689193725585938,
+            -79.23507690429688,
+            -99.49095916748047,
+            -51.17485809326172,
+            -18.934598922729492
+          ],
+          [
+            -33.74650573730469,
+            -83.60482788085938,
+            -104.47283935546875,
+            -52.28360366821289,
+            -20.51771354675293
+          ],
+          [
+            -35.65967559814453,
+            -87.72593688964844,
+            -109.22569274902344,
+            -53.08112716674805,
+            -22.04805564880371
+          ],
+          [
+            -37.496559143066406,
+            -91.6755142211914,
+            -113.50326538085938,
+            -53.25835037231445,
+            -23.63117027282715
+          ],
+          [
+            -39.33344268798828,
+            -95.460205078125,
+            -117.40060424804688,
+            -53.169734954833984,
+            -25.16151237487793
+          ],
+          [
+            -41.02586364746094,
+            -98.82437896728516,
+            -120.7520523071289,
+            -53.169734954833984,
+            -26.48077392578125
+          ]
+        ],
+        "max": [
+          [
+            12.493302345275879,
+            26.105972290039062,
+            30.445114135742188,
+            12.794731140136719,
+            11.564742088317871
+          ],
+          [
+            15.750332832336426,
+            31.736225128173828,
+            37.09910583496094,
+            15.896194458007812,
+            13.977540016174316
+          ],
+          [
+            19.13020133972168,
+            37.203006744384766,
+            43.658042907714844,
+            18.710693359375,
+            16.2994384765625
+          ],
+          [
+            22.294292449951172,
+            42.249263763427734,
+            50.089202880859375,
+            21.36908721923828,
+            18.040863037109375
+          ],
+          [
+            25.458383560180664,
+            47.207244873046875,
+            56.339202880859375,
+            23.910110473632812,
+            19.614429473876953
+          ],
+          [
+            28.61782455444336,
+            52.08119583129883,
+            62.13768005371094,
+            26.479888916015625,
+            20.93368911743164
+          ],
+          [
+            31.63031578063965,
+            56.62282180786133,
+            67.5559310913086,
+            28.606605529785156,
+            22.16421127319336
+          ],
+          [
+            34.57365417480469,
+            60.82803726196289,
+            72.5378189086914,
+            30.482940673828125,
+            24.314748764038086
+          ],
+          [
+            37.52121353149414,
+            65.02971649169922,
+            77.38572692871094,
+            32.077980041503906,
+            26.465286254882812
+          ],
+          [
+            40.74937438964844,
+            68.73030853271484,
+            81.73355102539062,
+            33.05272674560547,
+            28.470563888549805
+          ],
+          [
+            43.98704528808594,
+            72.0916519165039,
+            85.90021514892578,
+            34.44295120239258,
+            30.358840942382812
+          ],
+          [
+            46.99953842163086,
+            75.61852264404297,
+            89.70249938964844,
+            36.21521759033203,
+            31.941953659057617
+          ],
+          [
+            49.86929702758789,
+            79.2350082397461,
+            93.31466674804688,
+            37.83568572998047,
+            33.30570983886719
+          ],
+          [
+            52.73905563354492,
+            82.84844970703125,
+            96.39437866210938,
+            39.430721282958984,
+            34.72191619873047
+          ],
+          [
+            55.4576416015625,
+            85.87620544433594,
+            99.34114074707031,
+            40.79533004760742,
+            35.935638427734375
+          ],
+          [
+            57.95948028564453,
+            88.73335266113281,
+            101.96795654296875,
+            41.59284973144531,
+            37.29207229614258
+          ]
+        ],
+        "mean": [
+          [
+            0.055648088455200195,
+            -0.8320212960243225,
+            -1.514521598815918,
+            -0.11044922471046448,
+            0.0779842957854271
+          ],
+          [
+            0.053355250507593155,
+            -0.831329882144928,
+            -1.5161784887313843,
+            -0.11004702746868134,
+            0.07830151915550232
+          ],
+          [
+            0.05104710906744003,
+            -0.8308613896369934,
+            -1.5175775289535522,
+            -0.1095501035451889,
+            0.07863958179950714
+          ],
+          [
+            0.048748865723609924,
+            -0.8305501341819763,
+            -1.5187740325927734,
+            -0.10894875973463058,
+            0.07897468656301498
+          ],
+          [
+            0.046457186341285706,
+            -0.8303706049919128,
+            -1.5198224782943726,
+            -0.10824387520551682,
+            0.07932591438293457
+          ],
+          [
+            0.044185150414705276,
+            -0.8302662372589111,
+            -1.5207382440567017,
+            -0.10747325420379639,
+            0.07968516647815704
+          ],
+          [
+            0.04192883148789406,
+            -0.8302180171012878,
+            -1.5215797424316406,
+            -0.10666685551404953,
+            0.08004017919301987
+          ],
+          [
+            0.03968740254640579,
+            -0.8302181363105774,
+            -1.5223520994186401,
+            -0.10584192723035812,
+            0.08038759976625443
+          ],
+          [
+            0.037463363260030746,
+            -0.830231785774231,
+            -1.5230622291564941,
+            -0.10499752312898636,
+            0.0807308703660965
+          ],
+          [
+            0.03524709865450859,
+            -0.8302584290504456,
+            -1.5237103700637817,
+            -0.10413610190153122,
+            0.08106588572263718
+          ],
+          [
+            0.0330469124019146,
+            -0.8302862048149109,
+            -1.5243130922317505,
+            -0.10326503962278366,
+            0.08139366656541824
+          ],
+          [
+            0.03083740547299385,
+            -0.8303185701370239,
+            -1.524881362915039,
+            -0.10238658636808395,
+            0.08171631395816803
+          ],
+          [
+            0.0286197941750288,
+            -0.8303502202033997,
+            -1.5253970623016357,
+            -0.10150988399982452,
+            0.08205557614564896
+          ],
+          [
+            0.026385115459561348,
+            -0.8303899168968201,
+            -1.5258857011795044,
+            -0.10063113272190094,
+            0.08241897076368332
+          ],
+          [
+            0.024145090952515602,
+            -0.8304489850997925,
+            -1.5263406038284302,
+            -0.09974220395088196,
+            0.08279348164796829
+          ],
+          [
+            0.021909138187766075,
+            -0.830514132976532,
+            -1.52675199508667,
+            -0.09885106235742569,
+            0.0831804946064949
+          ]
+        ],
+        "std": [
+          [
+            2.3281056880950928,
+            5.052164554595947,
+            5.022761344909668,
+            2.1672933101654053,
+            1.1374194622039795
+          ],
+          [
+            2.95160174369812,
+            6.290643692016602,
+            6.232492923736572,
+            2.734529733657837,
+            1.4302462339401245
+          ],
+          [
+            3.5650782585144043,
+            7.519575595855713,
+            7.430261135101318,
+            3.2848517894744873,
+            1.710359811782837
+          ],
+          [
+            4.166775226593018,
+            8.734842300415039,
+            8.610282897949219,
+            3.8151462078094482,
+            1.9777880907058716
+          ],
+          [
+            4.755683422088623,
+            9.933793067932129,
+            9.769296646118164,
+            4.323939323425293,
+            2.2334964275360107
+          ],
+          [
+            5.3314666748046875,
+            11.115108489990234,
+            10.905523300170898,
+            4.810477256774902,
+            2.47870135307312
+          ],
+          [
+            5.89393949508667,
+            12.278027534484863,
+            12.017640113830566,
+            5.27458381652832,
+            2.7145771980285645
+          ],
+          [
+            6.4430437088012695,
+            13.421923637390137,
+            13.105111122131348,
+            5.716440677642822,
+            2.942037343978882
+          ],
+          [
+            6.979062557220459,
+            14.54690933227539,
+            14.167784690856934,
+            6.136516571044922,
+            3.1618080139160156
+          ],
+          [
+            7.502256393432617,
+            15.652772903442383,
+            15.205517768859863,
+            6.5353217124938965,
+            3.374497175216675
+          ],
+          [
+            8.012919425964355,
+            16.739797592163086,
+            16.218400955200195,
+            6.913644313812256,
+            3.5806076526641846
+          ],
+          [
+            8.511361122131348,
+            17.80817985534668,
+            17.206693649291992,
+            7.272296905517578,
+            3.780578851699829
+          ],
+          [
+            8.997858047485352,
+            18.858156204223633,
+            18.170631408691406,
+            7.611952304840088,
+            3.9747884273529053
+          ],
+          [
+            9.472798347473145,
+            19.890417098999023,
+            19.110652923583984,
+            7.9334397315979,
+            4.163602828979492
+          ],
+          [
+            9.936485290527344,
+            20.905296325683594,
+            20.026742935180664,
+            8.237541198730469,
+            4.34731912612915
+          ],
+          [
+            10.389312744140625,
+            21.903194427490234,
+            20.919414520263672,
+            8.524994850158691,
+            4.526242733001709
+          ]
+        ],
+        "q01": [
+          [
+            -6.665879402160645,
+            -14.016157150268555,
+            -18.17994812011719,
+            -6.566455993652344,
+            -3.179898805618286
+          ],
+          [
+            -8.507232131958007,
+            -17.456420440673828,
+            -22.29572769165039,
+            -8.339033508300782,
+            -4.072495708465576
+          ],
+          [
+            -10.31580192565918,
+            -20.88756576538086,
+            -26.324436950683594,
+            -10.068443908691407,
+            -4.926240749359131
+          ],
+          [
+            -11.974199905395508,
+            -24.180819549560546,
+            -30.219016723632812,
+            -11.673286437988281,
+            -5.754124221801757
+          ],
+          [
+            -13.622781486511231,
+            -27.586204528808594,
+            -34.089948883056636,
+            -13.132738342285156,
+            -6.494998493194579
+          ],
+          [
+            -15.208937492370605,
+            -30.715314025878907,
+            -37.71124755859375,
+            -14.523447875976562,
+            -7.202737331390381
+          ],
+          [
+            -16.70104965209961,
+            -34.017022399902345,
+            -41.36379302978516,
+            -15.868139343261719,
+            -7.887644081115722
+          ],
+          [
+            -18.089058227539063,
+            -37.035921325683596,
+            -45.10492156982422,
+            -17.046712951660155,
+            -8.534011497497557
+          ],
+          [
+            -19.427554321289062,
+            -40.144180603027344,
+            -48.51883255004883,
+            -18.154289855957032,
+            -9.11346351623535
+          ],
+          [
+            -20.61064712524414,
+            -43.16117599487305,
+            -51.93670974731445,
+            -19.193991088867186,
+            -9.762433395385742
+          ],
+          [
+            -21.711649627685546,
+            -45.93755310058594,
+            -55.125614166259766,
+            -20.116388397216795,
+            -10.3573006439209
+          ],
+          [
+            -22.738679428100586,
+            -48.97865203857422,
+            -58.49867599487305,
+            -20.9490185546875,
+            -10.945838890075683
+          ],
+          [
+            -23.753029708862304,
+            -51.72798324584961,
+            -61.444978332519526,
+            -21.6744921875,
+            -11.450961227416991
+          ],
+          [
+            -24.652152328491212,
+            -54.50247207641601,
+            -64.40270690917968,
+            -22.302978515625,
+            -11.953850250244141
+          ],
+          [
+            -25.526405487060547,
+            -57.1378970336914,
+            -67.23038177490234,
+            -22.905455627441405,
+            -12.480633811950684
+          ],
+          [
+            -26.35790428161621,
+            -59.974665832519534,
+            -70.08912353515625,
+            -23.482775268554686,
+            -12.981596565246582
+          ]
+        ],
+        "q99": [
+          [
+            8.16102737426758,
+            17.203563842773438,
+            12.228648223876954,
+            6.190132141113281,
+            4.2718777465820335
+          ],
+          [
+            10.346405105590824,
+            21.301721725463867,
+            15.796453704833986,
+            7.958911743164063,
+            5.424470520019533
+          ],
+          [
+            12.521966819763184,
+            25.372473144531252,
+            19.33406288146973,
+            9.619446105957032,
+            6.516082611083985
+          ],
+          [
+            14.666381454467773,
+            29.38535583496094,
+            22.816484527587896,
+            11.262754211425781,
+            7.482540168762211
+          ],
+          [
+            16.824214706420904,
+            33.260471496582035,
+            26.28074981689453,
+            12.785390930175783,
+            8.422623748779298
+          ],
+          [
+            18.910324478149413,
+            37.02602508544924,
+            29.526105651855474,
+            14.186925659179687,
+            9.320058593750003
+          ],
+          [
+            20.967150573730468,
+            40.583617248535155,
+            32.72448623657227,
+            15.530707397460938,
+            10.081392669677737
+          ],
+          [
+            22.92281455993653,
+            43.98616302490237,
+            35.79875701904297,
+            16.797211303710938,
+            10.83802040100098
+          ],
+          [
+            24.86040229797364,
+            47.42544555664064,
+            38.77677886962892,
+            18.068880004882814,
+            11.41708156585694
+          ],
+          [
+            26.8188102722168,
+            50.64716796875,
+            41.6482504272461,
+            19.176239013671875,
+            12.060648956298829
+          ],
+          [
+            28.741224975585936,
+            53.73454513549807,
+            44.526905212402355,
+            20.276117095947267,
+            12.697874450683594
+          ],
+          [
+            30.602312393188477,
+            56.670256195068376,
+            47.06060317993165,
+            21.301702880859377,
+            13.154049987792968
+          ],
+          [
+            32.47877929687501,
+            59.32594985961914,
+            49.71035003662111,
+            22.218926696777345,
+            13.643209037780762
+          ],
+          [
+            34.18646453857422,
+            62.087795410156254,
+            52.03034790039063,
+            23.099743499755867,
+            14.102179527282715
+          ],
+          [
+            35.960482940673835,
+            64.44493347167969,
+            54.46940231323242,
+            23.925567626953125,
+            14.555956344604493
+          ],
+          [
+            37.56228790283204,
+            66.62295715332031,
+            56.67596618652344,
+            24.686989135742195,
+            14.986904106140141
+          ]
+        ]
+      }
+    }
+  }
+}

experiment_cfg/final_model_config.json

@@ -0,0 +1,55 @@
+{
+  "model_type": "Gr00tN1d7",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Cosmos-Reason2-2B",
+  "backbone_model_type": "qwen",
+  "model_revision": null,
+  "tune_top_llm_layers": 0,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": false,
+  "backbone_trainable_params_fp32": true,
+  "extra_augmentation_config": null,
+  "apply_sincos_state_encoding": false,
+  "use_percentiles": true,
+  "use_relative_action": false,
+  "max_state_dim": 132,
+  "max_action_dim": 132,
+  "action_horizon": 40,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "state_history_length": 1,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.2,
+  "exclude_state": false,
+  "use_mean_std": false,
+  "max_num_embodiments": 32
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbcaea5ee88f1e0f1465043920a2647c67e7de17d24adfd1c477742a6168edec
+size 4986649584

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c39153e2c89aa5c7b6423b2fb05725d21fa1b594fe61dbebaffa7956736848e3
+size 4970792616

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0a78a2f4ce9af011f57e67e441ddd1b02e358278800f883914fae334bfed225
+size 2618758696

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88d086b13a9eb7c0a94e3d19d85a32160715d57b185332c1bcef1a8775354d52
+size 12964594710

processor_config.json

@@ -0,0 +1,1124 @@
+{
+  "processor_class": "Gr00tN1d7Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "real_g1_relative_eef_relative_joints": {
+        "video": {
+          "delta_indices": [
+            -20,
+            0
+          ],
+          "modality_keys": [
+            "ego_view"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef_9d",
+            "right_wrist_eef_9d",
+            "left_hand",
+            "right_hand",
+            "left_arm",
+            "right_arm",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef_9d",
+            "right_wrist_eef_9d",
+            "left_hand",
+            "right_hand",
+            "left_arm",
+            "right_arm",
+            "waist",
+            "base_height_command",
+            "navigate_command"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef_9d"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef_9d"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_arm"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_arm"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "waist"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "base_height_command"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "navigate_command"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "real_r1_pro_sharpa_relative_eef_mecka": {
+        "video": {
+          "delta_indices": [
+            -30,
+            0
+          ],
+          "modality_keys": [
+            "ego_view_cropratio_res320x240_freq30"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand_joints"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand_joints"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "real_r1_pro_sharpa_relative_eef_human": {
+        "video": {
+          "delta_indices": [
+            -20,
+            0
+          ],
+          "modality_keys": [
+            "ego_view_res320x240_freq20",
+            "left_wrist_view_res320x240_freq20",
+            "right_wrist_view_res320x240_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand_joints"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand_joints"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "real_r1_pro_sharpa_relative_eef": {
+        "video": {
+          "delta_indices": [
+            -20,
+            0
+          ],
+          "modality_keys": [
+            "ego_view_res320x240_freq20",
+            "left_wrist_view_res320x240_freq20",
+            "right_wrist_view_res320x240_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand_joints"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand_joints"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "xdof_relative_eef_relative_joint": {
+        "video": {
+          "delta_indices": [
+            -30,
+            0
+          ],
+          "modality_keys": [
+            "top_camera-images-rgb_320_240",
+            "left_camera-images-rgb_320_240",
+            "right_camera-images-rgb_320_240"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_gripper_pos",
+            "right_gripper_pos",
+            "left_joint_pos",
+            "right_joint_pos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_gripper_pos",
+            "right_gripper_pos",
+            "left_joint_pos",
+            "right_joint_pos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_gripper_pos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_gripper_pos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_joint_pos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_joint_pos"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "real_r1_pro_sharpa_relative_eef_maxinsights": {
+        "video": {
+          "delta_indices": [
+            -30,
+            0
+          ],
+          "modality_keys": [
+            "ego_view_cropratio_res320x240_freq30"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand_joints"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand_joints"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "xdof_relative_eef_relative_joint_subtask": {
+        "video": {
+          "delta_indices": [
+            -30,
+            0
+          ],
+          "modality_keys": [
+            "top_camera-images-rgb_320_240",
+            "left_camera-images-rgb_320_240",
+            "right_camera-images-rgb_320_240"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_gripper_pos",
+            "right_gripper_pos",
+            "left_joint_pos",
+            "right_joint_pos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_gripper_pos",
+            "right_gripper_pos",
+            "left_joint_pos",
+            "right_joint_pos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_gripper_pos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_gripper_pos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_joint_pos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_joint_pos"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.sub_task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "oxe_droid_relative_eef_relative_joint": {
+        "video": {
+          "delta_indices": [
+            -15,
+            0
+          ],
+          "modality_keys": [
+            "exterior_image_1_left",
+            "wrist_image_left"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "eef_9d",
+            "gripper_position",
+            "joint_position"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "eef_9d",
+            "gripper_position",
+            "joint_position"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "eef_9d"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "gripper_position"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "joint_position"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.language.language_instruction"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "top",
+            "side",
+            "wrist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": [
+      230,
+      230
+    ],
+    "image_target_size": [
+      256,
+      256
+    ],
+    "use_albumentations": true,
+    "random_rotation_angle": 0,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "letter_box_transform": false,
+    "model_name": "nvidia/Cosmos-Reason2-2B",
+    "model_type": "qwen",
+    "formalize_language": true,
+    "max_state_dim": 132,
+    "max_action_dim": 132,
+    "max_action_horizon": 40,
+    "use_percentiles": true,
+    "use_mean_std": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": false,
+    "use_relative_action": true,
+    "exclude_state": false,
+    "state_dropout_prob": 0.2
+  }
+}

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e9154c5e9ad35634697c0726384af7d8c924ac0c4f2e1497fe3a12c241f9e46
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f42d91efd686c2847e9f7c6381a82591c8707df06f61850f29ee39644815b943
+size 1465

trainer_state.json

@@ -0,0 +1,33 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 1,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [],
+  "logging_steps": 10,
+  "max_steps": 1,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 1,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c12ff511086145db8b0a574153fb71263f4cca6aa25ea6f78ad1f4ed3b1f7c1
+size 5841

wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d7", "run_id": "so101_from_a100"}