Upload folder using huggingface_hub

README.md

@@ -0,0 +1,19 @@
+---
+license: apache-2.0
+base_model: nvidia/GR00T-N1.7-3B
+library_name: gr00t
+tags: [robotics, gr00t, isaac-gr00t, lerobot, uf850, vla, manipulation]
+---
+# UF850 GR00T-N1.7 fine-tune (pick & place)
+
+Fine-tune of **NVIDIA Isaac-GR00T N1.7-3B** on UFactory 850 Meta-Quest teleoperation demos.
+
+- **Base model:** nvidia/GR00T-N1.7-3B
+- **Embodiment:** `NEW_EMBODIMENT` (7-DoF: joints j1..j6 + gripper)
+- **Task:** "Pick up the cube. Place it on box. Pick up the cube again. Place it on table."
+- **Data:** 101 LeRobot v2.1 episodes, base + wrist cameras (640x480 @ 30 fps)
+- **Training:** 20,000 steps, global batch 32, action-head fine-tune (frozen VLM backbone)
+- **Final loss:** 0.0237 (min 0.0198)
+
+Inference checkpoint — optimizer state omitted. Load with the Isaac-GR00T stack using the
+`NEW_EMBODIMENT` modality config (single_arm 0:6 + gripper 6, base/wrist cameras).

config.json

@@ -0,0 +1,88 @@
+{
+  "action_horizon": 40,
+  "add_pos_embed": true,
+  "apply_sincos_state_encoding": false,
+  "architectures": [
+    "Gr00tN1d7"
+  ],
+  "attn_dropout": 0.2,
+  "attn_implementation": null,
+  "backbone_embedding_dim": 2048,
+  "backbone_trainable_params_fp32": true,
+  "color_jitter_params": {
+    "brightness": 0.3,
+    "contrast": 0.4,
+    "hue": 0.08,
+    "saturation": 0.5
+  },
+  "crop_fraction": 0.95,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "dtype": "float32",
+  "exclude_state": false,
+  "formalize_language": true,
+  "hidden_size": 1024,
+  "image_crop_size": [
+    230,
+    230
+  ],
+  "image_target_size": [
+    256,
+    256
+  ],
+  "letter_box_transform": false,
+  "load_bf16": false,
+  "max_action_dim": 132,
+  "max_num_embodiments": 32,
+  "max_seq_len": 1024,
+  "max_state_dim": 132,
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Cosmos-Reason2-2B",
+  "model_type": "Gr00tN1d7",
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_inference_timesteps": 4,
+  "num_timestep_buckets": 1000,
+  "random_history_crop": true,
+  "random_rotation_angle": 0,
+  "reproject_vision": false,
+  "rtc_ramp_rate": 6.0,
+  "select_layer": 16,
+  "shortest_image_edge": 256,
+  "state_dropout_prob": 0.2,
+  "state_gaussian_noise_std": 0.0,
+  "transformers_version": "4.57.3",
+  "tune_diffusion_model": true,
+  "tune_linear": true,
+  "tune_llm": false,
+  "tune_projector": true,
+  "tune_top_llm_layers": 0,
+  "tune_visual": false,
+  "tune_vlln": true,
+  "use_albumentations": true,
+  "use_alternate_vl_dit": true,
+  "use_flash_attention": true,
+  "use_future_tokens": false,
+  "use_mean_std": false,
+  "use_percentiles": true,
+  "use_vl_self_attention": true,
+  "use_vlln": true,
+  "vl_self_attention_cfg": {
+    "attention_head_dim": 64,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "num_attention_heads": 32,
+    "num_layers": 4,
+    "positional_embeddings": null
+  }
+}

embodiment_id.json

@@ -0,0 +1,56 @@
+{
+  "robocasa_panda_omron": 13,
+  "oxe_droid": 17,
+  "oxe_fractal": 18,
+  "oxe_language_table": 19,
+  "oxe_bridge": 20,
+  "unknown": 22,
+  "gr1_unified": 20,
+  "agibot": 26,
+  "sim_behavior_r1_pro": 23,
+  "xdof": 24,
+  "xdof_oss_data": 25,
+  "unitree_g1_full_body_with_waist_height_nav_cmd": 25,
+  "real_r1_pro_sharpa": 27,
+  "real_r1_pro_sharpa_add_view": 27,
+  "real_r1_pro_sharpa_relative_arm_joint": 26,
+  "real_r1_pro_sharpa_delta_eef": 26,
+  "real_r1_pro_sharpa_absolute_eef": 26,
+  "real_r1_pro_sharpa_meanstd": 26,
+  "real_r1_pro_sharpa_relative_eef": 26,
+  "real_r1_pro_sharpa_relative_eef_add_view": 26,
+  "real_r1_pro_sharpa_relative_eef_relative_hand": 26,
+  "real_r1_pro_sharpa_relative_eef_human": 26,
+  "real_r1_pro_sharpa_relative_eef_human_add_view": 26,
+  "real_r1_pro_sharpa_relative_eef_human_relative_hand": 26,
+  "real_r1_pro_sharpa_relative_eef_egodex": 26,
+  "real_r1_pro_sharpa_relative_eef_egodex_relative_hand": 26,
+  "real_r1_pro_sharpa_relative_eef_egodex_wrist_only": 26,
+  "real_r1_pro_sharpa_relative_eef_maxinsights": 26,
+  "real_r1_pro_sharpa_relative_eef_maxinsights_relative_hand": 26,
+  "real_r1_pro_sharpa_relative_eef_mecka": 26,
+  "real_r1_pro_sharpa_relative_eef_mecka_relative_hand": 26,
+  "real_g1_relative_eef_absolute_joints": 25,
+  "real_g1_relative_eef_absolute_joints_wrist_cam": 25,
+  "real_g1_relative_eef_relative_joints": 25,
+  "real_r1_pro_sharpa_relative_eef_relative_hand_relative_joint": 26,
+  "real_r1_pro_sharpa_relative_joint": 29,
+  "oxe_droid_relative_eef_relative_joint": 24,
+  "oxe_droid_relative_eef_relative_joint_swapped": 24,
+  "oxe_droid_relative_eef_relative_joint_upweight_z": 24,
+  "oxe_droid_relative_eef_relative_joint_upweight_z_swapped": 24,
+  "oxe_droid_relative_eef_relative_joint_3view": 24,
+  "oxe_droid_relative_eef_relative_joint_3view_swapped": 24,
+  "oxe_droid_relative_eef": 24,
+  "oxe_droid_joint_position_relative": 24,
+  "xdof_relative_eef_relative_joint": 27,
+  "xdof_relative_eef_relative_joint_subtask": 27,
+  "xdof_relative_eef": 27,
+  "xdof_relative_joint": 28,
+  "simpler_env_google": 0,
+  "simpler_env_widowx": 1,
+  "libero_sim": 2,
+  "droid_sim": 3,
+  "unitree_g1_sonic": 11,
+  "new_embodiment": 10
+}

experiment_cfg/conf.yaml

@@ -0,0 +1,213 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d7
+  model_dtype: bfloat16
+  model_name: nvidia/Cosmos-Reason2-2B
+  backbone_model_type: qwen
+  model_revision: null
+  tune_top_llm_layers: 0
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 12
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  backbone_trainable_params_fp32: true
+  image_crop_size:
+  - 230
+  - 230
+  image_target_size:
+  - 256
+  - 256
+  shortest_image_edge: null
+  crop_fraction: null
+  random_rotation_angle: null
+  color_jitter_params: null
+  use_albumentations_transforms: true
+  extra_augmentation_config: null
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_percentiles: true
+  use_relative_action: true
+  max_state_dim: 132
+  max_action_dim: 132
+  action_horizon: 40
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  state_history_length: 1
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 16
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.2
+  exclude_state: false
+  use_mean_std: false
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - /dev/shm/uf850_data
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - base
+        - wrist
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: RELATIVE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 256
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: true
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: spawn
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: opencv
+training:
+  output_dir: /workspace/uf850_ckpt
+  experiment_name: null
+  max_steps: 20000
+  global_batch_size: 32
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.7-3B
+  skip_weight_loading: false
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 2000
+  save_total_limit: 2
+  save_vl_model: false
+  save_only_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 4
+  remove_unused_columns: false
+  use_wandb: false
+  wandb_project: finetune-gr00t-n1d7
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 20000
+save_steps: 2000

experiment_cfg/config.yaml

@@ -0,0 +1,239 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - /dev/shm/uf850_data
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - relative
+          state_key: null
+          type: &id002 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: *id002
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - base
+        - wrist
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: spawn
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: true
+  random_chop: 0.0
+  seed: 42
+  shard_size: 256
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: opencv
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d7.Gr00tN1d7Config
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  _output_attentions: false
+  add_cross_attention: false
+  architectures: null
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params: null
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 16
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  dtype: null
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  extra_augmentation_config: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Cosmos-Reason2-2B
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.2
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 4
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 32
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 20000
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: /workspace/uf850_ckpt
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_only_model: false
+  save_steps: 2000
+  save_total_limit: 2
+  save_vl_model: false
+  skip_weight_loading: false
+  start_from_checkpoint: nvidia/GR00T-N1.7-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: false
+  wandb_project: finetune-gr00t-n1d7
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

experiment_cfg/dataset_statistics.json

@@ -0,0 +1,932 @@
+{
+  "new_embodiment": {
+    "state": {
+      "single_arm": {
+        "min": [
+          -0.592655599117279,
+          -1.4958008527755737,
+          -2.4152991771698,
+          -3.011120080947876,
+          0.013632671907544136,
+          -1.3678452968597412
+        ],
+        "max": [
+          0.44287005066871643,
+          -0.13162055611610413,
+          -0.27703857421875,
+          0.9268300533294678,
+          1.356399655342102,
+          2.651588201522827
+        ],
+        "mean": [
+          -0.08583929389715195,
+          -0.6927860975265503,
+          -0.9374713897705078,
+          -0.37021225690841675,
+          0.6744750142097473,
+          0.048665501177310944
+        ],
+        "std": [
+          0.1581176072359085,
+          0.16969016194343567,
+          0.32089462876319885,
+          0.4413868486881256,
+          0.2593396008014679,
+          0.3505861461162567
+        ],
+        "q01": [
+          -0.41856096327304837,
+          -1.0545359003543853,
+          -1.6649452090263366,
+          -2.207083568572998,
+          0.1486940871179104,
+          -0.5505770820379258
+        ],
+        "q99": [
+          0.30973009020090103,
+          -0.328880171477795,
+          -0.36008988618850774,
+          0.45472134292125593,
+          1.1975745999813079,
+          1.7058103108405898
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.0
+        ],
+        "max": [
+          850.0
+        ],
+        "mean": [
+          585.05615234375
+        ],
+        "std": [
+          392.2166748046875
+        ],
+        "q01": [
+          0.0
+        ],
+        "q99": [
+          850.0
+        ]
+      }
+    },
+    "action": {
+      "single_arm": {
+        "min": [
+          -0.592655599117279,
+          -1.4958008527755737,
+          -2.4152991771698,
+          -3.011120080947876,
+          0.013632671907544136,
+          -1.3678452968597412
+        ],
+        "max": [
+          0.44287005066871643,
+          -0.13162055611610413,
+          -0.27703857421875,
+          0.9268300533294678,
+          1.356399655342102,
+          2.651588201522827
+        ],
+        "mean": [
+          -0.08585226535797119,
+          -0.6928074955940247,
+          -0.9374907612800598,
+          -0.37025517225265503,
+          0.6743330955505371,
+          0.04860541597008705
+        ],
+        "std": [
+          0.15810510516166687,
+          0.1696782261133194,
+          0.3208761513233185,
+          0.441365122795105,
+          0.25915467739105225,
+          0.3506162464618683
+        ],
+        "q01": [
+          -0.41882782101631166,
+          -1.0545359003543853,
+          -1.6649452090263366,
+          -2.207083568572998,
+          0.1486940871179104,
+          -0.5505770820379258
+        ],
+        "q99": [
+          0.30973009020090103,
+          -0.3288060897588754,
+          -0.35992349147796643,
+          0.45333125621079273,
+          1.1970528852939601,
+          1.7058103108405898
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.0
+        ],
+        "max": [
+          850.0
+        ],
+        "mean": [
+          585.05615234375
+        ],
+        "std": [
+          392.2166748046875
+        ],
+        "q01": [
+          0.0
+        ],
+        "q99": [
+          850.0
+        ]
+      }
+    },
+    "relative_action": {
+      "single_arm": {
+        "min": [
+          [
+            -0.12355075031518936,
+            -0.12977290153503418,
+            -0.16386601328849792,
+            -0.08984075486660004,
+            -0.13963323831558228,
+            -0.11960072815418243
+          ],
+          [
+            -0.24710150063037872,
+            -0.25485509634017944,
+            -0.32773202657699585,
+            -0.16824933886528015,
+            -0.27926647663116455,
+            -0.23920145630836487
+          ],
+          [
+            -0.24763162434101105,
+            -0.28482967615127563,
+            -0.45218825340270996,
+            -0.21913906931877136,
+            -0.34148597717285156,
+            -0.29530635476112366
+          ],
+          [
+            -0.24961964786052704,
+            -0.28964394330978394,
+            -0.45724189281463623,
+            -0.2748974561691284,
+            -0.3437540829181671,
+            -0.39374181628227234
+          ],
+          [
+            -0.2514687776565552,
+            -0.2944437265396118,
+            -0.4615743160247803,
+            -0.3186691999435425,
+            -0.34611383080482483,
+            -0.39358893036842346
+          ],
+          [
+            -0.2527675926685333,
+            -0.2981773614883423,
+            -0.4656026363372803,
+            -0.3822597563266754,
+            -0.3520084023475647,
+            -0.4343809485435486
+          ],
+          [
+            -0.2548658549785614,
+            -0.3021867871284485,
+            -0.4688936471939087,
+            -0.4390004277229309,
+            -0.3546655774116516,
+            -0.5136688351631165
+          ],
+          [
+            -0.25675681233406067,
+            -0.30662304162979126,
+            -0.47082948684692383,
+            -0.46055588126182556,
+            -0.35693368315696716,
+            -0.5528051257133484
+          ],
+          [
+            -0.2580749988555908,
+            -0.3166230320930481,
+            -0.4727663993835449,
+            -0.46055588126182556,
+            -0.3592934310436249,
+            -0.6277968287467957
+          ],
+          [
+            -0.2584461271762848,
+            -0.3188696503639221,
+            -0.47546589374542236,
+            -0.46735572814941406,
+            -0.36167746782302856,
+            -0.6999327540397644
+          ],
+          [
+            -0.2596096992492676,
+            -0.3332822322845459,
+            -0.4780818223953247,
+            -0.4978921413421631,
+            -0.397924542427063,
+            -0.7602668404579163
+          ],
+          [
+            -0.2622019350528717,
+            -0.3355288505554199,
+            -0.48239684104919434,
+            -0.528130054473877,
+            -0.4257296919822693,
+            -0.8020148873329163
+          ],
+          [
+            -0.2640029191970825,
+            -0.34203797578811646,
+            -0.4851517677307129,
+            -0.5586937665939331,
+            -0.4575371742248535,
+            -0.8226932287216187
+          ],
+          [
+            -0.2646830379962921,
+            -0.3494831323623657,
+            -0.48698997497558594,
+            -0.5976766347885132,
+            -0.4853423237800598,
+            -0.8551071286201477
+          ],
+          [
+            -0.26654842495918274,
+            -0.35311245918273926,
+            -0.48943591117858887,
+            -0.6128090620040894,
+            -0.5088322162628174,
+            -0.8822181224822998
+          ],
+          [
+            -0.2683042287826538,
+            -0.3590872287750244,
+            -0.4910116195678711,
+            -0.6433912515640259,
+            -0.5391519069671631,
+            -0.9078503251075745
+          ]
+        ],
+        "max": [
+          [
+            0.06967931985855103,
+            0.1426796317100525,
+            0.16652920842170715,
+            0.16475245356559753,
+            0.12320073693990707,
+            0.10718557238578796
+          ],
+          [
+            0.13935863971710205,
+            0.285359263420105,
+            0.3330584168434143,
+            0.32950490713119507,
+            0.24640147387981415,
+            0.21437114477157593
+          ],
+          [
+            0.14222566783428192,
+            0.2848570942878723,
+            0.49958762526512146,
+            0.4942573606967926,
+            0.3696022033691406,
+            0.22605326771736145
+          ],
+          [
+            0.1472679078578949,
+            0.28423231840133667,
+            0.6661168336868286,
+            0.6590098142623901,
+            0.3770415782928467,
+            0.23223954439163208
+          ],
+          [
+            0.15682385861873627,
+            0.2835988998413086,
+            0.6651667356491089,
+            0.6587706804275513,
+            0.38236403465270996,
+            0.26481837034225464
+          ],
+          [
+            0.1661832183599472,
+            0.28286415338516235,
+            0.6648765802383423,
+            0.6587662696838379,
+            0.3859591484069824,
+            0.31249791383743286
+          ],
+          [
+            0.17306526005268097,
+            0.2822335362434387,
+            0.6636160612106323,
+            0.6570796370506287,
+            0.38636088371276855,
+            0.3590624928474426
+          ],
+          [
+            0.17878583073616028,
+            0.28816407918930054,
+            0.663140594959259,
+            0.6570292711257935,
+            0.38576996326446533,
+            0.3997495770454407
+          ],
+          [
+            0.17295192182064056,
+            0.29492223262786865,
+            0.6627523899078369,
+            0.6566210985183716,
+            0.38337910175323486,
+            0.43638354539871216
+          ],
+          [
+            0.17799519002437592,
+            0.29758942127227783,
+            0.6605758666992188,
+            0.703202486038208,
+            0.38180720806121826,
+            0.45383220911026
+          ],
+          [
+            0.18265266716480255,
+            0.3032034635543823,
+            0.6587021350860596,
+            0.7628703117370605,
+            0.3764241933822632,
+            0.48166006803512573
+          ],
+          [
+            0.1885882467031479,
+            0.3092483878135681,
+            0.6556060910224915,
+            0.8046655654907227,
+            0.37104523181915283,
+            0.5092863440513611
+          ],
+          [
+            0.19281424582004547,
+            0.3133563995361328,
+            0.6535040736198425,
+            0.82636559009552,
+            0.36730897426605225,
+            0.5377307534217834
+          ],
+          [
+            0.19486497342586517,
+            0.31555628776550293,
+            0.648400604724884,
+            0.8594522476196289,
+            0.3646332025527954,
+            0.5743647217750549
+          ],
+          [
+            0.19893918931484222,
+            0.3193961977958679,
+            0.643994927406311,
+            0.8882213830947876,
+            0.3592824935913086,
+            0.5890528559684753
+          ],
+          [
+            0.2032027691602707,
+            0.3223579525947571,
+            0.6399683356285095,
+            0.9161117076873779,
+            0.3535003662109375,
+            0.6182076334953308
+          ]
+        ],
+        "mean": [
+          [
+            -1.1643587640719488e-05,
+            -2.7824391509057023e-05,
+            -3.667184864752926e-05,
+            -4.7328052460215986e-05,
+            -0.00015884953609202057,
+            -5.547764158109203e-05
+          ],
+          [
+            -2.2667458324576728e-05,
+            -5.402434180723503e-05,
+            -7.09103696863167e-05,
+            -9.270615555578843e-05,
+            -0.00031683416455052793,
+            -0.0001126150309573859
+          ],
+          [
+            -3.3560725569259375e-05,
+            -7.879747136030346e-05,
+            -0.00010333111276850104,
+            -0.00013696543464902788,
+            -0.00047433824511244893,
+            -0.00017093885981012136
+          ],
+          [
+            -4.3838790588779375e-05,
+            -0.0001017165050143376,
+            -0.00013411662075668573,
+            -0.00017976536764763296,
+            -0.0006323863053694367,
+            -0.00023106011212803423
+          ],
+          [
+            -5.300067277858034e-05,
+            -0.0001227815664606169,
+            -0.0001632482890272513,
+            -0.00022041099146008492,
+            -0.0007921468350104988,
+            -0.0002934948424808681
+          ],
+          [
+            -6.065802153898403e-05,
+            -0.0001430162665201351,
+            -0.00019106637046206743,
+            -0.00025859609013423324,
+            -0.0009533620905131102,
+            -0.00035773299168795347
+          ],
+          [
+            -6.741635297657922e-05,
+            -0.00016311084618791938,
+            -0.0002178666618419811,
+            -0.0002950174093712121,
+            -0.0011148689081892371,
+            -0.0004230593622196466
+          ],
+          [
+            -7.310021464945748e-05,
+            -0.0001823703496484086,
+            -0.00024334223417099565,
+            -0.00032892197486944497,
+            -0.001276938826777041,
+            -0.000489219615701586
+          ],
+          [
+            -7.778768485877663e-05,
+            -0.00020139127445872873,
+            -0.00026738905580714345,
+            -0.0003601715143304318,
+            -0.0014380884822458029,
+            -0.0005565285100601614
+          ],
+          [
+            -8.136028191074729e-05,
+            -0.00021956884302198887,
+            -0.00029012488084845245,
+            -0.0003890944935847074,
+            -0.0015986055368557572,
+            -0.0006240413640625775
+          ],
+          [
+            -8.387360867345706e-05,
+            -0.0002369963622186333,
+            -0.00031093033612705767,
+            -0.0004152455076109618,
+            -0.0017575263045728207,
+            -0.0006920687155798078
+          ],
+          [
+            -8.57252671266906e-05,
+            -0.00025390979135409,
+            -0.0003299105155747384,
+            -0.0004392821865621954,
+            -0.001914860913529992,
+            -0.0007613278576172888
+          ],
+          [
+            -8.67545822984539e-05,
+            -0.0002702662895899266,
+            -0.00034615854383446276,
+            -0.00046034055412746966,
+            -0.0020699421875178814,
+            -0.0008314322330988944
+          ],
+          [
+            -8.665885252412409e-05,
+            -0.00028599711367860436,
+            -0.00036020739935338497,
+            -0.0004781086463481188,
+            -0.0022232718765735626,
+            -0.0009027959895320237
+          ],
+          [
+            -8.521131530869752e-05,
+            -0.0003013015375472605,
+            -0.0003717018407769501,
+            -0.0004920642240904272,
+            -0.0023748220410197973,
+            -0.0009756491053849459
+          ],
+          [
+            -8.315555896842852e-05,
+            -0.0003166405949741602,
+            -0.00038077804492786527,
+            -0.0005042090779170394,
+            -0.0025232392363250256,
+            -0.0010490445420145988
+          ]
+        ],
+        "std": [
+          [
+            0.0032290988601744175,
+            0.006170458160340786,
+            0.008908393792808056,
+            0.009209783747792244,
+            0.006690251640975475,
+            0.008351418189704418
+          ],
+          [
+            0.006128471344709396,
+            0.011811385862529278,
+            0.017084389925003052,
+            0.017432956025004387,
+            0.012727884575724602,
+            0.01581396535038948
+          ],
+          [
+            0.00894330907613039,
+            0.017332714051008224,
+            0.02511991560459137,
+            0.02542029321193695,
+            0.01861514337360859,
+            0.023045286536216736
+          ],
+          [
+            0.011718402616679668,
+            0.022783277556300163,
+            0.033052798360586166,
+            0.033295996487140656,
+            0.024394921958446503,
+            0.030171610414981842
+          ],
+          [
+            0.01446112897247076,
+            0.02819165773689747,
+            0.04092729091644287,
+            0.04105884209275246,
+            0.030110569670796394,
+            0.03720058500766754
+          ],
+          [
+            0.01715722307562828,
+            0.033538445830345154,
+            0.04872387275099754,
+            0.048686619848012924,
+            0.03574802726507187,
+            0.04411231726408005
+          ],
+          [
+            0.019811982288956642,
+            0.03882130607962608,
+            0.0564434789121151,
+            0.056187164038419724,
+            0.0413142666220665,
+            0.050911273807287216
+          ],
+          [
+            0.022429028525948524,
+            0.044044703245162964,
+            0.06409874558448792,
+            0.0635669156908989,
+            0.046816907823085785,
+            0.05760900303721428
+          ],
+          [
+            0.025009647011756897,
+            0.04921016842126846,
+            0.07169140130281448,
+            0.07082932442426682,
+            0.052258118987083435,
+            0.0642051249742508
+          ],
+          [
+            0.02755027636885643,
+            0.054309576749801636,
+            0.07921512424945831,
+            0.0779670849442482,
+            0.05763443559408188,
+            0.07069504261016846
+          ],
+          [
+            0.030053986236453056,
+            0.05933916196227074,
+            0.0866628885269165,
+            0.08498358726501465,
+            0.06294465065002441,
+            0.07707913964986801
+          ],
+          [
+            0.032529864460229874,
+            0.06430266052484512,
+            0.09404037147760391,
+            0.0918952077627182,
+            0.06819884479045868,
+            0.0833641067147255
+          ],
+          [
+            0.0349782295525074,
+            0.06919905543327332,
+            0.10134579986333847,
+            0.09870505332946777,
+            0.07339312136173248,
+            0.08955203741788864
+          ],
+          [
+            0.037398893386125565,
+            0.07402431964874268,
+            0.10857424139976501,
+            0.10541565716266632,
+            0.07852877676486969,
+            0.09564216434955597
+          ],
+          [
+            0.039792418479919434,
+            0.078775554895401,
+            0.11572249233722687,
+            0.11202510446310043,
+            0.08360263705253601,
+            0.10163461416959763
+          ],
+          [
+            0.042157746851444244,
+            0.08345219492912292,
+            0.12279094755649567,
+            0.11853570491075516,
+            0.08861752599477768,
+            0.10753004997968674
+          ]
+        ],
+        "q01": [
+          [
+            -0.009268714543431998,
+            -0.011592475175857543,
+            -0.01831440806388855,
+            -0.024825823903083803,
+            -0.01634468197822571,
+            -0.03199689894914627
+          ],
+          [
+            -0.017236289083957673,
+            -0.0215373170375824,
+            -0.03404565572738647,
+            -0.046274811029434204,
+            -0.030144470930099487,
+            -0.059781843423843385
+          ],
+          [
+            -0.025366789400577544,
+            -0.03145905256271362,
+            -0.0496677827835083,
+            -0.06639246940612793,
+            -0.0438637912273407,
+            -0.09413704216480255
+          ],
+          [
+            -0.03328576520085335,
+            -0.041240596771240236,
+            -0.06569475293159485,
+            -0.08771294116973877,
+            -0.05808493912220001,
+            -0.12412740468978882
+          ],
+          [
+            -0.041257518157362935,
+            -0.051141791343688965,
+            -0.08181274771690368,
+            -0.10713644534349442,
+            -0.07209896445274352,
+            -0.15517061889171602
+          ],
+          [
+            -0.049353072270751,
+            -0.06130526304244995,
+            -0.09762808799743652,
+            -0.12659394681453706,
+            -0.08611830949783325,
+            -0.18616031646728515
+          ],
+          [
+            -0.05744967430830002,
+            -0.0712296211719513,
+            -0.11313812494277954,
+            -0.14622240781784057,
+            -0.09960022687911987,
+            -0.214241119325161
+          ],
+          [
+            -0.06542415797710419,
+            -0.08120648860931397,
+            -0.1290024411678314,
+            -0.1657324779033661,
+            -0.11374742388725281,
+            -0.24043977558612822
+          ],
+          [
+            -0.07297706604003906,
+            -0.09091942310333252,
+            -0.14475531101226807,
+            -0.18410858631134033,
+            -0.1277242559194565,
+            -0.26655630469322206
+          ],
+          [
+            -0.08005138725042343,
+            -0.10064849734306336,
+            -0.1605457830429077,
+            -0.2011710387468338,
+            -0.1411297345161438,
+            -0.29264324426651
+          ],
+          [
+            -0.0873410564661026,
+            -0.10989363551139832,
+            -0.1759151017665863,
+            -0.21879440784454346,
+            -0.1543375015258789,
+            -0.3165887701511383
+          ],
+          [
+            -0.09439642295241356,
+            -0.11963839292526245,
+            -0.19191760659217835,
+            -0.23589837551116943,
+            -0.1671001899242401,
+            -0.33964144945144653
+          ],
+          [
+            -0.10113533169031143,
+            -0.12853635907173155,
+            -0.2071603000164032,
+            -0.25196747422218324,
+            -0.18049648761749268,
+            -0.36394234538078307
+          ],
+          [
+            -0.10861974209547043,
+            -0.13719136953353883,
+            -0.22217320442199706,
+            -0.2697417187690735,
+            -0.193385089635849,
+            -0.38564474523067477
+          ],
+          [
+            -0.1153394678235054,
+            -0.14563777565956115,
+            -0.23786645889282226,
+            -0.2853887552022934,
+            -0.20657859563827516,
+            -0.4076483082771301
+          ],
+          [
+            -0.12171894907951356,
+            -0.15430085062980653,
+            -0.25233622670173644,
+            -0.3027268445491791,
+            -0.2191355788707733,
+            -0.42746653854846955
+          ]
+        ],
+        "q99": [
+          [
+            0.008507148921489717,
+            0.016275683641433718,
+            0.023879630565643312,
+            0.0332252609729767,
+            0.018261760473251343,
+            0.02287991702556612
+          ],
+          [
+            0.01590641420334578,
+            0.029940589666366584,
+            0.04449433565139771,
+            0.061001383066177375,
+            0.03363417983055115,
+            0.04352214038372041
+          ],
+          [
+            0.02317148551344872,
+            0.04354479908943177,
+            0.064463427066803,
+            0.09550023198127755,
+            0.04913925051689148,
+            0.06347175836563111
+          ],
+          [
+            0.030544601306319245,
+            0.057246526479721074,
+            0.08530706048011781,
+            0.12828858852386482,
+            0.06439893484115602,
+            0.0836253125965597
+          ],
+          [
+            0.03799312710762029,
+            0.0709479707479477,
+            0.10594189643859864,
+            0.1582103323936463,
+            0.07981338620185854,
+            0.1029581561684609
+          ],
+          [
+            0.04527667090296748,
+            0.08455618143081665,
+            0.12661605834960937,
+            0.19113413929939288,
+            0.09519806623458872,
+            0.12280465364456178
+          ],
+          [
+            0.052215367555618335,
+            0.09793634176254273,
+            0.14665939569473269,
+            0.21992011517286303,
+            0.11064429104328156,
+            0.14226103723049172
+          ],
+          [
+            0.05975700989365578,
+            0.1112301278114319,
+            0.16732881784439088,
+            0.24769704818725613,
+            0.12489534020423892,
+            0.16099104881286624
+          ],
+          [
+            0.06689389139413833,
+            0.12377568721771244,
+            0.18655168294906618,
+            0.2754820889234549,
+            0.1396961754560471,
+            0.1789368557929993
+          ],
+          [
+            0.07382299691438678,
+            0.13681530475616463,
+            0.20641826033592234,
+            0.3025679612159731,
+            0.15372511744499212,
+            0.19631955385208133
+          ],
+          [
+            0.08074128389358522,
+            0.14930570840835578,
+            0.22576861619949343,
+            0.3308370995521546,
+            0.1672333872318268,
+            0.21368486702442177
+          ],
+          [
+            0.08785801172256473,
+            0.16144273519515998,
+            0.24573646068573007,
+            0.3557658374309541,
+            0.1806690895557404,
+            0.22995711117982875
+          ],
+          [
+            0.09447186753153812,
+            0.1733730149269106,
+            0.2646211528778077,
+            0.3822640872001649,
+            0.19430121064186098,
+            0.24733826279640203
+          ],
+          [
+            0.10127870410680771,
+            0.1851757299900055,
+            0.2839685177803042,
+            0.4072193753719333,
+            0.20723481833934784,
+            0.2634331870079042
+          ],
+          [
+            0.10798155874013902,
+            0.19643956899642945,
+            0.30209650993347176,
+            0.4327797448635112,
+            0.21953258037567142,
+            0.2806586390733719
+          ],
+          [
+            0.11426104053854945,
+            0.20780228734016423,
+            0.3209846079349518,
+            0.45687500715255747,
+            0.23287598758935932,
+            0.2948111867904664
+          ]
+        ]
+      }
+    }
+  }
+}

experiment_cfg/final_model_config.json

@@ -0,0 +1,55 @@
+{
+  "model_type": "Gr00tN1d7",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Cosmos-Reason2-2B",
+  "backbone_model_type": "qwen",
+  "model_revision": null,
+  "tune_top_llm_layers": 0,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": false,
+  "backbone_trainable_params_fp32": true,
+  "extra_augmentation_config": null,
+  "apply_sincos_state_encoding": false,
+  "use_percentiles": true,
+  "use_relative_action": false,
+  "max_state_dim": 132,
+  "max_action_dim": 132,
+  "action_horizon": 40,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "state_history_length": 1,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.2,
+  "exclude_state": false,
+  "use_mean_std": false,
+  "max_num_embodiments": 32
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbcaea5ee88f1e0f1465043920a2647c67e7de17d24adfd1c477742a6168edec
+size 4986649584

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f165508f7007acc37292eb4a8d6de9d544b2bca2c27204a0d9c2f35ac8e8c9c
+size 4970792616

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43229c459b75a3e6e7ad2d59ab09a2dbc15a7601a013902c6f12c97dec73e6c2
+size 2618758696

processor_config.json

@@ -0,0 +1,1123 @@
+{
+  "processor_class": "Gr00tN1d7Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "real_g1_relative_eef_relative_joints": {
+        "video": {
+          "delta_indices": [
+            -20,
+            0
+          ],
+          "modality_keys": [
+            "ego_view"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef_9d",
+            "right_wrist_eef_9d",
+            "left_hand",
+            "right_hand",
+            "left_arm",
+            "right_arm",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef_9d",
+            "right_wrist_eef_9d",
+            "left_hand",
+            "right_hand",
+            "left_arm",
+            "right_arm",
+            "waist",
+            "base_height_command",
+            "navigate_command"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef_9d"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef_9d"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_arm"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_arm"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "waist"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "base_height_command"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "navigate_command"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "real_r1_pro_sharpa_relative_eef_mecka": {
+        "video": {
+          "delta_indices": [
+            -30,
+            0
+          ],
+          "modality_keys": [
+            "ego_view_cropratio_res320x240_freq30"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand_joints"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand_joints"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "real_r1_pro_sharpa_relative_eef_human": {
+        "video": {
+          "delta_indices": [
+            -20,
+            0
+          ],
+          "modality_keys": [
+            "ego_view_res320x240_freq20",
+            "left_wrist_view_res320x240_freq20",
+            "right_wrist_view_res320x240_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand_joints"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand_joints"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "real_r1_pro_sharpa_relative_eef": {
+        "video": {
+          "delta_indices": [
+            -20,
+            0
+          ],
+          "modality_keys": [
+            "ego_view_res320x240_freq20",
+            "left_wrist_view_res320x240_freq20",
+            "right_wrist_view_res320x240_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand_joints"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand_joints"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "xdof_relative_eef_relative_joint": {
+        "video": {
+          "delta_indices": [
+            -30,
+            0
+          ],
+          "modality_keys": [
+            "top_camera-images-rgb_320_240",
+            "left_camera-images-rgb_320_240",
+            "right_camera-images-rgb_320_240"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_gripper_pos",
+            "right_gripper_pos",
+            "left_joint_pos",
+            "right_joint_pos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_gripper_pos",
+            "right_gripper_pos",
+            "left_joint_pos",
+            "right_joint_pos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_gripper_pos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_gripper_pos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_joint_pos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_joint_pos"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "real_r1_pro_sharpa_relative_eef_maxinsights": {
+        "video": {
+          "delta_indices": [
+            -30,
+            0
+          ],
+          "modality_keys": [
+            "ego_view_cropratio_res320x240_freq30"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_hand_joints",
+            "right_hand_joints"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_hand_joints"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_hand_joints"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "xdof_relative_eef_relative_joint_subtask": {
+        "video": {
+          "delta_indices": [
+            -30,
+            0
+          ],
+          "modality_keys": [
+            "top_camera-images-rgb_320_240",
+            "left_camera-images-rgb_320_240",
+            "right_camera-images-rgb_320_240"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_gripper_pos",
+            "right_gripper_pos",
+            "left_joint_pos",
+            "right_joint_pos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "left_wrist_eef",
+            "right_wrist_eef",
+            "left_gripper_pos",
+            "right_gripper_pos",
+            "left_joint_pos",
+            "right_joint_pos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "left_wrist_eef"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "right_wrist_eef"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_gripper_pos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_gripper_pos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "left_joint_pos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "right_joint_pos"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.sub_task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "oxe_droid_relative_eef_relative_joint": {
+        "video": {
+          "delta_indices": [
+            -15,
+            0
+          ],
+          "modality_keys": [
+            "exterior_image_1_left",
+            "wrist_image_left"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "eef_9d",
+            "gripper_position",
+            "joint_position"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31,
+            32,
+            33,
+            34,
+            35,
+            36,
+            37,
+            38,
+            39
+          ],
+          "modality_keys": [
+            "eef_9d",
+            "gripper_position",
+            "joint_position"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "EEF",
+              "format": "XYZ_ROT6D",
+              "state_key": "eef_9d"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "gripper_position"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "joint_position"
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.language.language_instruction"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "base",
+            "wrist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": [
+      230,
+      230
+    ],
+    "image_target_size": [
+      256,
+      256
+    ],
+    "use_albumentations": true,
+    "random_rotation_angle": 0,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "letter_box_transform": false,
+    "model_name": "nvidia/Cosmos-Reason2-2B",
+    "model_type": "qwen",
+    "formalize_language": true,
+    "max_state_dim": 132,
+    "max_action_dim": 132,
+    "max_action_horizon": 40,
+    "use_percentiles": true,
+    "use_mean_std": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": false,
+    "use_relative_action": true,
+    "exclude_state": false,
+    "state_dropout_prob": 0.2
+  }
+}

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5c85d1a0cb57208d41ef75970d43a2efd7413f44810613ce6bde95b02b91c46
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fa802a80def971b73ec74284a6aa44d0b2ea101bd38ed41a3b1c1a0b4001f00
+size 1465

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be94bee33928ba678214ef179fc7ea8106ef88653513034abc49a43d64045187
+size 5841

wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d7", "run_id": "uf850_ckpt"}