Upload folder using huggingface_hub

checkpoint-1000/config.json

@@ -0,0 +1,70 @@
+{
+  "action_horizon": 50,
+  "add_pos_embed": true,
+  "apply_sincos_state_encoding": true,
+  "architectures": [
+    "Gr00tN1d6"
+  ],
+  "attn_dropout": 0.2,
+  "attn_implementation": null,
+  "backbone_embedding_dim": 2048,
+  "backbone_model_type": "eagle",
+  "backbone_trainable_params_fp32": true,
+  "collator_overwrite_image_inputs": false,
+  "color_jitter_params": {
+    "brightness": 0.1,
+    "contrast": 0.1,
+    "hue": 0.1,
+    "saturation": 0.1
+  },
+  "crop_fraction": 0.95,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "eagle_collator": true,
+  "formalize_language": true,
+  "gemma_collator": false,
+  "hidden_size": 1024,
+  "image_crop_size": null,
+  "image_target_size": null,
+  "input_embedding_dim": 1536,
+  "load_bf16": true,
+  "max_action_dim": 128,
+  "max_num_embodiments": 32,
+  "max_seq_len": 1024,
+  "max_state_dim": 128,
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "model_type": "Gr00tN1d6",
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_inference_timesteps": 4,
+  "num_timestep_buckets": 1000,
+  "random_rotation_angle": null,
+  "reproject_vision": false,
+  "select_layer": 16,
+  "shortest_image_edge": 256,
+  "state_dropout_prob": 0.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "tune_diffusion_model": true,
+  "tune_llm": false,
+  "tune_projector": true,
+  "tune_top_llm_layers": 4,
+  "tune_visual": false,
+  "tune_vlln": true,
+  "use_albumentations_transforms": true,
+  "use_alternate_vl_dit": true,
+  "use_flash_attention": true,
+  "use_relative_action": true,
+  "use_vlln": true
+}

checkpoint-1000/embodiment_id.json

@@ -0,0 +1,10 @@
+{
+  "robocasa_panda_omron": 13,
+  "gr1": 20,
+  "behavior_r1_pro": 24,
+  "unitree_g1": 8,
+  "oxe_google": 0,
+  "oxe_widowx": 1,
+  "libero_panda": 2,
+  "new_embodiment": 10
+}

checkpoint-1000/experiment_cfg/conf.yaml

@@ -0,0 +1,209 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d6
+  model_dtype: bfloat16
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  backbone_model_type: eagle
+  model_revision: null
+  tune_top_llm_layers: 4
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 16
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  collator_overwrite_image_inputs: false
+  eagle_collator: true
+  backbone_trainable_params_fp32: true
+  image_crop_size: null
+  image_target_size: null
+  shortest_image_edge: 256
+  crop_fraction: 0.95
+  random_rotation_angle: null
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    saturation: 0.5
+    hue: 0.08
+  use_albumentations_transforms: true
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_relative_action: true
+  max_state_dim: 29
+  max_action_dim: 29
+  action_horizon: 16
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 32
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.0
+  state_additive_noise_scale: 0.0
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - /content/dataset/azazdeaz/record-test/azazdeaz/record-test/
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: RELATIVE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: fork
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: torchcodec
+training:
+  output_dir: /content/so100_finetune
+  experiment_name: null
+  max_steps: 10000
+  global_batch_size: 36
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 1000
+  save_total_limit: 5
+  save_vl_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 4
+  remove_unused_columns: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 10000
+save_steps: 1000

checkpoint-1000/experiment_cfg/config.yaml

@@ -0,0 +1,243 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - /content/dataset/azazdeaz/record-test/azazdeaz/record-test/
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - relative
+          state_key: null
+          type: &id002 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: *id002
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: fork
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  random_chop: 0.0
+  seed: 42
+  shard_size: 1024
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: torchcodec
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
+  _attn_implementation_autoset: false
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  add_cross_attention: false
+  architectures: null
+  backbone_model_type: eagle
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    hue: 0.08
+    saturation: 0.5
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 32
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  eagle_collator: true
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_attentions: false
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.0
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torch_dtype: null
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 4
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 36
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 10000
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: /content/so100_finetune
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_steps: 1000
+  save_total_limit: 5
+  save_vl_model: false
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

checkpoint-1000/experiment_cfg/dataset_statistics.json

@@ -0,0 +1,824 @@
+{
+  "new_embodiment": {
+    "state": {
+      "single_arm": {
+        "min": [
+          -60.105262756347656,
+          -99.31827545166016,
+          -43.24324417114258,
+          44.688323974609375,
+          -80.68241119384766
+        ],
+        "max": [
+          34.31578826904297,
+          45.80315399169922,
+          99.62721252441406,
+          97.62950134277344,
+          -4.8293962478637695
+        ],
+        "mean": [
+          -3.5325422842834118,
+          -39.55833784368612,
+          47.61425589641663,
+          68.3154743394685,
+          -46.70357059936142
+        ],
+        "std": [
+          27.459571708144434,
+          41.074676874488844,
+          34.075122244859095,
+          13.351843152336091,
+          19.483248111140814
+        ],
+        "q01": [
+          -43.89320641903556,
+          -99.31827545166016,
+          2.96704117550838,
+          52.128458407208086,
+          -64.9047852897644
+        ],
+        "q99": [
+          23.91888349814585,
+          10.271718478515513,
+          99.48718710059721,
+          91.53274853632244,
+          -26.5701696578577
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.9803921580314636
+        ],
+        "max": [
+          35.9943962097168
+        ],
+        "mean": [
+          20.298597213926165
+        ],
+        "std": [
+          9.780378321306017
+        ],
+        "q01": [
+          6.1409783557640525
+        ],
+        "q99": [
+          29.83243577963609
+        ]
+      }
+    },
+    "action": {
+      "single_arm": {
+        "min": [
+          -60.843040466308594,
+          -100.0,
+          -47.45454406738281,
+          44.675209045410156,
+          -81.04986572265625
+        ],
+        "max": [
+          34.99722671508789,
+          45.694801330566406,
+          100.0,
+          97.96730041503906,
+          -4.566929340362549
+        ],
+        "mean": [
+          -3.4051861822555503,
+          -40.21667093678776,
+          46.219495035627304,
+          68.11321517887163,
+          -46.72253879798044
+        ],
+        "std": [
+          27.604988373472494,
+          40.826917813818575,
+          34.82453909741334,
+          13.439408862929378,
+          19.518525610689558
+        ],
+        "q01": [
+          -44.29914832538184,
+          -99.9431293020161,
+          -0.6294070457600729,
+          51.77255040025036,
+          -65.3110059408939
+        ],
+        "q99": [
+          24.517798066878782,
+          10.287138016986633,
+          99.9985524069893,
+          91.79085298874445,
+          -26.382156973884847
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.25553661584854126
+        ],
+        "max": [
+          36.28620147705078
+        ],
+        "mean": [
+          19.60203928355869
+        ],
+        "std": [
+          10.896771214917207
+        ],
+        "q01": [
+          2.708348265017052
+        ],
+        "q99": [
+          30.222431877944597
+        ]
+      }
+    },
+    "relative_action": {
+      "single_arm": {
+        "min": [
+          [
+            -9.055549621582031,
+            -13.95650863647461,
+            -14.800384521484375,
+            -13.61447525024414,
+            -7.874013900756836
+          ],
+          [
+            -11.160816192626953,
+            -16.68364715576172,
+            -17.25493621826172,
+            -16.177448272705078,
+            -9.501310348510742
+          ],
+          [
+            -13.476604461669922,
+            -19.667449951171875,
+            -19.89129638671875,
+            -18.372356414794922,
+            -11.23359489440918
+          ],
+          [
+            -15.26607894897461,
+            -22.73650360107422,
+            -22.314407348632812,
+            -20.567264556884766,
+            -12.755905151367188
+          ],
+          [
+            -16.950286865234375,
+            -25.550708770751953,
+            -24.457931518554688,
+            -22.246448516845703,
+            -14.120733261108398
+          ],
+          [
+            -18.84502410888672,
+            -28.619762420654297,
+            -26.764127731323242,
+            -23.572124481201172,
+            -15.538057327270508
+          ],
+          [
+            -20.50891876220703,
+            -31.262561798095703,
+            -29.18724250793457,
+            -24.625682830810547,
+            -16.79789924621582
+          ],
+          [
+            -21.950958251953125,
+            -33.90535354614258,
+            -31.703550338745117,
+            -25.421085357666016,
+            -17.9002628326416
+          ],
+          [
+            -23.351024627685547,
+            -36.545902252197266,
+            -33.976280212402344,
+            -25.947864532470703,
+            -18.89763641357422
+          ],
+          [
+            -24.79306411743164,
+            -39.18869400024414,
+            -36.27018737792969,
+            -26.389755249023438,
+            -19.737533569335938
+          ],
+          [
+            -26.056222915649414,
+            -41.40523910522461,
+            -38.54291534423828,
+            -26.566513061523438,
+            -20.57742691040039
+          ],
+          [
+            -27.2141170501709,
+            -43.45127487182617,
+            -40.633819580078125,
+            -26.654312133789062,
+            -21.732280731201172
+          ],
+          [
+            -28.372011184692383,
+            -45.497310638427734,
+            -42.90654754638672,
+            -26.654312133789062,
+            -22.834644317626953
+          ],
+          [
+            -29.5299072265625,
+            -47.542476654052734,
+            -44.9827995300293,
+            -26.654312133789062,
+            -23.674541473388672
+          ],
+          [
+            -30.687801361083984,
+            -49.50242614746094,
+            -47.19833755493164,
+            -26.654312133789062,
+            -24.304462432861328
+          ],
+          [
+            -31.84569549560547,
+            -51.29353713989258,
+            -49.528255462646484,
+            -26.654312133789062,
+            -25.406824111938477
+          ]
+        ],
+        "max": [
+          [
+            11.800273895263672,
+            13.390182495117188,
+            11.2484130859375,
+            12.953540802001953,
+            8.97637939453125
+          ],
+          [
+            14.462503433227539,
+            16.288734436035156,
+            13.857921600341797,
+            15.763023376464844,
+            10.813648223876953
+          ],
+          [
+            16.902881622314453,
+            19.443038940429688,
+            16.2484130859375,
+            18.39691162109375,
+            12.703411102294922
+          ],
+          [
+            19.40987205505371,
+            22.16992950439453,
+            18.857921600341797,
+            20.783126831054688,
+            14.645668029785156
+          ],
+          [
+            21.850250244140625,
+            24.812728881835938,
+            21.221553802490234,
+            22.815826416015625,
+            16.482940673828125
+          ],
+          [
+            24.290626525878906,
+            27.455524444580078,
+            23.551467895507812,
+            24.671768188476562,
+            18.530181884765625
+          ],
+          [
+            26.50115203857422,
+            30.011974334716797,
+            25.857921600341797,
+            26.515491485595703,
+            20.262466430664062
+          ],
+          [
+            28.71167755126953,
+            32.31278610229492,
+            28.187835693359375,
+            27.920230865478516,
+            22.099735260009766
+          ],
+          [
+            31.132732391357422,
+            34.52932357788086,
+            30.585193634033203,
+            29.34360122680664,
+            23.832019805908203
+          ],
+          [
+            33.24032974243164,
+            36.490108489990234,
+            32.91510772705078,
+            30.757648468017578,
+            25.56430435180664
+          ],
+          [
+            35.237003326416016,
+            38.621395111083984,
+            35.187835693359375,
+            32.26008224487305,
+            27.296588897705078
+          ],
+          [
+            37.0264778137207,
+            40.411678314208984,
+            37.27873992919922,
+            33.66482162475586,
+            28.87139129638672
+          ],
+          [
+            38.815948486328125,
+            42.372459411621094,
+            39.32906723022461,
+            34.99049758911133,
+            30.498687744140625
+          ],
+          [
+            40.147064208984375,
+            43.99224090576172,
+            41.238162994384766,
+            35.6975212097168,
+            31.9160099029541
+          ],
+          [
+            41.552791595458984,
+            45.61201858520508,
+            43.13301467895508,
+            36.0510368347168,
+            33.33333206176758
+          ],
+          [
+            43.0264778137207,
+            47.23110580444336,
+            45.22392654418945,
+            36.402225494384766,
+            34.75065612792969
+          ]
+        ],
+        "mean": [
+          [
+            0.11750347912311554,
+            -0.6653266549110413,
+            -1.4583468437194824,
+            -0.2053278237581253,
+            -0.01863335259258747
+          ],
+          [
+            0.12236570566892624,
+            -0.6657301187515259,
+            -1.4583476781845093,
+            -0.20791052281856537,
+            -0.02185939997434616
+          ],
+          [
+            0.12741880118846893,
+            -0.6660965085029602,
+            -1.4583467245101929,
+            -0.21049359440803528,
+            -0.025085171684622765
+          ],
+          [
+            0.13261470198631287,
+            -0.6664630174636841,
+            -1.4583468437194824,
+            -0.21307627856731415,
+            -0.02831093594431877
+          ],
+          [
+            0.13785837590694427,
+            -0.6668296456336975,
+            -1.4583486318588257,
+            -0.21565881371498108,
+            -0.03153714910149574
+          ],
+          [
+            0.14319713413715363,
+            -0.6671591401100159,
+            -1.4583512544631958,
+            -0.21824198961257935,
+            -0.03474012389779091
+          ],
+          [
+            0.14853604137897491,
+            -0.6674894094467163,
+            -1.458350419998169,
+            -0.22082529962062836,
+            -0.03794342279434204
+          ],
+          [
+            0.15382729470729828,
+            -0.6678190231323242,
+            -1.4583524465560913,
+            -0.22340813279151917,
+            -0.041146621108055115
+          ],
+          [
+            0.15907101333141327,
+            -0.6681490540504456,
+            -1.458353042602539,
+            -0.2259913831949234,
+            -0.04434990882873535
+          ],
+          [
+            0.16426703333854675,
+            -0.668441653251648,
+            -1.458351492881775,
+            -0.22857406735420227,
+            -0.04755344241857529
+          ],
+          [
+            0.1694149672985077,
+            -0.668734610080719,
+            -1.4583535194396973,
+            -0.23115694522857666,
+            -0.05075618624687195
+          ],
+          [
+            0.17465919256210327,
+            -0.6690278649330139,
+            -1.458351731300354,
+            -0.2337394654750824,
+            -0.05395958200097084
+          ],
+          [
+            0.17990276217460632,
+            -0.6693212985992432,
+            -1.4583524465560913,
+            -0.23632188141345978,
+            -0.05716199427843094
+          ],
+          [
+            0.18505056202411652,
+            -0.6696141958236694,
+            -1.4583513736724854,
+            -0.2389044612646103,
+            -0.060365449637174606
+          ],
+          [
+            0.1901988983154297,
+            -0.6699072122573853,
+            -1.4583531618118286,
+            -0.24148696660995483,
+            -0.06356889009475708
+          ],
+          [
+            0.19529956579208374,
+            -0.6702005863189697,
+            -1.4583516120910645,
+            -0.24407006800174713,
+            -0.0667722150683403
+          ]
+        ],
+        "std": [
+          [
+            2.756209135055542,
+            4.048781394958496,
+            4.834221839904785,
+            2.4045844078063965,
+            1.831181526184082
+          ],
+          [
+            3.375519037246704,
+            4.940067768096924,
+            5.704192638397217,
+            2.936358690261841,
+            2.2514567375183105
+          ],
+          [
+            3.9847497940063477,
+            5.8236775398254395,
+            6.570866107940674,
+            3.443351984024048,
+            2.663818359375
+          ],
+          [
+            4.582452774047852,
+            6.696892738342285,
+            7.430098056793213,
+            3.9259438514709473,
+            3.067777633666992
+          ],
+          [
+            5.167651653289795,
+            7.5579352378845215,
+            8.279448509216309,
+            4.385776996612549,
+            3.462792158126831
+          ],
+          [
+            5.740141868591309,
+            8.406184196472168,
+            9.11788558959961,
+            4.825246810913086,
+            3.848816156387329
+          ],
+          [
+            6.2996673583984375,
+            9.240605354309082,
+            9.944042205810547,
+            5.246663570404053,
+            4.225630283355713
+          ],
+          [
+            6.846652507781982,
+            10.06134033203125,
+            10.75770092010498,
+            5.652442932128906,
+            4.593677043914795
+          ],
+          [
+            7.3811469078063965,
+            10.86816120147705,
+            11.558343887329102,
+            6.044579982757568,
+            4.9531779289245605
+          ],
+          [
+            7.903446197509766,
+            11.661355972290039,
+            12.34595012664795,
+            6.424712657928467,
+            5.304342269897461
+          ],
+          [
+            8.414060592651367,
+            12.44128131866455,
+            13.120593070983887,
+            6.794016361236572,
+            5.647256851196289
+          ],
+          [
+            8.913477897644043,
+            13.208165168762207,
+            13.88218879699707,
+            7.153115272521973,
+            5.98213529586792
+          ],
+          [
+            9.402256965637207,
+            13.962782859802246,
+            14.631062507629395,
+            7.5023603439331055,
+            6.309183597564697
+          ],
+          [
+            9.880831718444824,
+            14.70537281036377,
+            15.367119789123535,
+            7.841666221618652,
+            6.628418445587158
+          ],
+          [
+            10.349787712097168,
+            15.43662166595459,
+            16.090578079223633,
+            8.171104431152344,
+            6.940072536468506
+          ],
+          [
+            10.809670448303223,
+            16.15700340270996,
+            16.801616668701172,
+            8.490680694580078,
+            7.244339466094971
+          ]
+        ],
+        "q01": [
+          [
+            -7.149912223815918,
+            -12.172602081298828,
+            -13.191639633178712,
+            -7.083407516479492,
+            -5.301837921142578
+          ],
+          [
+            -8.969949054718018,
+            -14.76386520385742,
+            -15.673493194580079,
+            -8.597449951171875,
+            -6.4955373382568355
+          ],
+          [
+            -10.634601535797119,
+            -17.415445404052733,
+            -18.04468208312988,
+            -9.786780090332032,
+            -7.755381164550781
+          ],
+          [
+            -12.332097396850585,
+            -19.81734550476074,
+            -20.49539176940918,
+            -10.772034606933593,
+            -8.81890007019043
+          ],
+          [
+            -14.054951286315918,
+            -22.659150772094726,
+            -22.971768417358398,
+            -11.810748291015624,
+            -10.076640815734862
+          ],
+          [
+            -15.467545299530029,
+            -24.854450073242187,
+            -25.42631248474121,
+            -12.89973159790039,
+            -11.391075839996338
+          ],
+          [
+            -16.85467258453369,
+            -27.482084159851073,
+            -27.82916431427002,
+            -13.832246704101562,
+            -12.532282905578613
+          ],
+          [
+            -18.258717765808104,
+            -29.645261459350586,
+            -29.891822814941406,
+            -14.627758026123047,
+            -13.543306350708008
+          ],
+          [
+            -19.36220642089844,
+            -31.862525711059572,
+            -32.179530944824215,
+            -15.569841918945311,
+            -14.78740104675293
+          ],
+          [
+            -20.866409225463865,
+            -34.33618782043457,
+            -34.25658073425293,
+            -16.842924346923827,
+            -15.576902465820313
+          ],
+          [
+            -21.95573440551758,
+            -36.528687438964845,
+            -36.38519187927246,
+            -17.423948669433592,
+            -16.82309829711914
+          ],
+          [
+            -23.461913070678712,
+            -38.73653938293457,
+            -38.30317817687988,
+            -17.647796630859375,
+            -17.742782592773438
+          ],
+          [
+            -24.53435989379883,
+            -40.874423828124996,
+            -40.44069320678711,
+            -18.515239715576172,
+            -18.879788017272947
+          ],
+          [
+            -25.933171768188476,
+            -43.23724739074707,
+            -42.29842636108398,
+            -19.954179382324217,
+            -19.801573486328124
+          ],
+          [
+            -27.245162963867188,
+            -45.308877029418944,
+            -43.92444366455078,
+            -20.61133575439453,
+            -20.9133874130249
+          ],
+          [
+            -28.540471458435057,
+            -47.20507698059082,
+            -45.95519866943359,
+            -20.804307861328123,
+            -21.69133777618408
+          ]
+        ],
+        "q99": [
+          [
+            8.69469722747799,
+            10.299373474121074,
+            9.785606613159175,
+            9.179881286621086,
+            6.271916503906227
+          ],
+          [
+            10.615129890441846,
+            12.666508712768518,
+            12.091986007690421,
+            11.210918426513642,
+            7.833070821762051
+          ],
+          [
+            12.4686942863464,
+            14.899181365966779,
+            14.446693725585934,
+            13.182954406738268,
+            9.380576934814396
+          ],
+          [
+            14.271173782348615,
+            17.33157432556147,
+            16.80153594970703,
+            15.230424041748023,
+            10.432546234130848
+          ],
+          [
+            16.08265884399413,
+            19.270514526367187,
+            19.092461013793933,
+            17.081269989013663,
+            12.112335987091052
+          ],
+          [
+            18.145391998291,
+            21.44050186157223,
+            21.280475997924803,
+            18.68309165954589,
+            13.620999450683572
+          ],
+          [
+            19.996333007812442,
+            23.52975692749016,
+            23.596975326538082,
+            20.103778152465818,
+            14.602626724243084
+          ],
+          [
+            21.60820228576658,
+            25.1919183349609,
+            25.76568862915039,
+            21.541843643188393,
+            16.10183769226073
+          ],
+          [
+            23.362480049133293,
+            27.08537643432608,
+            28.15810386657714,
+            22.8827821350097,
+            17.433070678710767
+          ],
+          [
+            25.225577430725092,
+            28.52370849609371,
+            30.372146911621076,
+            23.82381240844726,
+            18.712861175537086
+          ],
+          [
+            26.625544967651358,
+            30.16027656555174,
+            32.49975234985349,
+            24.50854393005371,
+            19.443568878173714
+          ],
+          [
+            28.116129837036127,
+            31.295492095947264,
+            34.53191467285156,
+            25.490360641479374,
+            20.561679687499943
+          ],
+          [
+            29.457272415161103,
+            32.63257324218748,
+            36.42262039184569,
+            26.227766571044913,
+            21.088712310791003
+          ],
+          [
+            30.8263483428955,
+            33.570856246948225,
+            38.26254928588863,
+            26.711691284179675,
+            22.396849746704
+          ],
+          [
+            32.008242797851516,
+            34.67565246582029,
+            40.05519142150879,
+            27.399809188842735,
+            22.793699951171842
+          ],
+          [
+            33.28292778015133,
+            35.806464462280275,
+            41.85474884033201,
+            27.812029418945265,
+            23.47191642761218
+          ]
+        ]
+      }
+    }
+  }
+}

checkpoint-1000/experiment_cfg/final_model_config.json

@@ -0,0 +1,53 @@
+{
+  "model_type": "Gr00tN1d6",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "backbone_model_type": "eagle",
+  "model_revision": null,
+  "tune_top_llm_layers": 4,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": true,
+  "collator_overwrite_image_inputs": false,
+  "eagle_collator": true,
+  "backbone_trainable_params_fp32": true,
+  "apply_sincos_state_encoding": true,
+  "use_relative_action": true,
+  "max_state_dim": 128,
+  "max_action_dim": 128,
+  "action_horizon": 50,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.0,
+  "state_additive_noise_scale": 0.0,
+  "max_num_embodiments": 32
+}

checkpoint-1000/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dc2fbc659733405270834529d9192d68993cb85b7997499c872ce83696ea0d3
+size 4990120184

checkpoint-1000/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d23cc8c9d0cca6f623ac055e00eaeff1b050c2af1d5366cf478fd2e2c411bdf
+size 4823190320

checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d37f434807d08f03597bf76c2277fc55bcc5109261b04921191f1cc8c0311ed
+size 12960193762

checkpoint-1000/processor_config.json

@@ -0,0 +1,455 @@
+{
+  "processor_class": "Gr00tN1d6Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "behavior_r1_pro": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "observation.images.rgb.head_256_256",
+            "observation.images.rgb.left_wrist_256_256",
+            "observation.images.rgb.right_wrist_256_256"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "robot_pos",
+            "robot_ori_cos",
+            "robot_ori_sin",
+            "robot_2d_ori",
+            "robot_2d_ori_cos",
+            "robot_2d_ori_sin",
+            "robot_lin_vel",
+            "robot_ang_vel",
+            "arm_left_qpos",
+            "arm_left_qpos_sin",
+            "arm_left_qpos_cos",
+            "eef_left_pos",
+            "eef_left_quat",
+            "gripper_left_qpos",
+            "arm_right_qpos",
+            "arm_right_qpos_sin",
+            "arm_right_qpos_cos",
+            "eef_right_pos",
+            "eef_right_quat",
+            "gripper_right_qpos",
+            "trunk_qpos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31
+          ],
+          "modality_keys": [
+            "base",
+            "torso",
+            "left_arm",
+            "left_gripper",
+            "right_arm",
+            "right_gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "trunk_qpos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_left_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_right_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "gr1": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "ego_view_bg_crop_pad_res256_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "robocasa_panda_omron": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "res256_image_side_0",
+            "res256_image_side_1",
+            "res256_image_wrist_0"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "end_effector_position_relative",
+            "end_effector_rotation_relative",
+            "gripper_qpos",
+            "base_position",
+            "base_rotation"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "end_effector_position",
+            "end_effector_rotation",
+            "gripper_close",
+            "base_motion",
+            "control_mode"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.action.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "front",
+            "wrist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": null,
+    "image_target_size": null,
+    "use_albumentations": true,
+    "random_rotation_angle": null,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "model_name": "nvidia/Eagle-Block2A-2B-v2",
+    "model_type": "eagle",
+    "formalize_language": true,
+    "max_state_dim": 128,
+    "max_action_dim": 128,
+    "max_action_horizon": 50,
+    "use_percentiles": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": true,
+    "use_relative_action": true
+  }
+}

checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19c8cd95b87e2c74d2b8eeb87e84e4bdde690b9a799e7bfed5ac657fbe0263a2
+size 14645

checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed5dcdc121951b51745034e467890f8a7b69b505c12b1a668d378806746fee59
+size 1465

checkpoint-1000/trainer_state.json

@@ -0,0 +1,634 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.1,
+  "eval_steps": 500,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "grad_norm": 2.416853189468384,
+      "learning_rate": 1.8e-06,
+      "loss": 1.1685,
+      "step": 10
+    },
+    {
+      "grad_norm": 1.0608258247375488,
+      "learning_rate": 3.8e-06,
+      "loss": 1.1645,
+      "step": 20
+    },
+    {
+      "grad_norm": 0.6068993210792542,
+      "learning_rate": 5.8e-06,
+      "loss": 1.1434,
+      "step": 30
+    },
+    {
+      "grad_norm": 0.4165553152561188,
+      "learning_rate": 7.8e-06,
+      "loss": 1.1309,
+      "step": 40
+    },
+    {
+      "grad_norm": 0.5539636015892029,
+      "learning_rate": 9.800000000000001e-06,
+      "loss": 1.1056,
+      "step": 50
+    },
+    {
+      "grad_norm": 0.48063579201698303,
+      "learning_rate": 1.18e-05,
+      "loss": 1.1079,
+      "step": 60
+    },
+    {
+      "grad_norm": 0.4390954077243805,
+      "learning_rate": 1.3800000000000002e-05,
+      "loss": 1.0909,
+      "step": 70
+    },
+    {
+      "grad_norm": 0.39306357502937317,
+      "learning_rate": 1.58e-05,
+      "loss": 1.0837,
+      "step": 80
+    },
+    {
+      "grad_norm": 0.5071999430656433,
+      "learning_rate": 1.78e-05,
+      "loss": 1.0625,
+      "step": 90
+    },
+    {
+      "grad_norm": 0.38793864846229553,
+      "learning_rate": 1.9800000000000004e-05,
+      "loss": 1.0615,
+      "step": 100
+    },
+    {
+      "grad_norm": 0.6121100187301636,
+      "learning_rate": 2.18e-05,
+      "loss": 1.0406,
+      "step": 110
+    },
+    {
+      "grad_norm": 0.6436424851417542,
+      "learning_rate": 2.38e-05,
+      "loss": 1.0538,
+      "step": 120
+    },
+    {
+      "grad_norm": 3.180732011795044,
+      "learning_rate": 2.58e-05,
+      "loss": 0.9922,
+      "step": 130
+    },
+    {
+      "grad_norm": 0.8403401374816895,
+      "learning_rate": 2.7800000000000005e-05,
+      "loss": 0.9564,
+      "step": 140
+    },
+    {
+      "grad_norm": 1.5241442918777466,
+      "learning_rate": 2.98e-05,
+      "loss": 0.8884,
+      "step": 150
+    },
+    {
+      "grad_norm": 1.140098214149475,
+      "learning_rate": 3.18e-05,
+      "loss": 0.8313,
+      "step": 160
+    },
+    {
+      "grad_norm": 1.406286597251892,
+      "learning_rate": 3.38e-05,
+      "loss": 0.7616,
+      "step": 170
+    },
+    {
+      "grad_norm": 2.2860851287841797,
+      "learning_rate": 3.58e-05,
+      "loss": 0.6981,
+      "step": 180
+    },
+    {
+      "grad_norm": 2.2754695415496826,
+      "learning_rate": 3.7800000000000004e-05,
+      "loss": 0.6285,
+      "step": 190
+    },
+    {
+      "grad_norm": 2.531214714050293,
+      "learning_rate": 3.9800000000000005e-05,
+      "loss": 0.4895,
+      "step": 200
+    },
+    {
+      "grad_norm": 2.7056477069854736,
+      "learning_rate": 4.18e-05,
+      "loss": 0.3478,
+      "step": 210
+    },
+    {
+      "grad_norm": 1.7513630390167236,
+      "learning_rate": 4.38e-05,
+      "loss": 0.2509,
+      "step": 220
+    },
+    {
+      "grad_norm": 1.6927777528762817,
+      "learning_rate": 4.58e-05,
+      "loss": 0.1911,
+      "step": 230
+    },
+    {
+      "grad_norm": 2.264531135559082,
+      "learning_rate": 4.78e-05,
+      "loss": 0.2006,
+      "step": 240
+    },
+    {
+      "grad_norm": 1.3290256261825562,
+      "learning_rate": 4.9800000000000004e-05,
+      "loss": 0.1654,
+      "step": 250
+    },
+    {
+      "grad_norm": 1.5439292192459106,
+      "learning_rate": 5.1800000000000005e-05,
+      "loss": 0.172,
+      "step": 260
+    },
+    {
+      "grad_norm": 1.5491207838058472,
+      "learning_rate": 5.380000000000001e-05,
+      "loss": 0.1494,
+      "step": 270
+    },
+    {
+      "grad_norm": 1.1139971017837524,
+      "learning_rate": 5.580000000000001e-05,
+      "loss": 0.143,
+      "step": 280
+    },
+    {
+      "grad_norm": 1.2239474058151245,
+      "learning_rate": 5.7799999999999995e-05,
+      "loss": 0.1281,
+      "step": 290
+    },
+    {
+      "grad_norm": 1.093898057937622,
+      "learning_rate": 5.9800000000000003e-05,
+      "loss": 0.1234,
+      "step": 300
+    },
+    {
+      "grad_norm": 1.1576344966888428,
+      "learning_rate": 6.18e-05,
+      "loss": 0.1147,
+      "step": 310
+    },
+    {
+      "grad_norm": 1.2395023107528687,
+      "learning_rate": 6.38e-05,
+      "loss": 0.1143,
+      "step": 320
+    },
+    {
+      "grad_norm": 1.2169476747512817,
+      "learning_rate": 6.58e-05,
+      "loss": 0.1144,
+      "step": 330
+    },
+    {
+      "grad_norm": 0.9942442774772644,
+      "learning_rate": 6.780000000000001e-05,
+      "loss": 0.1152,
+      "step": 340
+    },
+    {
+      "grad_norm": 1.0080304145812988,
+      "learning_rate": 6.98e-05,
+      "loss": 0.1081,
+      "step": 350
+    },
+    {
+      "grad_norm": 1.1731500625610352,
+      "learning_rate": 7.18e-05,
+      "loss": 0.1146,
+      "step": 360
+    },
+    {
+      "grad_norm": 0.8233430981636047,
+      "learning_rate": 7.38e-05,
+      "loss": 0.1135,
+      "step": 370
+    },
+    {
+      "grad_norm": 1.0143144130706787,
+      "learning_rate": 7.58e-05,
+      "loss": 0.0979,
+      "step": 380
+    },
+    {
+      "grad_norm": 0.8134552240371704,
+      "learning_rate": 7.780000000000001e-05,
+      "loss": 0.1075,
+      "step": 390
+    },
+    {
+      "grad_norm": 0.736839234828949,
+      "learning_rate": 7.98e-05,
+      "loss": 0.1023,
+      "step": 400
+    },
+    {
+      "grad_norm": 0.8744317889213562,
+      "learning_rate": 8.18e-05,
+      "loss": 0.1093,
+      "step": 410
+    },
+    {
+      "grad_norm": 0.7313347458839417,
+      "learning_rate": 8.38e-05,
+      "loss": 0.1022,
+      "step": 420
+    },
+    {
+      "grad_norm": 0.8588263988494873,
+      "learning_rate": 8.58e-05,
+      "loss": 0.0957,
+      "step": 430
+    },
+    {
+      "grad_norm": 0.8712506890296936,
+      "learning_rate": 8.78e-05,
+      "loss": 0.0986,
+      "step": 440
+    },
+    {
+      "grad_norm": 0.5078502893447876,
+      "learning_rate": 8.98e-05,
+      "loss": 0.0941,
+      "step": 450
+    },
+    {
+      "grad_norm": 0.8700484037399292,
+      "learning_rate": 9.180000000000001e-05,
+      "loss": 0.0968,
+      "step": 460
+    },
+    {
+      "grad_norm": 0.8209111094474792,
+      "learning_rate": 9.38e-05,
+      "loss": 0.1025,
+      "step": 470
+    },
+    {
+      "grad_norm": 0.7114784717559814,
+      "learning_rate": 9.58e-05,
+      "loss": 0.0907,
+      "step": 480
+    },
+    {
+      "grad_norm": 0.8576576709747314,
+      "learning_rate": 9.78e-05,
+      "loss": 0.085,
+      "step": 490
+    },
+    {
+      "grad_norm": 0.7132489085197449,
+      "learning_rate": 9.98e-05,
+      "loss": 0.0826,
+      "step": 500
+    },
+    {
+      "grad_norm": 0.5286405682563782,
+      "learning_rate": 9.9999778549206e-05,
+      "loss": 0.0773,
+      "step": 510
+    },
+    {
+      "grad_norm": 0.6641912460327148,
+      "learning_rate": 9.999901304280685e-05,
+      "loss": 0.0832,
+      "step": 520
+    },
+    {
+      "grad_norm": 0.6113771200180054,
+      "learning_rate": 9.999770075521164e-05,
+      "loss": 0.0818,
+      "step": 530
+    },
+    {
+      "grad_norm": 0.6849881410598755,
+      "learning_rate": 9.99958417007713e-05,
+      "loss": 0.0844,
+      "step": 540
+    },
+    {
+      "grad_norm": 0.510468602180481,
+      "learning_rate": 9.999343589981615e-05,
+      "loss": 0.0712,
+      "step": 550
+    },
+    {
+      "grad_norm": 0.4960484802722931,
+      "learning_rate": 9.999048337865568e-05,
+      "loss": 0.0677,
+      "step": 560
+    },
+    {
+      "grad_norm": 0.6222489476203918,
+      "learning_rate": 9.998698416957815e-05,
+      "loss": 0.0677,
+      "step": 570
+    },
+    {
+      "grad_norm": 0.6377440690994263,
+      "learning_rate": 9.998293831085037e-05,
+      "loss": 0.0757,
+      "step": 580
+    },
+    {
+      "grad_norm": 0.6644724011421204,
+      "learning_rate": 9.997834584671719e-05,
+      "loss": 0.0721,
+      "step": 590
+    },
+    {
+      "grad_norm": 0.42810145020484924,
+      "learning_rate": 9.997320682740107e-05,
+      "loss": 0.0741,
+      "step": 600
+    },
+    {
+      "grad_norm": 0.47897064685821533,
+      "learning_rate": 9.996752130910149e-05,
+      "loss": 0.0723,
+      "step": 610
+    },
+    {
+      "grad_norm": 0.512750506401062,
+      "learning_rate": 9.99612893539944e-05,
+      "loss": 0.0773,
+      "step": 620
+    },
+    {
+      "grad_norm": 0.5588595867156982,
+      "learning_rate": 9.995451103023144e-05,
+      "loss": 0.0772,
+      "step": 630
+    },
+    {
+      "grad_norm": 0.6482793092727661,
+      "learning_rate": 9.994718641193928e-05,
+      "loss": 0.071,
+      "step": 640
+    },
+    {
+      "grad_norm": 0.4272823631763458,
+      "learning_rate": 9.993931557921874e-05,
+      "loss": 0.0695,
+      "step": 650
+    },
+    {
+      "grad_norm": 0.5539937019348145,
+      "learning_rate": 9.993089861814402e-05,
+      "loss": 0.0548,
+      "step": 660
+    },
+    {
+      "grad_norm": 0.5200349688529968,
+      "learning_rate": 9.992193562076166e-05,
+      "loss": 0.0649,
+      "step": 670
+    },
+    {
+      "grad_norm": 0.35027024149894714,
+      "learning_rate": 9.991242668508954e-05,
+      "loss": 0.0599,
+      "step": 680
+    },
+    {
+      "grad_norm": 0.4623376429080963,
+      "learning_rate": 9.990237191511587e-05,
+      "loss": 0.0593,
+      "step": 690
+    },
+    {
+      "grad_norm": 0.42851006984710693,
+      "learning_rate": 9.989177142079802e-05,
+      "loss": 0.0518,
+      "step": 700
+    },
+    {
+      "grad_norm": 0.5397882461547852,
+      "learning_rate": 9.988062531806126e-05,
+      "loss": 0.0614,
+      "step": 710
+    },
+    {
+      "grad_norm": 0.298032283782959,
+      "learning_rate": 9.986893372879762e-05,
+      "loss": 0.0663,
+      "step": 720
+    },
+    {
+      "grad_norm": 0.6170604228973389,
+      "learning_rate": 9.985669678086443e-05,
+      "loss": 0.0594,
+      "step": 730
+    },
+    {
+      "grad_norm": 0.49378132820129395,
+      "learning_rate": 9.984391460808298e-05,
+      "loss": 0.0707,
+      "step": 740
+    },
+    {
+      "grad_norm": 0.5525211095809937,
+      "learning_rate": 9.983058735023709e-05,
+      "loss": 0.0643,
+      "step": 750
+    },
+    {
+      "grad_norm": 0.4275728166103363,
+      "learning_rate": 9.98167151530715e-05,
+      "loss": 0.0632,
+      "step": 760
+    },
+    {
+      "grad_norm": 0.3775525391101837,
+      "learning_rate": 9.980229816829034e-05,
+      "loss": 0.0595,
+      "step": 770
+    },
+    {
+      "grad_norm": 0.42739713191986084,
+      "learning_rate": 9.978733655355544e-05,
+      "loss": 0.0568,
+      "step": 780
+    },
+    {
+      "grad_norm": 0.4654758870601654,
+      "learning_rate": 9.977183047248464e-05,
+      "loss": 0.0516,
+      "step": 790
+    },
+    {
+      "grad_norm": 0.4031694233417511,
+      "learning_rate": 9.975578009464992e-05,
+      "loss": 0.0581,
+      "step": 800
+    },
+    {
+      "grad_norm": 0.34200575947761536,
+      "learning_rate": 9.97391855955757e-05,
+      "loss": 0.0457,
+      "step": 810
+    },
+    {
+      "grad_norm": 0.3750764727592468,
+      "learning_rate": 9.972204715673669e-05,
+      "loss": 0.0509,
+      "step": 820
+    },
+    {
+      "grad_norm": 0.5093910694122314,
+      "learning_rate": 9.970436496555617e-05,
+      "loss": 0.0489,
+      "step": 830
+    },
+    {
+      "grad_norm": 0.6330306529998779,
+      "learning_rate": 9.968613921540373e-05,
+      "loss": 0.0531,
+      "step": 840
+    },
+    {
+      "grad_norm": 0.44549983739852905,
+      "learning_rate": 9.966737010559326e-05,
+      "loss": 0.0569,
+      "step": 850
+    },
+    {
+      "grad_norm": 0.3454737961292267,
+      "learning_rate": 9.964805784138072e-05,
+      "loss": 0.0497,
+      "step": 860
+    },
+    {
+      "grad_norm": 0.29266422986984253,
+      "learning_rate": 9.962820263396195e-05,
+      "loss": 0.0472,
+      "step": 870
+    },
+    {
+      "grad_norm": 0.38099658489227295,
+      "learning_rate": 9.960780470047033e-05,
+      "loss": 0.0556,
+      "step": 880
+    },
+    {
+      "grad_norm": 0.37069812417030334,
+      "learning_rate": 9.958686426397437e-05,
+      "loss": 0.0464,
+      "step": 890
+    },
+    {
+      "grad_norm": 0.5007408261299133,
+      "learning_rate": 9.956538155347534e-05,
+      "loss": 0.0507,
+      "step": 900
+    },
+    {
+      "grad_norm": 0.45427390933036804,
+      "learning_rate": 9.95433568039047e-05,
+      "loss": 0.048,
+      "step": 910
+    },
+    {
+      "grad_norm": 0.38306254148483276,
+      "learning_rate": 9.952079025612162e-05,
+      "loss": 0.0535,
+      "step": 920
+    },
+    {
+      "grad_norm": 0.6907458901405334,
+      "learning_rate": 9.949768215691022e-05,
+      "loss": 0.0505,
+      "step": 930
+    },
+    {
+      "grad_norm": 0.5054745674133301,
+      "learning_rate": 9.9474032758977e-05,
+      "loss": 0.0551,
+      "step": 940
+    },
+    {
+      "grad_norm": 0.38975587487220764,
+      "learning_rate": 9.944984232094794e-05,
+      "loss": 0.0571,
+      "step": 950
+    },
+    {
+      "grad_norm": 0.4025571644306183,
+      "learning_rate": 9.942511110736584e-05,
+      "loss": 0.0486,
+      "step": 960
+    },
+    {
+      "grad_norm": 0.3891156017780304,
+      "learning_rate": 9.939983938868726e-05,
+      "loss": 0.0408,
+      "step": 970
+    },
+    {
+      "grad_norm": 0.38383904099464417,
+      "learning_rate": 9.93740274412797e-05,
+      "loss": 0.0468,
+      "step": 980
+    },
+    {
+      "grad_norm": 0.45907771587371826,
+      "learning_rate": 9.934767554741846e-05,
+      "loss": 0.0532,
+      "step": 990
+    },
+    {
+      "grad_norm": 0.28416934609413147,
+      "learning_rate": 9.932078399528361e-05,
+      "loss": 0.0454,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 36,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4524ac143a9f2191e97d0434f3b2f8c78c2ac53e3116f1e79b416f15557f3fa0
+size 5713

checkpoint-1000/wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d6", "run_id": "so100_finetune"}

checkpoint-2000/config.json

@@ -0,0 +1,70 @@
+{
+  "action_horizon": 50,
+  "add_pos_embed": true,
+  "apply_sincos_state_encoding": true,
+  "architectures": [
+    "Gr00tN1d6"
+  ],
+  "attn_dropout": 0.2,
+  "attn_implementation": null,
+  "backbone_embedding_dim": 2048,
+  "backbone_model_type": "eagle",
+  "backbone_trainable_params_fp32": true,
+  "collator_overwrite_image_inputs": false,
+  "color_jitter_params": {
+    "brightness": 0.1,
+    "contrast": 0.1,
+    "hue": 0.1,
+    "saturation": 0.1
+  },
+  "crop_fraction": 0.95,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "eagle_collator": true,
+  "formalize_language": true,
+  "gemma_collator": false,
+  "hidden_size": 1024,
+  "image_crop_size": null,
+  "image_target_size": null,
+  "input_embedding_dim": 1536,
+  "load_bf16": true,
+  "max_action_dim": 128,
+  "max_num_embodiments": 32,
+  "max_seq_len": 1024,
+  "max_state_dim": 128,
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "model_type": "Gr00tN1d6",
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_inference_timesteps": 4,
+  "num_timestep_buckets": 1000,
+  "random_rotation_angle": null,
+  "reproject_vision": false,
+  "select_layer": 16,
+  "shortest_image_edge": 256,
+  "state_dropout_prob": 0.0,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "tune_diffusion_model": true,
+  "tune_llm": false,
+  "tune_projector": true,
+  "tune_top_llm_layers": 4,
+  "tune_visual": false,
+  "tune_vlln": true,
+  "use_albumentations_transforms": true,
+  "use_alternate_vl_dit": true,
+  "use_flash_attention": true,
+  "use_relative_action": true,
+  "use_vlln": true
+}

checkpoint-2000/embodiment_id.json

@@ -0,0 +1,10 @@
+{
+  "robocasa_panda_omron": 13,
+  "gr1": 20,
+  "behavior_r1_pro": 24,
+  "unitree_g1": 8,
+  "oxe_google": 0,
+  "oxe_widowx": 1,
+  "libero_panda": 2,
+  "new_embodiment": 10
+}

checkpoint-2000/experiment_cfg/conf.yaml

@@ -0,0 +1,209 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d6
+  model_dtype: bfloat16
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  backbone_model_type: eagle
+  model_revision: null
+  tune_top_llm_layers: 4
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 16
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  collator_overwrite_image_inputs: false
+  eagle_collator: true
+  backbone_trainable_params_fp32: true
+  image_crop_size: null
+  image_target_size: null
+  shortest_image_edge: 256
+  crop_fraction: 0.95
+  random_rotation_angle: null
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    saturation: 0.5
+    hue: 0.08
+  use_albumentations_transforms: true
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_relative_action: true
+  max_state_dim: 29
+  max_action_dim: 29
+  action_horizon: 16
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 32
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.0
+  state_additive_noise_scale: 0.0
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - /content/dataset/azazdeaz/record-test/azazdeaz/record-test/
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: RELATIVE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: fork
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: torchcodec
+training:
+  output_dir: /content/so100_finetune
+  experiment_name: null
+  max_steps: 10000
+  global_batch_size: 36
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 1000
+  save_total_limit: 5
+  save_vl_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 4
+  remove_unused_columns: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 10000
+save_steps: 1000

checkpoint-2000/experiment_cfg/config.yaml

@@ -0,0 +1,243 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - /content/dataset/azazdeaz/record-test/azazdeaz/record-test/
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - relative
+          state_key: null
+          type: &id002 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: *id002
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: fork
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  random_chop: 0.0
+  seed: 42
+  shard_size: 1024
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: torchcodec
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
+  _attn_implementation_autoset: false
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  add_cross_attention: false
+  architectures: null
+  backbone_model_type: eagle
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    hue: 0.08
+    saturation: 0.5
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 32
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  eagle_collator: true
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_attentions: false
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.0
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torch_dtype: null
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 4
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 36
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 10000
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: /content/so100_finetune
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_steps: 1000
+  save_total_limit: 5
+  save_vl_model: false
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

checkpoint-2000/experiment_cfg/dataset_statistics.json

@@ -0,0 +1,824 @@
+{
+  "new_embodiment": {
+    "state": {
+      "single_arm": {
+        "min": [
+          -60.105262756347656,
+          -99.31827545166016,
+          -43.24324417114258,
+          44.688323974609375,
+          -80.68241119384766
+        ],
+        "max": [
+          34.31578826904297,
+          45.80315399169922,
+          99.62721252441406,
+          97.62950134277344,
+          -4.8293962478637695
+        ],
+        "mean": [
+          -3.5325422842834118,
+          -39.55833784368612,
+          47.61425589641663,
+          68.3154743394685,
+          -46.70357059936142
+        ],
+        "std": [
+          27.459571708144434,
+          41.074676874488844,
+          34.075122244859095,
+          13.351843152336091,
+          19.483248111140814
+        ],
+        "q01": [
+          -43.89320641903556,
+          -99.31827545166016,
+          2.96704117550838,
+          52.128458407208086,
+          -64.9047852897644
+        ],
+        "q99": [
+          23.91888349814585,
+          10.271718478515513,
+          99.48718710059721,
+          91.53274853632244,
+          -26.5701696578577
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.9803921580314636
+        ],
+        "max": [
+          35.9943962097168
+        ],
+        "mean": [
+          20.298597213926165
+        ],
+        "std": [
+          9.780378321306017
+        ],
+        "q01": [
+          6.1409783557640525
+        ],
+        "q99": [
+          29.83243577963609
+        ]
+      }
+    },
+    "action": {
+      "single_arm": {
+        "min": [
+          -60.843040466308594,
+          -100.0,
+          -47.45454406738281,
+          44.675209045410156,
+          -81.04986572265625
+        ],
+        "max": [
+          34.99722671508789,
+          45.694801330566406,
+          100.0,
+          97.96730041503906,
+          -4.566929340362549
+        ],
+        "mean": [
+          -3.4051861822555503,
+          -40.21667093678776,
+          46.219495035627304,
+          68.11321517887163,
+          -46.72253879798044
+        ],
+        "std": [
+          27.604988373472494,
+          40.826917813818575,
+          34.82453909741334,
+          13.439408862929378,
+          19.518525610689558
+        ],
+        "q01": [
+          -44.29914832538184,
+          -99.9431293020161,
+          -0.6294070457600729,
+          51.77255040025036,
+          -65.3110059408939
+        ],
+        "q99": [
+          24.517798066878782,
+          10.287138016986633,
+          99.9985524069893,
+          91.79085298874445,
+          -26.382156973884847
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.25553661584854126
+        ],
+        "max": [
+          36.28620147705078
+        ],
+        "mean": [
+          19.60203928355869
+        ],
+        "std": [
+          10.896771214917207
+        ],
+        "q01": [
+          2.708348265017052
+        ],
+        "q99": [
+          30.222431877944597
+        ]
+      }
+    },
+    "relative_action": {
+      "single_arm": {
+        "min": [
+          [
+            -9.055549621582031,
+            -13.95650863647461,
+            -14.800384521484375,
+            -13.61447525024414,
+            -7.874013900756836
+          ],
+          [
+            -11.160816192626953,
+            -16.68364715576172,
+            -17.25493621826172,
+            -16.177448272705078,
+            -9.501310348510742
+          ],
+          [
+            -13.476604461669922,
+            -19.667449951171875,
+            -19.89129638671875,
+            -18.372356414794922,
+            -11.23359489440918
+          ],
+          [
+            -15.26607894897461,
+            -22.73650360107422,
+            -22.314407348632812,
+            -20.567264556884766,
+            -12.755905151367188
+          ],
+          [
+            -16.950286865234375,
+            -25.550708770751953,
+            -24.457931518554688,
+            -22.246448516845703,
+            -14.120733261108398
+          ],
+          [
+            -18.84502410888672,
+            -28.619762420654297,
+            -26.764127731323242,
+            -23.572124481201172,
+            -15.538057327270508
+          ],
+          [
+            -20.50891876220703,
+            -31.262561798095703,
+            -29.18724250793457,
+            -24.625682830810547,
+            -16.79789924621582
+          ],
+          [
+            -21.950958251953125,
+            -33.90535354614258,
+            -31.703550338745117,
+            -25.421085357666016,
+            -17.9002628326416
+          ],
+          [
+            -23.351024627685547,
+            -36.545902252197266,
+            -33.976280212402344,
+            -25.947864532470703,
+            -18.89763641357422
+          ],
+          [
+            -24.79306411743164,
+            -39.18869400024414,
+            -36.27018737792969,
+            -26.389755249023438,
+            -19.737533569335938
+          ],
+          [
+            -26.056222915649414,
+            -41.40523910522461,
+            -38.54291534423828,
+            -26.566513061523438,
+            -20.57742691040039
+          ],
+          [
+            -27.2141170501709,
+            -43.45127487182617,
+            -40.633819580078125,
+            -26.654312133789062,
+            -21.732280731201172
+          ],
+          [
+            -28.372011184692383,
+            -45.497310638427734,
+            -42.90654754638672,
+            -26.654312133789062,
+            -22.834644317626953
+          ],
+          [
+            -29.5299072265625,
+            -47.542476654052734,
+            -44.9827995300293,
+            -26.654312133789062,
+            -23.674541473388672
+          ],
+          [
+            -30.687801361083984,
+            -49.50242614746094,
+            -47.19833755493164,
+            -26.654312133789062,
+            -24.304462432861328
+          ],
+          [
+            -31.84569549560547,
+            -51.29353713989258,
+            -49.528255462646484,
+            -26.654312133789062,
+            -25.406824111938477
+          ]
+        ],
+        "max": [
+          [
+            11.800273895263672,
+            13.390182495117188,
+            11.2484130859375,
+            12.953540802001953,
+            8.97637939453125
+          ],
+          [
+            14.462503433227539,
+            16.288734436035156,
+            13.857921600341797,
+            15.763023376464844,
+            10.813648223876953
+          ],
+          [
+            16.902881622314453,
+            19.443038940429688,
+            16.2484130859375,
+            18.39691162109375,
+            12.703411102294922
+          ],
+          [
+            19.40987205505371,
+            22.16992950439453,
+            18.857921600341797,
+            20.783126831054688,
+            14.645668029785156
+          ],
+          [
+            21.850250244140625,
+            24.812728881835938,
+            21.221553802490234,
+            22.815826416015625,
+            16.482940673828125
+          ],
+          [
+            24.290626525878906,
+            27.455524444580078,
+            23.551467895507812,
+            24.671768188476562,
+            18.530181884765625
+          ],
+          [
+            26.50115203857422,
+            30.011974334716797,
+            25.857921600341797,
+            26.515491485595703,
+            20.262466430664062
+          ],
+          [
+            28.71167755126953,
+            32.31278610229492,
+            28.187835693359375,
+            27.920230865478516,
+            22.099735260009766
+          ],
+          [
+            31.132732391357422,
+            34.52932357788086,
+            30.585193634033203,
+            29.34360122680664,
+            23.832019805908203
+          ],
+          [
+            33.24032974243164,
+            36.490108489990234,
+            32.91510772705078,
+            30.757648468017578,
+            25.56430435180664
+          ],
+          [
+            35.237003326416016,
+            38.621395111083984,
+            35.187835693359375,
+            32.26008224487305,
+            27.296588897705078
+          ],
+          [
+            37.0264778137207,
+            40.411678314208984,
+            37.27873992919922,
+            33.66482162475586,
+            28.87139129638672
+          ],
+          [
+            38.815948486328125,
+            42.372459411621094,
+            39.32906723022461,
+            34.99049758911133,
+            30.498687744140625
+          ],
+          [
+            40.147064208984375,
+            43.99224090576172,
+            41.238162994384766,
+            35.6975212097168,
+            31.9160099029541
+          ],
+          [
+            41.552791595458984,
+            45.61201858520508,
+            43.13301467895508,
+            36.0510368347168,
+            33.33333206176758
+          ],
+          [
+            43.0264778137207,
+            47.23110580444336,
+            45.22392654418945,
+            36.402225494384766,
+            34.75065612792969
+          ]
+        ],
+        "mean": [
+          [
+            0.11750347912311554,
+            -0.6653266549110413,
+            -1.4583468437194824,
+            -0.2053278237581253,
+            -0.01863335259258747
+          ],
+          [
+            0.12236570566892624,
+            -0.6657301187515259,
+            -1.4583476781845093,
+            -0.20791052281856537,
+            -0.02185939997434616
+          ],
+          [
+            0.12741880118846893,
+            -0.6660965085029602,
+            -1.4583467245101929,
+            -0.21049359440803528,
+            -0.025085171684622765
+          ],
+          [
+            0.13261470198631287,
+            -0.6664630174636841,
+            -1.4583468437194824,
+            -0.21307627856731415,
+            -0.02831093594431877
+          ],
+          [
+            0.13785837590694427,
+            -0.6668296456336975,
+            -1.4583486318588257,
+            -0.21565881371498108,
+            -0.03153714910149574
+          ],
+          [
+            0.14319713413715363,
+            -0.6671591401100159,
+            -1.4583512544631958,
+            -0.21824198961257935,
+            -0.03474012389779091
+          ],
+          [
+            0.14853604137897491,
+            -0.6674894094467163,
+            -1.458350419998169,
+            -0.22082529962062836,
+            -0.03794342279434204
+          ],
+          [
+            0.15382729470729828,
+            -0.6678190231323242,
+            -1.4583524465560913,
+            -0.22340813279151917,
+            -0.041146621108055115
+          ],
+          [
+            0.15907101333141327,
+            -0.6681490540504456,
+            -1.458353042602539,
+            -0.2259913831949234,
+            -0.04434990882873535
+          ],
+          [
+            0.16426703333854675,
+            -0.668441653251648,
+            -1.458351492881775,
+            -0.22857406735420227,
+            -0.04755344241857529
+          ],
+          [
+            0.1694149672985077,
+            -0.668734610080719,
+            -1.4583535194396973,
+            -0.23115694522857666,
+            -0.05075618624687195
+          ],
+          [
+            0.17465919256210327,
+            -0.6690278649330139,
+            -1.458351731300354,
+            -0.2337394654750824,
+            -0.05395958200097084
+          ],
+          [
+            0.17990276217460632,
+            -0.6693212985992432,
+            -1.4583524465560913,
+            -0.23632188141345978,
+            -0.05716199427843094
+          ],
+          [
+            0.18505056202411652,
+            -0.6696141958236694,
+            -1.4583513736724854,
+            -0.2389044612646103,
+            -0.060365449637174606
+          ],
+          [
+            0.1901988983154297,
+            -0.6699072122573853,
+            -1.4583531618118286,
+            -0.24148696660995483,
+            -0.06356889009475708
+          ],
+          [
+            0.19529956579208374,
+            -0.6702005863189697,
+            -1.4583516120910645,
+            -0.24407006800174713,
+            -0.0667722150683403
+          ]
+        ],
+        "std": [
+          [
+            2.756209135055542,
+            4.048781394958496,
+            4.834221839904785,
+            2.4045844078063965,
+            1.831181526184082
+          ],
+          [
+            3.375519037246704,
+            4.940067768096924,
+            5.704192638397217,
+            2.936358690261841,
+            2.2514567375183105
+          ],
+          [
+            3.9847497940063477,
+            5.8236775398254395,
+            6.570866107940674,
+            3.443351984024048,
+            2.663818359375
+          ],
+          [
+            4.582452774047852,
+            6.696892738342285,
+            7.430098056793213,
+            3.9259438514709473,
+            3.067777633666992
+          ],
+          [
+            5.167651653289795,
+            7.5579352378845215,
+            8.279448509216309,
+            4.385776996612549,
+            3.462792158126831
+          ],
+          [
+            5.740141868591309,
+            8.406184196472168,
+            9.11788558959961,
+            4.825246810913086,
+            3.848816156387329
+          ],
+          [
+            6.2996673583984375,
+            9.240605354309082,
+            9.944042205810547,
+            5.246663570404053,
+            4.225630283355713
+          ],
+          [
+            6.846652507781982,
+            10.06134033203125,
+            10.75770092010498,
+            5.652442932128906,
+            4.593677043914795
+          ],
+          [
+            7.3811469078063965,
+            10.86816120147705,
+            11.558343887329102,
+            6.044579982757568,
+            4.9531779289245605
+          ],
+          [
+            7.903446197509766,
+            11.661355972290039,
+            12.34595012664795,
+            6.424712657928467,
+            5.304342269897461
+          ],
+          [
+            8.414060592651367,
+            12.44128131866455,
+            13.120593070983887,
+            6.794016361236572,
+            5.647256851196289
+          ],
+          [
+            8.913477897644043,
+            13.208165168762207,
+            13.88218879699707,
+            7.153115272521973,
+            5.98213529586792
+          ],
+          [
+            9.402256965637207,
+            13.962782859802246,
+            14.631062507629395,
+            7.5023603439331055,
+            6.309183597564697
+          ],
+          [
+            9.880831718444824,
+            14.70537281036377,
+            15.367119789123535,
+            7.841666221618652,
+            6.628418445587158
+          ],
+          [
+            10.349787712097168,
+            15.43662166595459,
+            16.090578079223633,
+            8.171104431152344,
+            6.940072536468506
+          ],
+          [
+            10.809670448303223,
+            16.15700340270996,
+            16.801616668701172,
+            8.490680694580078,
+            7.244339466094971
+          ]
+        ],
+        "q01": [
+          [
+            -7.149912223815918,
+            -12.172602081298828,
+            -13.191639633178712,
+            -7.083407516479492,
+            -5.301837921142578
+          ],
+          [
+            -8.969949054718018,
+            -14.76386520385742,
+            -15.673493194580079,
+            -8.597449951171875,
+            -6.4955373382568355
+          ],
+          [
+            -10.634601535797119,
+            -17.415445404052733,
+            -18.04468208312988,
+            -9.786780090332032,
+            -7.755381164550781
+          ],
+          [
+            -12.332097396850585,
+            -19.81734550476074,
+            -20.49539176940918,
+            -10.772034606933593,
+            -8.81890007019043
+          ],
+          [
+            -14.054951286315918,
+            -22.659150772094726,
+            -22.971768417358398,
+            -11.810748291015624,
+            -10.076640815734862
+          ],
+          [
+            -15.467545299530029,
+            -24.854450073242187,
+            -25.42631248474121,
+            -12.89973159790039,
+            -11.391075839996338
+          ],
+          [
+            -16.85467258453369,
+            -27.482084159851073,
+            -27.82916431427002,
+            -13.832246704101562,
+            -12.532282905578613
+          ],
+          [
+            -18.258717765808104,
+            -29.645261459350586,
+            -29.891822814941406,
+            -14.627758026123047,
+            -13.543306350708008
+          ],
+          [
+            -19.36220642089844,
+            -31.862525711059572,
+            -32.179530944824215,
+            -15.569841918945311,
+            -14.78740104675293
+          ],
+          [
+            -20.866409225463865,
+            -34.33618782043457,
+            -34.25658073425293,
+            -16.842924346923827,
+            -15.576902465820313
+          ],
+          [
+            -21.95573440551758,
+            -36.528687438964845,
+            -36.38519187927246,
+            -17.423948669433592,
+            -16.82309829711914
+          ],
+          [
+            -23.461913070678712,
+            -38.73653938293457,
+            -38.30317817687988,
+            -17.647796630859375,
+            -17.742782592773438
+          ],
+          [
+            -24.53435989379883,
+            -40.874423828124996,
+            -40.44069320678711,
+            -18.515239715576172,
+            -18.879788017272947
+          ],
+          [
+            -25.933171768188476,
+            -43.23724739074707,
+            -42.29842636108398,
+            -19.954179382324217,
+            -19.801573486328124
+          ],
+          [
+            -27.245162963867188,
+            -45.308877029418944,
+            -43.92444366455078,
+            -20.61133575439453,
+            -20.9133874130249
+          ],
+          [
+            -28.540471458435057,
+            -47.20507698059082,
+            -45.95519866943359,
+            -20.804307861328123,
+            -21.69133777618408
+          ]
+        ],
+        "q99": [
+          [
+            8.69469722747799,
+            10.299373474121074,
+            9.785606613159175,
+            9.179881286621086,
+            6.271916503906227
+          ],
+          [
+            10.615129890441846,
+            12.666508712768518,
+            12.091986007690421,
+            11.210918426513642,
+            7.833070821762051
+          ],
+          [
+            12.4686942863464,
+            14.899181365966779,
+            14.446693725585934,
+            13.182954406738268,
+            9.380576934814396
+          ],
+          [
+            14.271173782348615,
+            17.33157432556147,
+            16.80153594970703,
+            15.230424041748023,
+            10.432546234130848
+          ],
+          [
+            16.08265884399413,
+            19.270514526367187,
+            19.092461013793933,
+            17.081269989013663,
+            12.112335987091052
+          ],
+          [
+            18.145391998291,
+            21.44050186157223,
+            21.280475997924803,
+            18.68309165954589,
+            13.620999450683572
+          ],
+          [
+            19.996333007812442,
+            23.52975692749016,
+            23.596975326538082,
+            20.103778152465818,
+            14.602626724243084
+          ],
+          [
+            21.60820228576658,
+            25.1919183349609,
+            25.76568862915039,
+            21.541843643188393,
+            16.10183769226073
+          ],
+          [
+            23.362480049133293,
+            27.08537643432608,
+            28.15810386657714,
+            22.8827821350097,
+            17.433070678710767
+          ],
+          [
+            25.225577430725092,
+            28.52370849609371,
+            30.372146911621076,
+            23.82381240844726,
+            18.712861175537086
+          ],
+          [
+            26.625544967651358,
+            30.16027656555174,
+            32.49975234985349,
+            24.50854393005371,
+            19.443568878173714
+          ],
+          [
+            28.116129837036127,
+            31.295492095947264,
+            34.53191467285156,
+            25.490360641479374,
+            20.561679687499943
+          ],
+          [
+            29.457272415161103,
+            32.63257324218748,
+            36.42262039184569,
+            26.227766571044913,
+            21.088712310791003
+          ],
+          [
+            30.8263483428955,
+            33.570856246948225,
+            38.26254928588863,
+            26.711691284179675,
+            22.396849746704
+          ],
+          [
+            32.008242797851516,
+            34.67565246582029,
+            40.05519142150879,
+            27.399809188842735,
+            22.793699951171842
+          ],
+          [
+            33.28292778015133,
+            35.806464462280275,
+            41.85474884033201,
+            27.812029418945265,
+            23.47191642761218
+          ]
+        ]
+      }
+    }
+  }
+}

checkpoint-2000/experiment_cfg/final_model_config.json

@@ -0,0 +1,53 @@
+{
+  "model_type": "Gr00tN1d6",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "backbone_model_type": "eagle",
+  "model_revision": null,
+  "tune_top_llm_layers": 4,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": true,
+  "collator_overwrite_image_inputs": false,
+  "eagle_collator": true,
+  "backbone_trainable_params_fp32": true,
+  "apply_sincos_state_encoding": true,
+  "use_relative_action": true,
+  "max_state_dim": 128,
+  "max_action_dim": 128,
+  "action_horizon": 50,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.0,
+  "state_additive_noise_scale": 0.0,
+  "max_num_embodiments": 32
+}

checkpoint-2000/model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efb7418ffa7dcabd9858be0df3f27ccbff8cb2615abe8b53e7786c26f698be4c
+size 4990120184

checkpoint-2000/model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db2ed5815745a9febc81afe19fbc32c5f03de7a9c65d582d41ba874a7302e729
+size 4823190320

checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3299fa1359c27c77bb5210e4b402861412b721a9bf5c7e5e96a969c7034e3a03
+size 12960193762

checkpoint-2000/processor_config.json

@@ -0,0 +1,455 @@
+{
+  "processor_class": "Gr00tN1d6Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "behavior_r1_pro": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "observation.images.rgb.head_256_256",
+            "observation.images.rgb.left_wrist_256_256",
+            "observation.images.rgb.right_wrist_256_256"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "robot_pos",
+            "robot_ori_cos",
+            "robot_ori_sin",
+            "robot_2d_ori",
+            "robot_2d_ori_cos",
+            "robot_2d_ori_sin",
+            "robot_lin_vel",
+            "robot_ang_vel",
+            "arm_left_qpos",
+            "arm_left_qpos_sin",
+            "arm_left_qpos_cos",
+            "eef_left_pos",
+            "eef_left_quat",
+            "gripper_left_qpos",
+            "arm_right_qpos",
+            "arm_right_qpos_sin",
+            "arm_right_qpos_cos",
+            "eef_right_pos",
+            "eef_right_quat",
+            "gripper_right_qpos",
+            "trunk_qpos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31
+          ],
+          "modality_keys": [
+            "base",
+            "torso",
+            "left_arm",
+            "left_gripper",
+            "right_arm",
+            "right_gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "trunk_qpos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_left_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_right_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "gr1": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "ego_view_bg_crop_pad_res256_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "robocasa_panda_omron": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "res256_image_side_0",
+            "res256_image_side_1",
+            "res256_image_wrist_0"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "end_effector_position_relative",
+            "end_effector_rotation_relative",
+            "gripper_qpos",
+            "base_position",
+            "base_rotation"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "end_effector_position",
+            "end_effector_rotation",
+            "gripper_close",
+            "base_motion",
+            "control_mode"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.action.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "front",
+            "wrist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": null,
+    "image_target_size": null,
+    "use_albumentations": true,
+    "random_rotation_angle": null,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "model_name": "nvidia/Eagle-Block2A-2B-v2",
+    "model_type": "eagle",
+    "formalize_language": true,
+    "max_state_dim": 128,
+    "max_action_dim": 128,
+    "max_action_horizon": 50,
+    "use_percentiles": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": true,
+    "use_relative_action": true
+  }
+}

checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5efd8171a93a38e78841ea34d2929fad62621f2e3b4db564fe3f9ead855ddf85
+size 14645

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c7bf0ea49bd9803cab03463db93f95b3262938087679ec5f7931fc0cbfeb1b1
+size 1465

checkpoint-2000/trainer_state.json

@@ -0,0 +1,1234 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.2,
+  "eval_steps": 500,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "grad_norm": 2.416853189468384,
+      "learning_rate": 1.8e-06,
+      "loss": 1.1685,
+      "step": 10
+    },
+    {
+      "grad_norm": 1.0608258247375488,
+      "learning_rate": 3.8e-06,
+      "loss": 1.1645,
+      "step": 20
+    },
+    {
+      "grad_norm": 0.6068993210792542,
+      "learning_rate": 5.8e-06,
+      "loss": 1.1434,
+      "step": 30
+    },
+    {
+      "grad_norm": 0.4165553152561188,
+      "learning_rate": 7.8e-06,
+      "loss": 1.1309,
+      "step": 40
+    },
+    {
+      "grad_norm": 0.5539636015892029,
+      "learning_rate": 9.800000000000001e-06,
+      "loss": 1.1056,
+      "step": 50
+    },
+    {
+      "grad_norm": 0.48063579201698303,
+      "learning_rate": 1.18e-05,
+      "loss": 1.1079,
+      "step": 60
+    },
+    {
+      "grad_norm": 0.4390954077243805,
+      "learning_rate": 1.3800000000000002e-05,
+      "loss": 1.0909,
+      "step": 70
+    },
+    {
+      "grad_norm": 0.39306357502937317,
+      "learning_rate": 1.58e-05,
+      "loss": 1.0837,
+      "step": 80
+    },
+    {
+      "grad_norm": 0.5071999430656433,
+      "learning_rate": 1.78e-05,
+      "loss": 1.0625,
+      "step": 90
+    },
+    {
+      "grad_norm": 0.38793864846229553,
+      "learning_rate": 1.9800000000000004e-05,
+      "loss": 1.0615,
+      "step": 100
+    },
+    {
+      "grad_norm": 0.6121100187301636,
+      "learning_rate": 2.18e-05,
+      "loss": 1.0406,
+      "step": 110
+    },
+    {
+      "grad_norm": 0.6436424851417542,
+      "learning_rate": 2.38e-05,
+      "loss": 1.0538,
+      "step": 120
+    },
+    {
+      "grad_norm": 3.180732011795044,
+      "learning_rate": 2.58e-05,
+      "loss": 0.9922,
+      "step": 130
+    },
+    {
+      "grad_norm": 0.8403401374816895,
+      "learning_rate": 2.7800000000000005e-05,
+      "loss": 0.9564,
+      "step": 140
+    },
+    {
+      "grad_norm": 1.5241442918777466,
+      "learning_rate": 2.98e-05,
+      "loss": 0.8884,
+      "step": 150
+    },
+    {
+      "grad_norm": 1.140098214149475,
+      "learning_rate": 3.18e-05,
+      "loss": 0.8313,
+      "step": 160
+    },
+    {
+      "grad_norm": 1.406286597251892,
+      "learning_rate": 3.38e-05,
+      "loss": 0.7616,
+      "step": 170
+    },
+    {
+      "grad_norm": 2.2860851287841797,
+      "learning_rate": 3.58e-05,
+      "loss": 0.6981,
+      "step": 180
+    },
+    {
+      "grad_norm": 2.2754695415496826,
+      "learning_rate": 3.7800000000000004e-05,
+      "loss": 0.6285,
+      "step": 190
+    },
+    {
+      "grad_norm": 2.531214714050293,
+      "learning_rate": 3.9800000000000005e-05,
+      "loss": 0.4895,
+      "step": 200
+    },
+    {
+      "grad_norm": 2.7056477069854736,
+      "learning_rate": 4.18e-05,
+      "loss": 0.3478,
+      "step": 210
+    },
+    {
+      "grad_norm": 1.7513630390167236,
+      "learning_rate": 4.38e-05,
+      "loss": 0.2509,
+      "step": 220
+    },
+    {
+      "grad_norm": 1.6927777528762817,
+      "learning_rate": 4.58e-05,
+      "loss": 0.1911,
+      "step": 230
+    },
+    {
+      "grad_norm": 2.264531135559082,
+      "learning_rate": 4.78e-05,
+      "loss": 0.2006,
+      "step": 240
+    },
+    {
+      "grad_norm": 1.3290256261825562,
+      "learning_rate": 4.9800000000000004e-05,
+      "loss": 0.1654,
+      "step": 250
+    },
+    {
+      "grad_norm": 1.5439292192459106,
+      "learning_rate": 5.1800000000000005e-05,
+      "loss": 0.172,
+      "step": 260
+    },
+    {
+      "grad_norm": 1.5491207838058472,
+      "learning_rate": 5.380000000000001e-05,
+      "loss": 0.1494,
+      "step": 270
+    },
+    {
+      "grad_norm": 1.1139971017837524,
+      "learning_rate": 5.580000000000001e-05,
+      "loss": 0.143,
+      "step": 280
+    },
+    {
+      "grad_norm": 1.2239474058151245,
+      "learning_rate": 5.7799999999999995e-05,
+      "loss": 0.1281,
+      "step": 290
+    },
+    {
+      "grad_norm": 1.093898057937622,
+      "learning_rate": 5.9800000000000003e-05,
+      "loss": 0.1234,
+      "step": 300
+    },
+    {
+      "grad_norm": 1.1576344966888428,
+      "learning_rate": 6.18e-05,
+      "loss": 0.1147,
+      "step": 310
+    },
+    {
+      "grad_norm": 1.2395023107528687,
+      "learning_rate": 6.38e-05,
+      "loss": 0.1143,
+      "step": 320
+    },
+    {
+      "grad_norm": 1.2169476747512817,
+      "learning_rate": 6.58e-05,
+      "loss": 0.1144,
+      "step": 330
+    },
+    {
+      "grad_norm": 0.9942442774772644,
+      "learning_rate": 6.780000000000001e-05,
+      "loss": 0.1152,
+      "step": 340
+    },
+    {
+      "grad_norm": 1.0080304145812988,
+      "learning_rate": 6.98e-05,
+      "loss": 0.1081,
+      "step": 350
+    },
+    {
+      "grad_norm": 1.1731500625610352,
+      "learning_rate": 7.18e-05,
+      "loss": 0.1146,
+      "step": 360
+    },
+    {
+      "grad_norm": 0.8233430981636047,
+      "learning_rate": 7.38e-05,
+      "loss": 0.1135,
+      "step": 370
+    },
+    {
+      "grad_norm": 1.0143144130706787,
+      "learning_rate": 7.58e-05,
+      "loss": 0.0979,
+      "step": 380
+    },
+    {
+      "grad_norm": 0.8134552240371704,
+      "learning_rate": 7.780000000000001e-05,
+      "loss": 0.1075,
+      "step": 390
+    },
+    {
+      "grad_norm": 0.736839234828949,
+      "learning_rate": 7.98e-05,
+      "loss": 0.1023,
+      "step": 400
+    },
+    {
+      "grad_norm": 0.8744317889213562,
+      "learning_rate": 8.18e-05,
+      "loss": 0.1093,
+      "step": 410
+    },
+    {
+      "grad_norm": 0.7313347458839417,
+      "learning_rate": 8.38e-05,
+      "loss": 0.1022,
+      "step": 420
+    },
+    {
+      "grad_norm": 0.8588263988494873,
+      "learning_rate": 8.58e-05,
+      "loss": 0.0957,
+      "step": 430
+    },
+    {
+      "grad_norm": 0.8712506890296936,
+      "learning_rate": 8.78e-05,
+      "loss": 0.0986,
+      "step": 440
+    },
+    {
+      "grad_norm": 0.5078502893447876,
+      "learning_rate": 8.98e-05,
+      "loss": 0.0941,
+      "step": 450
+    },
+    {
+      "grad_norm": 0.8700484037399292,
+      "learning_rate": 9.180000000000001e-05,
+      "loss": 0.0968,
+      "step": 460
+    },
+    {
+      "grad_norm": 0.8209111094474792,
+      "learning_rate": 9.38e-05,
+      "loss": 0.1025,
+      "step": 470
+    },
+    {
+      "grad_norm": 0.7114784717559814,
+      "learning_rate": 9.58e-05,
+      "loss": 0.0907,
+      "step": 480
+    },
+    {
+      "grad_norm": 0.8576576709747314,
+      "learning_rate": 9.78e-05,
+      "loss": 0.085,
+      "step": 490
+    },
+    {
+      "grad_norm": 0.7132489085197449,
+      "learning_rate": 9.98e-05,
+      "loss": 0.0826,
+      "step": 500
+    },
+    {
+      "grad_norm": 0.5286405682563782,
+      "learning_rate": 9.9999778549206e-05,
+      "loss": 0.0773,
+      "step": 510
+    },
+    {
+      "grad_norm": 0.6641912460327148,
+      "learning_rate": 9.999901304280685e-05,
+      "loss": 0.0832,
+      "step": 520
+    },
+    {
+      "grad_norm": 0.6113771200180054,
+      "learning_rate": 9.999770075521164e-05,
+      "loss": 0.0818,
+      "step": 530
+    },
+    {
+      "grad_norm": 0.6849881410598755,
+      "learning_rate": 9.99958417007713e-05,
+      "loss": 0.0844,
+      "step": 540
+    },
+    {
+      "grad_norm": 0.510468602180481,
+      "learning_rate": 9.999343589981615e-05,
+      "loss": 0.0712,
+      "step": 550
+    },
+    {
+      "grad_norm": 0.4960484802722931,
+      "learning_rate": 9.999048337865568e-05,
+      "loss": 0.0677,
+      "step": 560
+    },
+    {
+      "grad_norm": 0.6222489476203918,
+      "learning_rate": 9.998698416957815e-05,
+      "loss": 0.0677,
+      "step": 570
+    },
+    {
+      "grad_norm": 0.6377440690994263,
+      "learning_rate": 9.998293831085037e-05,
+      "loss": 0.0757,
+      "step": 580
+    },
+    {
+      "grad_norm": 0.6644724011421204,
+      "learning_rate": 9.997834584671719e-05,
+      "loss": 0.0721,
+      "step": 590
+    },
+    {
+      "grad_norm": 0.42810145020484924,
+      "learning_rate": 9.997320682740107e-05,
+      "loss": 0.0741,
+      "step": 600
+    },
+    {
+      "grad_norm": 0.47897064685821533,
+      "learning_rate": 9.996752130910149e-05,
+      "loss": 0.0723,
+      "step": 610
+    },
+    {
+      "grad_norm": 0.512750506401062,
+      "learning_rate": 9.99612893539944e-05,
+      "loss": 0.0773,
+      "step": 620
+    },
+    {
+      "grad_norm": 0.5588595867156982,
+      "learning_rate": 9.995451103023144e-05,
+      "loss": 0.0772,
+      "step": 630
+    },
+    {
+      "grad_norm": 0.6482793092727661,
+      "learning_rate": 9.994718641193928e-05,
+      "loss": 0.071,
+      "step": 640
+    },
+    {
+      "grad_norm": 0.4272823631763458,
+      "learning_rate": 9.993931557921874e-05,
+      "loss": 0.0695,
+      "step": 650
+    },
+    {
+      "grad_norm": 0.5539937019348145,
+      "learning_rate": 9.993089861814402e-05,
+      "loss": 0.0548,
+      "step": 660
+    },
+    {
+      "grad_norm": 0.5200349688529968,
+      "learning_rate": 9.992193562076166e-05,
+      "loss": 0.0649,
+      "step": 670
+    },
+    {
+      "grad_norm": 0.35027024149894714,
+      "learning_rate": 9.991242668508954e-05,
+      "loss": 0.0599,
+      "step": 680
+    },
+    {
+      "grad_norm": 0.4623376429080963,
+      "learning_rate": 9.990237191511587e-05,
+      "loss": 0.0593,
+      "step": 690
+    },
+    {
+      "grad_norm": 0.42851006984710693,
+      "learning_rate": 9.989177142079802e-05,
+      "loss": 0.0518,
+      "step": 700
+    },
+    {
+      "grad_norm": 0.5397882461547852,
+      "learning_rate": 9.988062531806126e-05,
+      "loss": 0.0614,
+      "step": 710
+    },
+    {
+      "grad_norm": 0.298032283782959,
+      "learning_rate": 9.986893372879762e-05,
+      "loss": 0.0663,
+      "step": 720
+    },
+    {
+      "grad_norm": 0.6170604228973389,
+      "learning_rate": 9.985669678086443e-05,
+      "loss": 0.0594,
+      "step": 730
+    },
+    {
+      "grad_norm": 0.49378132820129395,
+      "learning_rate": 9.984391460808298e-05,
+      "loss": 0.0707,
+      "step": 740
+    },
+    {
+      "grad_norm": 0.5525211095809937,
+      "learning_rate": 9.983058735023709e-05,
+      "loss": 0.0643,
+      "step": 750
+    },
+    {
+      "grad_norm": 0.4275728166103363,
+      "learning_rate": 9.98167151530715e-05,
+      "loss": 0.0632,
+      "step": 760
+    },
+    {
+      "grad_norm": 0.3775525391101837,
+      "learning_rate": 9.980229816829034e-05,
+      "loss": 0.0595,
+      "step": 770
+    },
+    {
+      "grad_norm": 0.42739713191986084,
+      "learning_rate": 9.978733655355544e-05,
+      "loss": 0.0568,
+      "step": 780
+    },
+    {
+      "grad_norm": 0.4654758870601654,
+      "learning_rate": 9.977183047248464e-05,
+      "loss": 0.0516,
+      "step": 790
+    },
+    {
+      "grad_norm": 0.4031694233417511,
+      "learning_rate": 9.975578009464992e-05,
+      "loss": 0.0581,
+      "step": 800
+    },
+    {
+      "grad_norm": 0.34200575947761536,
+      "learning_rate": 9.97391855955757e-05,
+      "loss": 0.0457,
+      "step": 810
+    },
+    {
+      "grad_norm": 0.3750764727592468,
+      "learning_rate": 9.972204715673669e-05,
+      "loss": 0.0509,
+      "step": 820
+    },
+    {
+      "grad_norm": 0.5093910694122314,
+      "learning_rate": 9.970436496555617e-05,
+      "loss": 0.0489,
+      "step": 830
+    },
+    {
+      "grad_norm": 0.6330306529998779,
+      "learning_rate": 9.968613921540373e-05,
+      "loss": 0.0531,
+      "step": 840
+    },
+    {
+      "grad_norm": 0.44549983739852905,
+      "learning_rate": 9.966737010559326e-05,
+      "loss": 0.0569,
+      "step": 850
+    },
+    {
+      "grad_norm": 0.3454737961292267,
+      "learning_rate": 9.964805784138072e-05,
+      "loss": 0.0497,
+      "step": 860
+    },
+    {
+      "grad_norm": 0.29266422986984253,
+      "learning_rate": 9.962820263396195e-05,
+      "loss": 0.0472,
+      "step": 870
+    },
+    {
+      "grad_norm": 0.38099658489227295,
+      "learning_rate": 9.960780470047033e-05,
+      "loss": 0.0556,
+      "step": 880
+    },
+    {
+      "grad_norm": 0.37069812417030334,
+      "learning_rate": 9.958686426397437e-05,
+      "loss": 0.0464,
+      "step": 890
+    },
+    {
+      "grad_norm": 0.5007408261299133,
+      "learning_rate": 9.956538155347534e-05,
+      "loss": 0.0507,
+      "step": 900
+    },
+    {
+      "grad_norm": 0.45427390933036804,
+      "learning_rate": 9.95433568039047e-05,
+      "loss": 0.048,
+      "step": 910
+    },
+    {
+      "grad_norm": 0.38306254148483276,
+      "learning_rate": 9.952079025612162e-05,
+      "loss": 0.0535,
+      "step": 920
+    },
+    {
+      "grad_norm": 0.6907458901405334,
+      "learning_rate": 9.949768215691022e-05,
+      "loss": 0.0505,
+      "step": 930
+    },
+    {
+      "grad_norm": 0.5054745674133301,
+      "learning_rate": 9.9474032758977e-05,
+      "loss": 0.0551,
+      "step": 940
+    },
+    {
+      "grad_norm": 0.38975587487220764,
+      "learning_rate": 9.944984232094794e-05,
+      "loss": 0.0571,
+      "step": 950
+    },
+    {
+      "grad_norm": 0.4025571644306183,
+      "learning_rate": 9.942511110736584e-05,
+      "loss": 0.0486,
+      "step": 960
+    },
+    {
+      "grad_norm": 0.3891156017780304,
+      "learning_rate": 9.939983938868726e-05,
+      "loss": 0.0408,
+      "step": 970
+    },
+    {
+      "grad_norm": 0.38383904099464417,
+      "learning_rate": 9.93740274412797e-05,
+      "loss": 0.0468,
+      "step": 980
+    },
+    {
+      "grad_norm": 0.45907771587371826,
+      "learning_rate": 9.934767554741846e-05,
+      "loss": 0.0532,
+      "step": 990
+    },
+    {
+      "grad_norm": 0.28416934609413147,
+      "learning_rate": 9.932078399528361e-05,
+      "loss": 0.0454,
+      "step": 1000
+    },
+    {
+      "grad_norm": 0.5906932950019836,
+      "learning_rate": 9.929335307895689e-05,
+      "loss": 0.0627,
+      "step": 1010
+    },
+    {
+      "grad_norm": 0.31994298100471497,
+      "learning_rate": 9.926538309841839e-05,
+      "loss": 0.0447,
+      "step": 1020
+    },
+    {
+      "grad_norm": 0.39826470613479614,
+      "learning_rate": 9.923687435954334e-05,
+      "loss": 0.0452,
+      "step": 1030
+    },
+    {
+      "grad_norm": 0.38305869698524475,
+      "learning_rate": 9.920782717409873e-05,
+      "loss": 0.0491,
+      "step": 1040
+    },
+    {
+      "grad_norm": 0.5647104978561401,
+      "learning_rate": 9.917824185973994e-05,
+      "loss": 0.0485,
+      "step": 1050
+    },
+    {
+      "grad_norm": 0.45855265855789185,
+      "learning_rate": 9.914811874000723e-05,
+      "loss": 0.0486,
+      "step": 1060
+    },
+    {
+      "grad_norm": 0.5201910734176636,
+      "learning_rate": 9.911745814432218e-05,
+      "loss": 0.0474,
+      "step": 1070
+    },
+    {
+      "grad_norm": 0.27793142199516296,
+      "learning_rate": 9.90862604079842e-05,
+      "loss": 0.0476,
+      "step": 1080
+    },
+    {
+      "grad_norm": 0.2942734658718109,
+      "learning_rate": 9.90545258721667e-05,
+      "loss": 0.0467,
+      "step": 1090
+    },
+    {
+      "grad_norm": 0.4430975914001465,
+      "learning_rate": 9.90222548839135e-05,
+      "loss": 0.0472,
+      "step": 1100
+    },
+    {
+      "grad_norm": 0.4262060225009918,
+      "learning_rate": 9.898944779613495e-05,
+      "loss": 0.0484,
+      "step": 1110
+    },
+    {
+      "grad_norm": 0.3938030004501343,
+      "learning_rate": 9.89561049676041e-05,
+      "loss": 0.046,
+      "step": 1120
+    },
+    {
+      "grad_norm": 0.4077751934528351,
+      "learning_rate": 9.89222267629528e-05,
+      "loss": 0.0446,
+      "step": 1130
+    },
+    {
+      "grad_norm": 0.3400469422340393,
+      "learning_rate": 9.888781355266763e-05,
+      "loss": 0.0458,
+      "step": 1140
+    },
+    {
+      "grad_norm": 0.37941423058509827,
+      "learning_rate": 9.885286571308598e-05,
+      "loss": 0.0419,
+      "step": 1150
+    },
+    {
+      "grad_norm": 0.32883521914482117,
+      "learning_rate": 9.881738362639182e-05,
+      "loss": 0.0446,
+      "step": 1160
+    },
+    {
+      "grad_norm": 0.3390735387802124,
+      "learning_rate": 9.878136768061154e-05,
+      "loss": 0.0431,
+      "step": 1170
+    },
+    {
+      "grad_norm": 0.24958841502666473,
+      "learning_rate": 9.874481826960979e-05,
+      "loss": 0.0388,
+      "step": 1180
+    },
+    {
+      "grad_norm": 0.33186545968055725,
+      "learning_rate": 9.870773579308503e-05,
+      "loss": 0.0424,
+      "step": 1190
+    },
+    {
+      "grad_norm": 0.2618488073348999,
+      "learning_rate": 9.867012065656533e-05,
+      "loss": 0.038,
+      "step": 1200
+    },
+    {
+      "grad_norm": 0.3369114100933075,
+      "learning_rate": 9.863197327140376e-05,
+      "loss": 0.0435,
+      "step": 1210
+    },
+    {
+      "grad_norm": 0.38006556034088135,
+      "learning_rate": 9.859329405477403e-05,
+      "loss": 0.0371,
+      "step": 1220
+    },
+    {
+      "grad_norm": 0.3106013536453247,
+      "learning_rate": 9.855408342966585e-05,
+      "loss": 0.041,
+      "step": 1230
+    },
+    {
+      "grad_norm": 0.5991314053535461,
+      "learning_rate": 9.851434182488033e-05,
+      "loss": 0.0404,
+      "step": 1240
+    },
+    {
+      "grad_norm": 0.4325745701789856,
+      "learning_rate": 9.84740696750253e-05,
+      "loss": 0.04,
+      "step": 1250
+    },
+    {
+      "grad_norm": 0.4575672149658203,
+      "learning_rate": 9.843326742051055e-05,
+      "loss": 0.0446,
+      "step": 1260
+    },
+    {
+      "grad_norm": 0.3128894865512848,
+      "learning_rate": 9.839193550754297e-05,
+      "loss": 0.045,
+      "step": 1270
+    },
+    {
+      "grad_norm": 0.5669920444488525,
+      "learning_rate": 9.835007438812177e-05,
+      "loss": 0.0448,
+      "step": 1280
+    },
+    {
+      "grad_norm": 0.3259505331516266,
+      "learning_rate": 9.830768452003341e-05,
+      "loss": 0.0484,
+      "step": 1290
+    },
+    {
+      "grad_norm": 0.3997496962547302,
+      "learning_rate": 9.826476636684671e-05,
+      "loss": 0.0384,
+      "step": 1300
+    },
+    {
+      "grad_norm": 0.28296664357185364,
+      "learning_rate": 9.822132039790773e-05,
+      "loss": 0.0496,
+      "step": 1310
+    },
+    {
+      "grad_norm": 0.3684375584125519,
+      "learning_rate": 9.817734708833461e-05,
+      "loss": 0.0494,
+      "step": 1320
+    },
+    {
+      "grad_norm": 0.5209456086158752,
+      "learning_rate": 9.813284691901243e-05,
+      "loss": 0.0432,
+      "step": 1330
+    },
+    {
+      "grad_norm": 0.3386315703392029,
+      "learning_rate": 9.808782037658792e-05,
+      "loss": 0.0501,
+      "step": 1340
+    },
+    {
+      "grad_norm": 0.6467092037200928,
+      "learning_rate": 9.804226795346411e-05,
+      "loss": 0.0421,
+      "step": 1350
+    },
+    {
+      "grad_norm": 0.5830236077308655,
+      "learning_rate": 9.799619014779503e-05,
+      "loss": 0.0496,
+      "step": 1360
+    },
+    {
+      "grad_norm": 0.3474409580230713,
+      "learning_rate": 9.794958746348013e-05,
+      "loss": 0.0521,
+      "step": 1370
+    },
+    {
+      "grad_norm": 0.34878355264663696,
+      "learning_rate": 9.790246041015896e-05,
+      "loss": 0.0434,
+      "step": 1380
+    },
+    {
+      "grad_norm": 0.24419234693050385,
+      "learning_rate": 9.785480950320538e-05,
+      "loss": 0.038,
+      "step": 1390
+    },
+    {
+      "grad_norm": 0.3212246894836426,
+      "learning_rate": 9.78066352637221e-05,
+      "loss": 0.0386,
+      "step": 1400
+    },
+    {
+      "grad_norm": 0.39363548159599304,
+      "learning_rate": 9.775793821853488e-05,
+      "loss": 0.0381,
+      "step": 1410
+    },
+    {
+      "grad_norm": 0.31847864389419556,
+      "learning_rate": 9.77087189001868e-05,
+      "loss": 0.0354,
+      "step": 1420
+    },
+    {
+      "grad_norm": 0.38333189487457275,
+      "learning_rate": 9.765897784693243e-05,
+      "loss": 0.037,
+      "step": 1430
+    },
+    {
+      "grad_norm": 0.4181508421897888,
+      "learning_rate": 9.760871560273197e-05,
+      "loss": 0.0343,
+      "step": 1440
+    },
+    {
+      "grad_norm": 0.29742392897605896,
+      "learning_rate": 9.755793271724526e-05,
+      "loss": 0.0397,
+      "step": 1450
+    },
+    {
+      "grad_norm": 0.42186254262924194,
+      "learning_rate": 9.750662974582584e-05,
+      "loss": 0.0405,
+      "step": 1460
+    },
+    {
+      "grad_norm": 0.2908092141151428,
+      "learning_rate": 9.745480724951473e-05,
+      "loss": 0.0383,
+      "step": 1470
+    },
+    {
+      "grad_norm": 0.3689759373664856,
+      "learning_rate": 9.740246579503447e-05,
+      "loss": 0.034,
+      "step": 1480
+    },
+    {
+      "grad_norm": 0.3124215602874756,
+      "learning_rate": 9.734960595478284e-05,
+      "loss": 0.0335,
+      "step": 1490
+    },
+    {
+      "grad_norm": 0.3377379775047302,
+      "learning_rate": 9.729622830682657e-05,
+      "loss": 0.0426,
+      "step": 1500
+    },
+    {
+      "grad_norm": 0.26830872893333435,
+      "learning_rate": 9.724233343489504e-05,
+      "loss": 0.0332,
+      "step": 1510
+    },
+    {
+      "grad_norm": 0.3746989965438843,
+      "learning_rate": 9.718792192837396e-05,
+      "loss": 0.0415,
+      "step": 1520
+    },
+    {
+      "grad_norm": 0.3922579288482666,
+      "learning_rate": 9.713299438229886e-05,
+      "loss": 0.0394,
+      "step": 1530
+    },
+    {
+      "grad_norm": 0.2619168758392334,
+      "learning_rate": 9.707755139734855e-05,
+      "loss": 0.0379,
+      "step": 1540
+    },
+    {
+      "grad_norm": 0.2860689163208008,
+      "learning_rate": 9.702159357983866e-05,
+      "loss": 0.0333,
+      "step": 1550
+    },
+    {
+      "grad_norm": 0.47750091552734375,
+      "learning_rate": 9.696512154171492e-05,
+      "loss": 0.0427,
+      "step": 1560
+    },
+    {
+      "grad_norm": 0.335984468460083,
+      "learning_rate": 9.690813590054645e-05,
+      "loss": 0.0415,
+      "step": 1570
+    },
+    {
+      "grad_norm": 0.4124501943588257,
+      "learning_rate": 9.685063727951914e-05,
+      "loss": 0.0414,
+      "step": 1580
+    },
+    {
+      "grad_norm": 0.3823433220386505,
+      "learning_rate": 9.679262630742865e-05,
+      "loss": 0.0377,
+      "step": 1590
+    },
+    {
+      "grad_norm": 0.361517071723938,
+      "learning_rate": 9.673410361867373e-05,
+      "loss": 0.0394,
+      "step": 1600
+    },
+    {
+      "grad_norm": 0.33740976452827454,
+      "learning_rate": 9.667506985324909e-05,
+      "loss": 0.0393,
+      "step": 1610
+    },
+    {
+      "grad_norm": 0.3104957342147827,
+      "learning_rate": 9.661552565673855e-05,
+      "loss": 0.0323,
+      "step": 1620
+    },
+    {
+      "grad_norm": 0.34105607867240906,
+      "learning_rate": 9.655547168030789e-05,
+      "loss": 0.034,
+      "step": 1630
+    },
+    {
+      "grad_norm": 0.33195221424102783,
+      "learning_rate": 9.649490858069777e-05,
+      "loss": 0.0364,
+      "step": 1640
+    },
+    {
+      "grad_norm": 0.5268750190734863,
+      "learning_rate": 9.643383702021658e-05,
+      "loss": 0.0418,
+      "step": 1650
+    },
+    {
+      "grad_norm": 0.32975631952285767,
+      "learning_rate": 9.637225766673307e-05,
+      "loss": 0.0369,
+      "step": 1660
+    },
+    {
+      "grad_norm": 0.25505009293556213,
+      "learning_rate": 9.631017119366922e-05,
+      "loss": 0.0392,
+      "step": 1670
+    },
+    {
+      "grad_norm": 0.4074243903160095,
+      "learning_rate": 9.624757827999273e-05,
+      "loss": 0.0355,
+      "step": 1680
+    },
+    {
+      "grad_norm": 0.37926310300827026,
+      "learning_rate": 9.618447961020971e-05,
+      "loss": 0.0465,
+      "step": 1690
+    },
+    {
+      "grad_norm": 0.3103801906108856,
+      "learning_rate": 9.612087587435707e-05,
+      "loss": 0.0381,
+      "step": 1700
+    },
+    {
+      "grad_norm": 0.27850306034088135,
+      "learning_rate": 9.605676776799508e-05,
+      "loss": 0.0345,
+      "step": 1710
+    },
+    {
+      "grad_norm": 0.41181617975234985,
+      "learning_rate": 9.599215599219973e-05,
+      "loss": 0.036,
+      "step": 1720
+    },
+    {
+      "grad_norm": 0.29745417833328247,
+      "learning_rate": 9.592704125355505e-05,
+      "loss": 0.0359,
+      "step": 1730
+    },
+    {
+      "grad_norm": 0.25739115476608276,
+      "learning_rate": 9.586142426414538e-05,
+      "loss": 0.0322,
+      "step": 1740
+    },
+    {
+      "grad_norm": 0.3614145517349243,
+      "learning_rate": 9.57953057415476e-05,
+      "loss": 0.0386,
+      "step": 1750
+    },
+    {
+      "grad_norm": 0.3207153379917145,
+      "learning_rate": 9.572868640882328e-05,
+      "loss": 0.0419,
+      "step": 1760
+    },
+    {
+      "grad_norm": 0.2426140457391739,
+      "learning_rate": 9.56615669945108e-05,
+      "loss": 0.0318,
+      "step": 1770
+    },
+    {
+      "grad_norm": 0.3300987184047699,
+      "learning_rate": 9.55939482326173e-05,
+      "loss": 0.0299,
+      "step": 1780
+    },
+    {
+      "grad_norm": 0.29514288902282715,
+      "learning_rate": 9.552583086261069e-05,
+      "loss": 0.0323,
+      "step": 1790
+    },
+    {
+      "grad_norm": 0.2129458338022232,
+      "learning_rate": 9.545721562941168e-05,
+      "loss": 0.0397,
+      "step": 1800
+    },
+    {
+      "grad_norm": 0.1863127201795578,
+      "learning_rate": 9.538810328338543e-05,
+      "loss": 0.0417,
+      "step": 1810
+    },
+    {
+      "grad_norm": 0.2662620544433594,
+      "learning_rate": 9.531849458033349e-05,
+      "loss": 0.0334,
+      "step": 1820
+    },
+    {
+      "grad_norm": 0.3173840343952179,
+      "learning_rate": 9.524839028148547e-05,
+      "loss": 0.0302,
+      "step": 1830
+    },
+    {
+      "grad_norm": 0.2131727635860443,
+      "learning_rate": 9.517779115349077e-05,
+      "loss": 0.0291,
+      "step": 1840
+    },
+    {
+      "grad_norm": 0.21706818044185638,
+      "learning_rate": 9.510669796841014e-05,
+      "loss": 0.0275,
+      "step": 1850
+    },
+    {
+      "grad_norm": 0.3188416361808777,
+      "learning_rate": 9.503511150370727e-05,
+      "loss": 0.0308,
+      "step": 1860
+    },
+    {
+      "grad_norm": 0.29916301369667053,
+      "learning_rate": 9.496303254224024e-05,
+      "loss": 0.0327,
+      "step": 1870
+    },
+    {
+      "grad_norm": 0.35951587557792664,
+      "learning_rate": 9.489046187225306e-05,
+      "loss": 0.0373,
+      "step": 1880
+    },
+    {
+      "grad_norm": 0.3958148956298828,
+      "learning_rate": 9.481740028736692e-05,
+      "loss": 0.0336,
+      "step": 1890
+    },
+    {
+      "grad_norm": 0.24587172269821167,
+      "learning_rate": 9.474384858657164e-05,
+      "loss": 0.0322,
+      "step": 1900
+    },
+    {
+      "grad_norm": 0.3606860339641571,
+      "learning_rate": 9.466980757421679e-05,
+      "loss": 0.041,
+      "step": 1910
+    },
+    {
+      "grad_norm": 0.36055126786231995,
+      "learning_rate": 9.459527806000305e-05,
+      "loss": 0.0343,
+      "step": 1920
+    },
+    {
+      "grad_norm": 0.26927992701530457,
+      "learning_rate": 9.452026085897325e-05,
+      "loss": 0.0293,
+      "step": 1930
+    },
+    {
+      "grad_norm": 0.31135281920433044,
+      "learning_rate": 9.444475679150348e-05,
+      "loss": 0.0349,
+      "step": 1940
+    },
+    {
+      "grad_norm": 0.36437752842903137,
+      "learning_rate": 9.436876668329411e-05,
+      "loss": 0.0317,
+      "step": 1950
+    },
+    {
+      "grad_norm": 0.30771905183792114,
+      "learning_rate": 9.429229136536079e-05,
+      "loss": 0.0307,
+      "step": 1960
+    },
+    {
+      "grad_norm": 0.3096272945404053,
+      "learning_rate": 9.421533167402534e-05,
+      "loss": 0.0265,
+      "step": 1970
+    },
+    {
+      "grad_norm": 0.31442776322364807,
+      "learning_rate": 9.413788845090666e-05,
+      "loss": 0.033,
+      "step": 1980
+    },
+    {
+      "grad_norm": 0.3305257558822632,
+      "learning_rate": 9.405996254291136e-05,
+      "loss": 0.0313,
+      "step": 1990
+    },
+    {
+      "grad_norm": 0.2100379914045334,
+      "learning_rate": 9.398155480222474e-05,
+      "loss": 0.0352,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 10000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 9223372036854775807,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 36,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4524ac143a9f2191e97d0434f3b2f8c78c2ac53e3116f1e79b416f15557f3fa0
+size 5713

checkpoint-2000/wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d6", "run_id": "so100_finetune"}

experiment_cfg/conf.yaml

@@ -0,0 +1,209 @@
+load_config_path: null
+model:
+  model_type: Gr00tN1d6
+  model_dtype: bfloat16
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  backbone_model_type: eagle
+  model_revision: null
+  tune_top_llm_layers: 4
+  backbone_embedding_dim: 2048
+  tune_llm: false
+  tune_visual: false
+  select_layer: 16
+  reproject_vision: false
+  use_flash_attention: true
+  load_bf16: false
+  collator_overwrite_image_inputs: false
+  eagle_collator: true
+  backbone_trainable_params_fp32: true
+  image_crop_size: null
+  image_target_size: null
+  shortest_image_edge: 256
+  crop_fraction: 0.95
+  random_rotation_angle: null
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    saturation: 0.5
+    hue: 0.08
+  use_albumentations_transforms: true
+  formalize_language: true
+  apply_sincos_state_encoding: false
+  use_relative_action: true
+  max_state_dim: 29
+  max_action_dim: 29
+  action_horizon: 16
+  hidden_size: 1024
+  input_embedding_dim: 1536
+  add_pos_embed: true
+  attn_dropout: 0.2
+  use_vlln: true
+  max_seq_len: 1024
+  use_alternate_vl_dit: true
+  attend_text_every_n_blocks: 2
+  diffusion_model_cfg:
+    positional_embeddings: null
+    num_layers: 32
+    num_attention_heads: 32
+    attention_head_dim: 48
+    norm_type: ada_norm
+    dropout: 0.2
+    final_dropout: true
+    output_dim: 1024
+    interleave_self_attention: true
+  num_inference_timesteps: 4
+  noise_beta_alpha: 1.5
+  noise_beta_beta: 1.0
+  noise_s: 0.999
+  num_timestep_buckets: 1000
+  tune_projector: true
+  tune_diffusion_model: true
+  tune_vlln: true
+  state_dropout_prob: 0.0
+  state_additive_noise_scale: 0.0
+  max_num_embodiments: 32
+data:
+  datasets:
+  - dataset_paths:
+    - /content/dataset/azazdeaz/record-test/azazdeaz/record-test/
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    dataset_type: physical_embodiment
+    val_dataset_path: null
+  modality_configs:
+    new_embodiment:
+      video:
+        delta_indices:
+        - 0
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      state:
+        delta_indices:
+        - 0
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+      action:
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs:
+        - rep: RELATIVE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+        - rep: ABSOLUTE
+          type: NON_EEF
+          format: DEFAULT
+          state_key: null
+      language:
+        delta_indices:
+        - 0
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+        mean_std_embedding_keys: null
+        action_configs: null
+  download_cache: false
+  shard_size: 1024
+  episode_sampling_rate: 0.1
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  mode: single_turn
+  random_chop: 0.0
+  mock_dataset_mode: false
+  shuffle: true
+  seed: 42
+  multiprocessing_context: fork
+  allow_padding: false
+  subsample_ratio: 1.0
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  video_backend: torchcodec
+training:
+  output_dir: /content/so100_finetune
+  experiment_name: null
+  max_steps: 10000
+  global_batch_size: 36
+  batch_size: null
+  gradient_accumulation_steps: 1
+  learning_rate: 0.0001
+  lr_scheduler_type: cosine
+  weight_decay: 1.0e-05
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  max_grad_norm: 1.0
+  optim: adamw_torch
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  fp16: false
+  bf16: true
+  eval_bf16: true
+  logging_steps: 10
+  save_steps: 1000
+  save_total_limit: 5
+  save_vl_model: false
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  max_concurrent_uploads: 2
+  eval_strategy: 'no'
+  eval_steps: 500
+  eval_set_split_ratio: 0.1
+  eval_batch_size: 2
+  save_best_eval_metric_name: ''
+  save_best_eval_metric_greater_is_better: true
+  deepspeed_stage: 2
+  gradient_checkpointing: false
+  transformers_trust_remote_code: true
+  transformers_local_files_only: false
+  transformers_cache_dir: null
+  transformers_access_token: null
+  use_ddp: false
+  ddp_bucket_cap_mb: 100
+  num_gpus: 1
+  dataloader_num_workers: 4
+  remove_unused_columns: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  enable_profiling: false
+  max_retries: 3
+  assert_loss_less_than: null
+  add_rl_callback: false
+  enable_open_loop_eval: false
+  open_loop_eval_traj_ids:
+  - 0
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_plot_indices: null
+max_steps: 10000
+save_steps: 1000

experiment_cfg/config.yaml

@@ -0,0 +1,243 @@
+!!python/object:gr00t.configs.base_config.Config
+data: !!python/object:gr00t.configs.data.data_config.DataConfig
+  allow_padding: false
+  datasets:
+  - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
+    dataset_paths:
+    - /content/dataset/azazdeaz/record-test/azazdeaz/record-test/
+    dataset_type: physical_embodiment
+    embodiment_tag: new_embodiment
+    mix_ratio: 1.0
+    val_dataset_path: null
+  download_cache: false
+  episode_sampling_rate: 0.1
+  image_crop_size:
+  - 244
+  - 244
+  image_target_size:
+  - 224
+  - 224
+  mock_dataset_mode: false
+  modality_configs:
+    new_embodiment:
+      action: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs:
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
+          - default
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - relative
+          state_key: null
+          type: &id002 !!python/object/apply:gr00t.data.types.ActionType
+          - non_eef
+        - !!python/object:gr00t.data.types.ActionConfig
+          format: *id001
+          rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
+          - absolute
+          state_key: null
+          type: *id002
+        delta_indices:
+        - 0
+        - 1
+        - 2
+        - 3
+        - 4
+        - 5
+        - 6
+        - 7
+        - 8
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+        - 14
+        - 15
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      language: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - annotation.human.task_description
+        sin_cos_embedding_keys: null
+      state: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - single_arm
+        - gripper
+        sin_cos_embedding_keys: null
+      video: !!python/object:gr00t.data.types.ModalityConfig
+        action_configs: null
+        delta_indices:
+        - 0
+        mean_std_embedding_keys: null
+        modality_keys:
+        - front
+        - wrist
+        sin_cos_embedding_keys: null
+  mode: single_turn
+  multiprocessing_context: fork
+  num_shards_per_epoch: 100000
+  override_pretraining_statistics: false
+  random_chop: 0.0
+  seed: 42
+  shard_size: 1024
+  shuffle: true
+  subsample_ratio: 1.0
+  video_backend: torchcodec
+load_config_path: null
+model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
+  _attn_implementation_autoset: false
+  _attn_implementation_internal: null
+  _commit_hash: null
+  _name_or_path: ''
+  add_cross_attention: false
+  architectures: null
+  backbone_model_type: eagle
+  backbone_trainable_params_fp32: true
+  bad_words_ids: null
+  begin_suppress_tokens: null
+  bos_token_id: null
+  chunk_size_feed_forward: 0
+  color_jitter_params:
+    brightness: 0.3
+    contrast: 0.4
+    hue: 0.08
+    saturation: 0.5
+  cross_attention_hidden_size: null
+  decoder_start_token_id: null
+  diffusion_model_cfg:
+    attention_head_dim: 48
+    dropout: 0.2
+    final_dropout: true
+    interleave_self_attention: true
+    norm_type: ada_norm
+    num_attention_heads: 32
+    num_layers: 32
+    output_dim: 1024
+    positional_embeddings: null
+  diversity_penalty: 0.0
+  do_sample: false
+  eagle_collator: true
+  early_stopping: false
+  encoder_no_repeat_ngram_size: 0
+  eos_token_id: null
+  exponential_decay_length_penalty: null
+  finetuning_task: null
+  forced_bos_token_id: null
+  forced_eos_token_id: null
+  id2label:
+    0: LABEL_0
+    1: LABEL_1
+  is_decoder: false
+  is_encoder_decoder: false
+  label2id:
+    LABEL_0: 0
+    LABEL_1: 1
+  length_penalty: 1.0
+  load_bf16: false
+  max_length: 20
+  min_length: 0
+  model_name: nvidia/Eagle-Block2A-2B-v2
+  no_repeat_ngram_size: 0
+  num_beam_groups: 1
+  num_beams: 1
+  num_return_sequences: 1
+  output_attentions: false
+  output_hidden_states: false
+  output_scores: false
+  pad_token_id: null
+  prefix: null
+  problem_type: null
+  pruned_heads: {}
+  random_rotation_angle: null
+  remove_invalid_values: false
+  repetition_penalty: 1.0
+  reproject_vision: false
+  return_dict: true
+  return_dict_in_generate: false
+  sep_token_id: null
+  state_dropout_prob: 0.0
+  suppress_tokens: null
+  task_specific_params: null
+  temperature: 1.0
+  tf_legacy_loss: false
+  tie_encoder_decoder: false
+  tie_word_embeddings: true
+  tokenizer_class: null
+  top_k: 50
+  top_p: 1.0
+  torch_dtype: null
+  torchscript: false
+  transformers_version: null
+  tune_diffusion_model: true
+  tune_llm: false
+  tune_projector: true
+  tune_visual: false
+  typical_p: 1.0
+  use_bfloat16: false
+  use_relative_action: true
+training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
+  add_rl_callback: false
+  assert_loss_less_than: null
+  batch_size: null
+  bf16: true
+  dataloader_num_workers: 4
+  ddp_bucket_cap_mb: 100
+  deepspeed_stage: 2
+  enable_open_loop_eval: false
+  enable_profiling: false
+  eval_batch_size: 2
+  eval_bf16: true
+  eval_set_split_ratio: 0.1
+  eval_steps: 500
+  eval_strategy: 'no'
+  experiment_name: null
+  fp16: false
+  global_batch_size: 36
+  gradient_accumulation_steps: 1
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 10
+  lr_scheduler_type: cosine
+  max_concurrent_uploads: 2
+  max_grad_norm: 1.0
+  max_retries: 3
+  max_steps: 10000
+  num_gpus: 1
+  open_loop_eval_plot_indices: null
+  open_loop_eval_steps_per_traj: 100
+  open_loop_eval_traj_ids:
+  - 0
+  optim: adamw_torch
+  output_dir: /content/so100_finetune
+  remove_unused_columns: false
+  save_best_eval_metric_greater_is_better: true
+  save_best_eval_metric_name: ''
+  save_steps: 1000
+  save_total_limit: 5
+  save_vl_model: false
+  start_from_checkpoint: nvidia/GR00T-N1.6-3B
+  tf32: true
+  transformers_access_token: null
+  transformers_cache_dir: null
+  transformers_local_files_only: false
+  transformers_trust_remote_code: true
+  upload_checkpoints: false
+  upload_every: 1000
+  upload_last_n_checkpoints: 5
+  use_ddp: false
+  use_wandb: true
+  wandb_project: finetune-gr00t-n1d6
+  warmup_ratio: 0.05
+  warmup_steps: 0
+  weight_decay: 1.0e-05

experiment_cfg/dataset_statistics.json

@@ -0,0 +1,824 @@
+{
+  "new_embodiment": {
+    "state": {
+      "single_arm": {
+        "min": [
+          -60.105262756347656,
+          -99.31827545166016,
+          -43.24324417114258,
+          44.688323974609375,
+          -80.68241119384766
+        ],
+        "max": [
+          34.31578826904297,
+          45.80315399169922,
+          99.62721252441406,
+          97.62950134277344,
+          -4.8293962478637695
+        ],
+        "mean": [
+          -3.5325422842834118,
+          -39.55833784368612,
+          47.61425589641663,
+          68.3154743394685,
+          -46.70357059936142
+        ],
+        "std": [
+          27.459571708144434,
+          41.074676874488844,
+          34.075122244859095,
+          13.351843152336091,
+          19.483248111140814
+        ],
+        "q01": [
+          -43.89320641903556,
+          -99.31827545166016,
+          2.96704117550838,
+          52.128458407208086,
+          -64.9047852897644
+        ],
+        "q99": [
+          23.91888349814585,
+          10.271718478515513,
+          99.48718710059721,
+          91.53274853632244,
+          -26.5701696578577
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.9803921580314636
+        ],
+        "max": [
+          35.9943962097168
+        ],
+        "mean": [
+          20.298597213926165
+        ],
+        "std": [
+          9.780378321306017
+        ],
+        "q01": [
+          6.1409783557640525
+        ],
+        "q99": [
+          29.83243577963609
+        ]
+      }
+    },
+    "action": {
+      "single_arm": {
+        "min": [
+          -60.843040466308594,
+          -100.0,
+          -47.45454406738281,
+          44.675209045410156,
+          -81.04986572265625
+        ],
+        "max": [
+          34.99722671508789,
+          45.694801330566406,
+          100.0,
+          97.96730041503906,
+          -4.566929340362549
+        ],
+        "mean": [
+          -3.4051861822555503,
+          -40.21667093678776,
+          46.219495035627304,
+          68.11321517887163,
+          -46.72253879798044
+        ],
+        "std": [
+          27.604988373472494,
+          40.826917813818575,
+          34.82453909741334,
+          13.439408862929378,
+          19.518525610689558
+        ],
+        "q01": [
+          -44.29914832538184,
+          -99.9431293020161,
+          -0.6294070457600729,
+          51.77255040025036,
+          -65.3110059408939
+        ],
+        "q99": [
+          24.517798066878782,
+          10.287138016986633,
+          99.9985524069893,
+          91.79085298874445,
+          -26.382156973884847
+        ]
+      },
+      "gripper": {
+        "min": [
+          0.25553661584854126
+        ],
+        "max": [
+          36.28620147705078
+        ],
+        "mean": [
+          19.60203928355869
+        ],
+        "std": [
+          10.896771214917207
+        ],
+        "q01": [
+          2.708348265017052
+        ],
+        "q99": [
+          30.222431877944597
+        ]
+      }
+    },
+    "relative_action": {
+      "single_arm": {
+        "min": [
+          [
+            -9.055549621582031,
+            -13.95650863647461,
+            -14.800384521484375,
+            -13.61447525024414,
+            -7.874013900756836
+          ],
+          [
+            -11.160816192626953,
+            -16.68364715576172,
+            -17.25493621826172,
+            -16.177448272705078,
+            -9.501310348510742
+          ],
+          [
+            -13.476604461669922,
+            -19.667449951171875,
+            -19.89129638671875,
+            -18.372356414794922,
+            -11.23359489440918
+          ],
+          [
+            -15.26607894897461,
+            -22.73650360107422,
+            -22.314407348632812,
+            -20.567264556884766,
+            -12.755905151367188
+          ],
+          [
+            -16.950286865234375,
+            -25.550708770751953,
+            -24.457931518554688,
+            -22.246448516845703,
+            -14.120733261108398
+          ],
+          [
+            -18.84502410888672,
+            -28.619762420654297,
+            -26.764127731323242,
+            -23.572124481201172,
+            -15.538057327270508
+          ],
+          [
+            -20.50891876220703,
+            -31.262561798095703,
+            -29.18724250793457,
+            -24.625682830810547,
+            -16.79789924621582
+          ],
+          [
+            -21.950958251953125,
+            -33.90535354614258,
+            -31.703550338745117,
+            -25.421085357666016,
+            -17.9002628326416
+          ],
+          [
+            -23.351024627685547,
+            -36.545902252197266,
+            -33.976280212402344,
+            -25.947864532470703,
+            -18.89763641357422
+          ],
+          [
+            -24.79306411743164,
+            -39.18869400024414,
+            -36.27018737792969,
+            -26.389755249023438,
+            -19.737533569335938
+          ],
+          [
+            -26.056222915649414,
+            -41.40523910522461,
+            -38.54291534423828,
+            -26.566513061523438,
+            -20.57742691040039
+          ],
+          [
+            -27.2141170501709,
+            -43.45127487182617,
+            -40.633819580078125,
+            -26.654312133789062,
+            -21.732280731201172
+          ],
+          [
+            -28.372011184692383,
+            -45.497310638427734,
+            -42.90654754638672,
+            -26.654312133789062,
+            -22.834644317626953
+          ],
+          [
+            -29.5299072265625,
+            -47.542476654052734,
+            -44.9827995300293,
+            -26.654312133789062,
+            -23.674541473388672
+          ],
+          [
+            -30.687801361083984,
+            -49.50242614746094,
+            -47.19833755493164,
+            -26.654312133789062,
+            -24.304462432861328
+          ],
+          [
+            -31.84569549560547,
+            -51.29353713989258,
+            -49.528255462646484,
+            -26.654312133789062,
+            -25.406824111938477
+          ]
+        ],
+        "max": [
+          [
+            11.800273895263672,
+            13.390182495117188,
+            11.2484130859375,
+            12.953540802001953,
+            8.97637939453125
+          ],
+          [
+            14.462503433227539,
+            16.288734436035156,
+            13.857921600341797,
+            15.763023376464844,
+            10.813648223876953
+          ],
+          [
+            16.902881622314453,
+            19.443038940429688,
+            16.2484130859375,
+            18.39691162109375,
+            12.703411102294922
+          ],
+          [
+            19.40987205505371,
+            22.16992950439453,
+            18.857921600341797,
+            20.783126831054688,
+            14.645668029785156
+          ],
+          [
+            21.850250244140625,
+            24.812728881835938,
+            21.221553802490234,
+            22.815826416015625,
+            16.482940673828125
+          ],
+          [
+            24.290626525878906,
+            27.455524444580078,
+            23.551467895507812,
+            24.671768188476562,
+            18.530181884765625
+          ],
+          [
+            26.50115203857422,
+            30.011974334716797,
+            25.857921600341797,
+            26.515491485595703,
+            20.262466430664062
+          ],
+          [
+            28.71167755126953,
+            32.31278610229492,
+            28.187835693359375,
+            27.920230865478516,
+            22.099735260009766
+          ],
+          [
+            31.132732391357422,
+            34.52932357788086,
+            30.585193634033203,
+            29.34360122680664,
+            23.832019805908203
+          ],
+          [
+            33.24032974243164,
+            36.490108489990234,
+            32.91510772705078,
+            30.757648468017578,
+            25.56430435180664
+          ],
+          [
+            35.237003326416016,
+            38.621395111083984,
+            35.187835693359375,
+            32.26008224487305,
+            27.296588897705078
+          ],
+          [
+            37.0264778137207,
+            40.411678314208984,
+            37.27873992919922,
+            33.66482162475586,
+            28.87139129638672
+          ],
+          [
+            38.815948486328125,
+            42.372459411621094,
+            39.32906723022461,
+            34.99049758911133,
+            30.498687744140625
+          ],
+          [
+            40.147064208984375,
+            43.99224090576172,
+            41.238162994384766,
+            35.6975212097168,
+            31.9160099029541
+          ],
+          [
+            41.552791595458984,
+            45.61201858520508,
+            43.13301467895508,
+            36.0510368347168,
+            33.33333206176758
+          ],
+          [
+            43.0264778137207,
+            47.23110580444336,
+            45.22392654418945,
+            36.402225494384766,
+            34.75065612792969
+          ]
+        ],
+        "mean": [
+          [
+            0.11750347912311554,
+            -0.6653266549110413,
+            -1.4583468437194824,
+            -0.2053278237581253,
+            -0.01863335259258747
+          ],
+          [
+            0.12236570566892624,
+            -0.6657301187515259,
+            -1.4583476781845093,
+            -0.20791052281856537,
+            -0.02185939997434616
+          ],
+          [
+            0.12741880118846893,
+            -0.6660965085029602,
+            -1.4583467245101929,
+            -0.21049359440803528,
+            -0.025085171684622765
+          ],
+          [
+            0.13261470198631287,
+            -0.6664630174636841,
+            -1.4583468437194824,
+            -0.21307627856731415,
+            -0.02831093594431877
+          ],
+          [
+            0.13785837590694427,
+            -0.6668296456336975,
+            -1.4583486318588257,
+            -0.21565881371498108,
+            -0.03153714910149574
+          ],
+          [
+            0.14319713413715363,
+            -0.6671591401100159,
+            -1.4583512544631958,
+            -0.21824198961257935,
+            -0.03474012389779091
+          ],
+          [
+            0.14853604137897491,
+            -0.6674894094467163,
+            -1.458350419998169,
+            -0.22082529962062836,
+            -0.03794342279434204
+          ],
+          [
+            0.15382729470729828,
+            -0.6678190231323242,
+            -1.4583524465560913,
+            -0.22340813279151917,
+            -0.041146621108055115
+          ],
+          [
+            0.15907101333141327,
+            -0.6681490540504456,
+            -1.458353042602539,
+            -0.2259913831949234,
+            -0.04434990882873535
+          ],
+          [
+            0.16426703333854675,
+            -0.668441653251648,
+            -1.458351492881775,
+            -0.22857406735420227,
+            -0.04755344241857529
+          ],
+          [
+            0.1694149672985077,
+            -0.668734610080719,
+            -1.4583535194396973,
+            -0.23115694522857666,
+            -0.05075618624687195
+          ],
+          [
+            0.17465919256210327,
+            -0.6690278649330139,
+            -1.458351731300354,
+            -0.2337394654750824,
+            -0.05395958200097084
+          ],
+          [
+            0.17990276217460632,
+            -0.6693212985992432,
+            -1.4583524465560913,
+            -0.23632188141345978,
+            -0.05716199427843094
+          ],
+          [
+            0.18505056202411652,
+            -0.6696141958236694,
+            -1.4583513736724854,
+            -0.2389044612646103,
+            -0.060365449637174606
+          ],
+          [
+            0.1901988983154297,
+            -0.6699072122573853,
+            -1.4583531618118286,
+            -0.24148696660995483,
+            -0.06356889009475708
+          ],
+          [
+            0.19529956579208374,
+            -0.6702005863189697,
+            -1.4583516120910645,
+            -0.24407006800174713,
+            -0.0667722150683403
+          ]
+        ],
+        "std": [
+          [
+            2.756209135055542,
+            4.048781394958496,
+            4.834221839904785,
+            2.4045844078063965,
+            1.831181526184082
+          ],
+          [
+            3.375519037246704,
+            4.940067768096924,
+            5.704192638397217,
+            2.936358690261841,
+            2.2514567375183105
+          ],
+          [
+            3.9847497940063477,
+            5.8236775398254395,
+            6.570866107940674,
+            3.443351984024048,
+            2.663818359375
+          ],
+          [
+            4.582452774047852,
+            6.696892738342285,
+            7.430098056793213,
+            3.9259438514709473,
+            3.067777633666992
+          ],
+          [
+            5.167651653289795,
+            7.5579352378845215,
+            8.279448509216309,
+            4.385776996612549,
+            3.462792158126831
+          ],
+          [
+            5.740141868591309,
+            8.406184196472168,
+            9.11788558959961,
+            4.825246810913086,
+            3.848816156387329
+          ],
+          [
+            6.2996673583984375,
+            9.240605354309082,
+            9.944042205810547,
+            5.246663570404053,
+            4.225630283355713
+          ],
+          [
+            6.846652507781982,
+            10.06134033203125,
+            10.75770092010498,
+            5.652442932128906,
+            4.593677043914795
+          ],
+          [
+            7.3811469078063965,
+            10.86816120147705,
+            11.558343887329102,
+            6.044579982757568,
+            4.9531779289245605
+          ],
+          [
+            7.903446197509766,
+            11.661355972290039,
+            12.34595012664795,
+            6.424712657928467,
+            5.304342269897461
+          ],
+          [
+            8.414060592651367,
+            12.44128131866455,
+            13.120593070983887,
+            6.794016361236572,
+            5.647256851196289
+          ],
+          [
+            8.913477897644043,
+            13.208165168762207,
+            13.88218879699707,
+            7.153115272521973,
+            5.98213529586792
+          ],
+          [
+            9.402256965637207,
+            13.962782859802246,
+            14.631062507629395,
+            7.5023603439331055,
+            6.309183597564697
+          ],
+          [
+            9.880831718444824,
+            14.70537281036377,
+            15.367119789123535,
+            7.841666221618652,
+            6.628418445587158
+          ],
+          [
+            10.349787712097168,
+            15.43662166595459,
+            16.090578079223633,
+            8.171104431152344,
+            6.940072536468506
+          ],
+          [
+            10.809670448303223,
+            16.15700340270996,
+            16.801616668701172,
+            8.490680694580078,
+            7.244339466094971
+          ]
+        ],
+        "q01": [
+          [
+            -7.149912223815918,
+            -12.172602081298828,
+            -13.191639633178712,
+            -7.083407516479492,
+            -5.301837921142578
+          ],
+          [
+            -8.969949054718018,
+            -14.76386520385742,
+            -15.673493194580079,
+            -8.597449951171875,
+            -6.4955373382568355
+          ],
+          [
+            -10.634601535797119,
+            -17.415445404052733,
+            -18.04468208312988,
+            -9.786780090332032,
+            -7.755381164550781
+          ],
+          [
+            -12.332097396850585,
+            -19.81734550476074,
+            -20.49539176940918,
+            -10.772034606933593,
+            -8.81890007019043
+          ],
+          [
+            -14.054951286315918,
+            -22.659150772094726,
+            -22.971768417358398,
+            -11.810748291015624,
+            -10.076640815734862
+          ],
+          [
+            -15.467545299530029,
+            -24.854450073242187,
+            -25.42631248474121,
+            -12.89973159790039,
+            -11.391075839996338
+          ],
+          [
+            -16.85467258453369,
+            -27.482084159851073,
+            -27.82916431427002,
+            -13.832246704101562,
+            -12.532282905578613
+          ],
+          [
+            -18.258717765808104,
+            -29.645261459350586,
+            -29.891822814941406,
+            -14.627758026123047,
+            -13.543306350708008
+          ],
+          [
+            -19.36220642089844,
+            -31.862525711059572,
+            -32.179530944824215,
+            -15.569841918945311,
+            -14.78740104675293
+          ],
+          [
+            -20.866409225463865,
+            -34.33618782043457,
+            -34.25658073425293,
+            -16.842924346923827,
+            -15.576902465820313
+          ],
+          [
+            -21.95573440551758,
+            -36.528687438964845,
+            -36.38519187927246,
+            -17.423948669433592,
+            -16.82309829711914
+          ],
+          [
+            -23.461913070678712,
+            -38.73653938293457,
+            -38.30317817687988,
+            -17.647796630859375,
+            -17.742782592773438
+          ],
+          [
+            -24.53435989379883,
+            -40.874423828124996,
+            -40.44069320678711,
+            -18.515239715576172,
+            -18.879788017272947
+          ],
+          [
+            -25.933171768188476,
+            -43.23724739074707,
+            -42.29842636108398,
+            -19.954179382324217,
+            -19.801573486328124
+          ],
+          [
+            -27.245162963867188,
+            -45.308877029418944,
+            -43.92444366455078,
+            -20.61133575439453,
+            -20.9133874130249
+          ],
+          [
+            -28.540471458435057,
+            -47.20507698059082,
+            -45.95519866943359,
+            -20.804307861328123,
+            -21.69133777618408
+          ]
+        ],
+        "q99": [
+          [
+            8.69469722747799,
+            10.299373474121074,
+            9.785606613159175,
+            9.179881286621086,
+            6.271916503906227
+          ],
+          [
+            10.615129890441846,
+            12.666508712768518,
+            12.091986007690421,
+            11.210918426513642,
+            7.833070821762051
+          ],
+          [
+            12.4686942863464,
+            14.899181365966779,
+            14.446693725585934,
+            13.182954406738268,
+            9.380576934814396
+          ],
+          [
+            14.271173782348615,
+            17.33157432556147,
+            16.80153594970703,
+            15.230424041748023,
+            10.432546234130848
+          ],
+          [
+            16.08265884399413,
+            19.270514526367187,
+            19.092461013793933,
+            17.081269989013663,
+            12.112335987091052
+          ],
+          [
+            18.145391998291,
+            21.44050186157223,
+            21.280475997924803,
+            18.68309165954589,
+            13.620999450683572
+          ],
+          [
+            19.996333007812442,
+            23.52975692749016,
+            23.596975326538082,
+            20.103778152465818,
+            14.602626724243084
+          ],
+          [
+            21.60820228576658,
+            25.1919183349609,
+            25.76568862915039,
+            21.541843643188393,
+            16.10183769226073
+          ],
+          [
+            23.362480049133293,
+            27.08537643432608,
+            28.15810386657714,
+            22.8827821350097,
+            17.433070678710767
+          ],
+          [
+            25.225577430725092,
+            28.52370849609371,
+            30.372146911621076,
+            23.82381240844726,
+            18.712861175537086
+          ],
+          [
+            26.625544967651358,
+            30.16027656555174,
+            32.49975234985349,
+            24.50854393005371,
+            19.443568878173714
+          ],
+          [
+            28.116129837036127,
+            31.295492095947264,
+            34.53191467285156,
+            25.490360641479374,
+            20.561679687499943
+          ],
+          [
+            29.457272415161103,
+            32.63257324218748,
+            36.42262039184569,
+            26.227766571044913,
+            21.088712310791003
+          ],
+          [
+            30.8263483428955,
+            33.570856246948225,
+            38.26254928588863,
+            26.711691284179675,
+            22.396849746704
+          ],
+          [
+            32.008242797851516,
+            34.67565246582029,
+            40.05519142150879,
+            27.399809188842735,
+            22.793699951171842
+          ],
+          [
+            33.28292778015133,
+            35.806464462280275,
+            41.85474884033201,
+            27.812029418945265,
+            23.47191642761218
+          ]
+        ]
+      }
+    }
+  }
+}

experiment_cfg/final_model_config.json

@@ -0,0 +1,53 @@
+{
+  "model_type": "Gr00tN1d6",
+  "model_dtype": "bfloat16",
+  "model_name": "nvidia/Eagle-Block2A-2B-v2",
+  "backbone_model_type": "eagle",
+  "model_revision": null,
+  "tune_top_llm_layers": 4,
+  "backbone_embedding_dim": 2048,
+  "tune_llm": false,
+  "tune_visual": false,
+  "select_layer": 16,
+  "reproject_vision": false,
+  "use_flash_attention": true,
+  "load_bf16": true,
+  "collator_overwrite_image_inputs": false,
+  "eagle_collator": true,
+  "backbone_trainable_params_fp32": true,
+  "apply_sincos_state_encoding": true,
+  "use_relative_action": true,
+  "max_state_dim": 128,
+  "max_action_dim": 128,
+  "action_horizon": 50,
+  "hidden_size": 1024,
+  "input_embedding_dim": 1536,
+  "add_pos_embed": true,
+  "attn_dropout": 0.2,
+  "use_vlln": true,
+  "max_seq_len": 1024,
+  "use_alternate_vl_dit": true,
+  "attend_text_every_n_blocks": 2,
+  "diffusion_model_cfg": {
+    "attention_head_dim": 48,
+    "dropout": 0.2,
+    "final_dropout": true,
+    "interleave_self_attention": true,
+    "norm_type": "ada_norm",
+    "num_attention_heads": 32,
+    "num_layers": 32,
+    "output_dim": 1024,
+    "positional_embeddings": null
+  },
+  "num_inference_timesteps": 4,
+  "noise_beta_alpha": 1.5,
+  "noise_beta_beta": 1.0,
+  "noise_s": 0.999,
+  "num_timestep_buckets": 1000,
+  "tune_projector": true,
+  "tune_diffusion_model": true,
+  "tune_vlln": true,
+  "state_dropout_prob": 0.0,
+  "state_additive_noise_scale": 0.0,
+  "max_num_embodiments": 32
+}

processor/embodiment_id.json

@@ -0,0 +1,10 @@
+{
+  "robocasa_panda_omron": 13,
+  "gr1": 20,
+  "behavior_r1_pro": 24,
+  "unitree_g1": 8,
+  "oxe_google": 0,
+  "oxe_widowx": 1,
+  "libero_panda": 2,
+  "new_embodiment": 10
+}

processor/processor_config.json

@@ -0,0 +1,455 @@
+{
+  "processor_class": "Gr00tN1d6Processor",
+  "processor_kwargs": {
+    "modality_configs": {
+      "behavior_r1_pro": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "observation.images.rgb.head_256_256",
+            "observation.images.rgb.left_wrist_256_256",
+            "observation.images.rgb.right_wrist_256_256"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "robot_pos",
+            "robot_ori_cos",
+            "robot_ori_sin",
+            "robot_2d_ori",
+            "robot_2d_ori_cos",
+            "robot_2d_ori_sin",
+            "robot_lin_vel",
+            "robot_ang_vel",
+            "arm_left_qpos",
+            "arm_left_qpos_sin",
+            "arm_left_qpos_cos",
+            "eef_left_pos",
+            "eef_left_quat",
+            "gripper_left_qpos",
+            "arm_right_qpos",
+            "arm_right_qpos_sin",
+            "arm_right_qpos_cos",
+            "eef_right_pos",
+            "eef_right_quat",
+            "gripper_right_qpos",
+            "trunk_qpos"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            31
+          ],
+          "modality_keys": [
+            "base",
+            "torso",
+            "left_arm",
+            "left_gripper",
+            "right_arm",
+            "right_gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "trunk_qpos"
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_left_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": "arm_right_qpos"
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.coarse_action"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "gr1": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "ego_view_bg_crop_pad_res256_freq20"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "left_arm",
+            "right_arm",
+            "left_hand",
+            "right_hand",
+            "waist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "task"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "robocasa_panda_omron": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "res256_image_side_0",
+            "res256_image_side_1",
+            "res256_image_wrist_0"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "end_effector_position_relative",
+            "end_effector_rotation_relative",
+            "gripper_qpos",
+            "base_position",
+            "base_rotation"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "end_effector_position",
+            "end_effector_rotation",
+            "gripper_close",
+            "base_motion",
+            "control_mode"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.action.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      },
+      "new_embodiment": {
+        "video": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "front",
+            "wrist"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "state": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        },
+        "action": {
+          "delta_indices": [
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+            12,
+            13,
+            14,
+            15
+          ],
+          "modality_keys": [
+            "single_arm",
+            "gripper"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": [
+            {
+              "rep": "RELATIVE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            },
+            {
+              "rep": "ABSOLUTE",
+              "type": "NON_EEF",
+              "format": "DEFAULT",
+              "state_key": null
+            }
+          ]
+        },
+        "language": {
+          "delta_indices": [
+            0
+          ],
+          "modality_keys": [
+            "annotation.human.task_description"
+          ],
+          "sin_cos_embedding_keys": null,
+          "mean_std_embedding_keys": null,
+          "action_configs": null
+        }
+      }
+    },
+    "image_crop_size": null,
+    "image_target_size": null,
+    "use_albumentations": true,
+    "random_rotation_angle": null,
+    "color_jitter_params": {
+      "brightness": 0.3,
+      "contrast": 0.4,
+      "saturation": 0.5,
+      "hue": 0.08
+    },
+    "shortest_image_edge": 256,
+    "crop_fraction": 0.95,
+    "model_name": "nvidia/Eagle-Block2A-2B-v2",
+    "model_type": "eagle",
+    "formalize_language": true,
+    "max_state_dim": 128,
+    "max_action_dim": 128,
+    "max_action_horizon": 50,
+    "use_percentiles": false,
+    "clip_outliers": true,
+    "apply_sincos_state_encoding": true,
+    "use_relative_action": true
+  }
+}

wandb_config.json

@@ -0,0 +1 @@
+{"project": "finetune-gr00t-n1d6", "run_id": "so100_finetune"}