Hugging Face's logo

Simon531
/
GR00T-SO101-FT

Safetensors
Gr00tN1d6
Model card Files
xet
 Community
Simon531 commited on
Commit
b4c8898
·
verified ·
1 Parent(s): 5859a07

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. experiment_cfg/conf.yaml +209 -0
  2. experiment_cfg/config.yaml +243 -0
  3. experiment_cfg/dataset_statistics.json +824 -0
  4. experiment_cfg/final_model_config.json +53 -0
  5. experiment_cfg/final_processor_config.json +0 -0
experiment_cfg/conf.yaml ADDED
@@ -0,0 +1,209 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ load_config_path: null
2
+ model:
3
+ model_type: Gr00tN1d6
4
+ model_dtype: bfloat16
5
+ model_name: nvidia/Eagle-Block2A-2B-v2
6
+ backbone_model_type: eagle
7
+ model_revision: null
8
+ tune_top_llm_layers: 4
9
+ backbone_embedding_dim: 2048
10
+ tune_llm: false
11
+ tune_visual: false
12
+ select_layer: 16
13
+ reproject_vision: false
14
+ use_flash_attention: true
15
+ load_bf16: false
16
+ collator_overwrite_image_inputs: false
17
+ eagle_collator: true
18
+ backbone_trainable_params_fp32: true
19
+ image_crop_size: null
20
+ image_target_size: null
21
+ shortest_image_edge: 256
22
+ crop_fraction: 0.95
23
+ random_rotation_angle: null
24
+ color_jitter_params:
25
+ brightness: 0.3
26
+ contrast: 0.4
27
+ saturation: 0.5
28
+ hue: 0.08
29
+ use_albumentations_transforms: true
30
+ formalize_language: true
31
+ apply_sincos_state_encoding: false
32
+ use_relative_action: true
33
+ max_state_dim: 29
34
+ max_action_dim: 29
35
+ action_horizon: 16
36
+ hidden_size: 1024
37
+ input_embedding_dim: 1536
38
+ add_pos_embed: true
39
+ attn_dropout: 0.2
40
+ use_vlln: true
41
+ max_seq_len: 1024
42
+ use_alternate_vl_dit: true
43
+ attend_text_every_n_blocks: 2
44
+ diffusion_model_cfg:
45
+ positional_embeddings: null
46
+ num_layers: 32
47
+ num_attention_heads: 32
48
+ attention_head_dim: 48
49
+ norm_type: ada_norm
50
+ dropout: 0.2
51
+ final_dropout: true
52
+ output_dim: 1024
53
+ interleave_self_attention: true
54
+ num_inference_timesteps: 4
55
+ noise_beta_alpha: 1.5
56
+ noise_beta_beta: 1.0
57
+ noise_s: 0.999
58
+ num_timestep_buckets: 1000
59
+ tune_projector: true
60
+ tune_diffusion_model: true
61
+ tune_vlln: true
62
+ state_dropout_prob: 0.0
63
+ state_additive_noise_scale: 0.0
64
+ max_num_embodiments: 32
65
+ data:
66
+ datasets:
67
+ - dataset_paths:
68
+ - demo_data/test_fps30
69
+ embodiment_tag: new_embodiment
70
+ mix_ratio: 1.0
71
+ dataset_type: physical_embodiment
72
+ val_dataset_path: null
73
+ modality_configs:
74
+ new_embodiment:
75
+ video:
76
+ delta_indices:
77
+ - 0
78
+ modality_keys:
79
+ - front
80
+ - wrist
81
+ sin_cos_embedding_keys: null
82
+ mean_std_embedding_keys: null
83
+ action_configs: null
84
+ state:
85
+ delta_indices:
86
+ - 0
87
+ modality_keys:
88
+ - single_arm
89
+ - gripper
90
+ sin_cos_embedding_keys: null
91
+ mean_std_embedding_keys: null
92
+ action_configs: null
93
+ action:
94
+ delta_indices:
95
+ - 0
96
+ - 1
97
+ - 2
98
+ - 3
99
+ - 4
100
+ - 5
101
+ - 6
102
+ - 7
103
+ - 8
104
+ - 9
105
+ - 10
106
+ - 11
107
+ - 12
108
+ - 13
109
+ - 14
110
+ - 15
111
+ modality_keys:
112
+ - single_arm
113
+ - gripper
114
+ sin_cos_embedding_keys: null
115
+ mean_std_embedding_keys: null
116
+ action_configs:
117
+ - rep: RELATIVE
118
+ type: NON_EEF
119
+ format: DEFAULT
120
+ state_key: null
121
+ - rep: ABSOLUTE
122
+ type: NON_EEF
123
+ format: DEFAULT
124
+ state_key: null
125
+ language:
126
+ delta_indices:
127
+ - 0
128
+ modality_keys:
129
+ - annotation.human.task_description
130
+ sin_cos_embedding_keys: null
131
+ mean_std_embedding_keys: null
132
+ action_configs: null
133
+ download_cache: false
134
+ shard_size: 1024
135
+ episode_sampling_rate: 0.1
136
+ num_shards_per_epoch: 100000
137
+ override_pretraining_statistics: false
138
+ mode: single_turn
139
+ random_chop: 0.0
140
+ mock_dataset_mode: false
141
+ shuffle: true
142
+ seed: 42
143
+ multiprocessing_context: fork
144
+ allow_padding: false
145
+ subsample_ratio: 1.0
146
+ image_crop_size:
147
+ - 244
148
+ - 244
149
+ image_target_size:
150
+ - 224
151
+ - 224
152
+ video_backend: torchcodec
153
+ training:
154
+ output_dir: /root/robotics/checkpoints/so101_finetune
155
+ experiment_name: null
156
+ max_steps: 10000
157
+ global_batch_size: 32
158
+ batch_size: null
159
+ gradient_accumulation_steps: 1
160
+ learning_rate: 0.0001
161
+ lr_scheduler_type: cosine
162
+ weight_decay: 1.0e-05
163
+ warmup_ratio: 0.05
164
+ warmup_steps: 0
165
+ max_grad_norm: 1.0
166
+ optim: adamw_torch
167
+ start_from_checkpoint: nvidia/GR00T-N1.6-3B
168
+ tf32: true
169
+ fp16: false
170
+ bf16: true
171
+ eval_bf16: true
172
+ logging_steps: 10
173
+ save_steps: 1000
174
+ save_total_limit: 5
175
+ save_vl_model: false
176
+ upload_checkpoints: false
177
+ upload_every: 1000
178
+ upload_last_n_checkpoints: 5
179
+ max_concurrent_uploads: 2
180
+ eval_strategy: 'no'
181
+ eval_steps: 500
182
+ eval_set_split_ratio: 0.1
183
+ eval_batch_size: 2
184
+ save_best_eval_metric_name: ''
185
+ save_best_eval_metric_greater_is_better: true
186
+ deepspeed_stage: 2
187
+ gradient_checkpointing: false
188
+ transformers_trust_remote_code: true
189
+ transformers_local_files_only: false
190
+ transformers_cache_dir: null
191
+ transformers_access_token: null
192
+ use_ddp: false
193
+ ddp_bucket_cap_mb: 100
194
+ num_gpus: 1
195
+ dataloader_num_workers: 2
196
+ remove_unused_columns: false
197
+ use_wandb: false
198
+ wandb_project: finetune-gr00t-n1d6
199
+ enable_profiling: false
200
+ max_retries: 3
201
+ assert_loss_less_than: null
202
+ add_rl_callback: false
203
+ enable_open_loop_eval: false
204
+ open_loop_eval_traj_ids:
205
+ - 0
206
+ open_loop_eval_steps_per_traj: 100
207
+ open_loop_eval_plot_indices: null
208
+ max_steps: 10000
209
+ save_steps: 1000
experiment_cfg/config.yaml ADDED
@@ -0,0 +1,243 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ !!python/object:gr00t.configs.base_config.Config
2
+ data: !!python/object:gr00t.configs.data.data_config.DataConfig
3
+ allow_padding: false
4
+ datasets:
5
+ - !!python/object:gr00t.configs.data.data_config.SingleDatasetConfig
6
+ dataset_paths:
7
+ - demo_data/test_fps30
8
+ dataset_type: physical_embodiment
9
+ embodiment_tag: new_embodiment
10
+ mix_ratio: 1.0
11
+ val_dataset_path: null
12
+ download_cache: false
13
+ episode_sampling_rate: 0.1
14
+ image_crop_size:
15
+ - 244
16
+ - 244
17
+ image_target_size:
18
+ - 224
19
+ - 224
20
+ mock_dataset_mode: false
21
+ modality_configs:
22
+ new_embodiment:
23
+ action: !!python/object:gr00t.data.types.ModalityConfig
24
+ action_configs:
25
+ - !!python/object:gr00t.data.types.ActionConfig
26
+ format: &id001 !!python/object/apply:gr00t.data.types.ActionFormat
27
+ - default
28
+ rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
29
+ - relative
30
+ state_key: null
31
+ type: &id002 !!python/object/apply:gr00t.data.types.ActionType
32
+ - non_eef
33
+ - !!python/object:gr00t.data.types.ActionConfig
34
+ format: *id001
35
+ rep: !!python/object/apply:gr00t.data.types.ActionRepresentation
36
+ - absolute
37
+ state_key: null
38
+ type: *id002
39
+ delta_indices:
40
+ - 0
41
+ - 1
42
+ - 2
43
+ - 3
44
+ - 4
45
+ - 5
46
+ - 6
47
+ - 7
48
+ - 8
49
+ - 9
50
+ - 10
51
+ - 11
52
+ - 12
53
+ - 13
54
+ - 14
55
+ - 15
56
+ mean_std_embedding_keys: null
57
+ modality_keys:
58
+ - single_arm
59
+ - gripper
60
+ sin_cos_embedding_keys: null
61
+ language: !!python/object:gr00t.data.types.ModalityConfig
62
+ action_configs: null
63
+ delta_indices:
64
+ - 0
65
+ mean_std_embedding_keys: null
66
+ modality_keys:
67
+ - annotation.human.task_description
68
+ sin_cos_embedding_keys: null
69
+ state: !!python/object:gr00t.data.types.ModalityConfig
70
+ action_configs: null
71
+ delta_indices:
72
+ - 0
73
+ mean_std_embedding_keys: null
74
+ modality_keys:
75
+ - single_arm
76
+ - gripper
77
+ sin_cos_embedding_keys: null
78
+ video: !!python/object:gr00t.data.types.ModalityConfig
79
+ action_configs: null
80
+ delta_indices:
81
+ - 0
82
+ mean_std_embedding_keys: null
83
+ modality_keys:
84
+ - front
85
+ - wrist
86
+ sin_cos_embedding_keys: null
87
+ mode: single_turn
88
+ multiprocessing_context: fork
89
+ num_shards_per_epoch: 100000
90
+ override_pretraining_statistics: false
91
+ random_chop: 0.0
92
+ seed: 42
93
+ shard_size: 1024
94
+ shuffle: true
95
+ subsample_ratio: 1.0
96
+ video_backend: torchcodec
97
+ load_config_path: null
98
+ model: !!python/object:gr00t.configs.model.gr00t_n1d6.Gr00tN1d6Config
99
+ _attn_implementation_autoset: false
100
+ _attn_implementation_internal: null
101
+ _commit_hash: null
102
+ _name_or_path: ''
103
+ add_cross_attention: false
104
+ architectures: null
105
+ backbone_model_type: eagle
106
+ backbone_trainable_params_fp32: true
107
+ bad_words_ids: null
108
+ begin_suppress_tokens: null
109
+ bos_token_id: null
110
+ chunk_size_feed_forward: 0
111
+ color_jitter_params:
112
+ brightness: 0.3
113
+ contrast: 0.4
114
+ hue: 0.08
115
+ saturation: 0.5
116
+ cross_attention_hidden_size: null
117
+ decoder_start_token_id: null
118
+ diffusion_model_cfg:
119
+ attention_head_dim: 48
120
+ dropout: 0.2
121
+ final_dropout: true
122
+ interleave_self_attention: true
123
+ norm_type: ada_norm
124
+ num_attention_heads: 32
125
+ num_layers: 32
126
+ output_dim: 1024
127
+ positional_embeddings: null
128
+ diversity_penalty: 0.0
129
+ do_sample: false
130
+ eagle_collator: true
131
+ early_stopping: false
132
+ encoder_no_repeat_ngram_size: 0
133
+ eos_token_id: null
134
+ exponential_decay_length_penalty: null
135
+ finetuning_task: null
136
+ forced_bos_token_id: null
137
+ forced_eos_token_id: null
138
+ id2label:
139
+ 0: LABEL_0
140
+ 1: LABEL_1
141
+ is_decoder: false
142
+ is_encoder_decoder: false
143
+ label2id:
144
+ LABEL_0: 0
145
+ LABEL_1: 1
146
+ length_penalty: 1.0
147
+ load_bf16: false
148
+ max_length: 20
149
+ min_length: 0
150
+ model_name: nvidia/Eagle-Block2A-2B-v2
151
+ no_repeat_ngram_size: 0
152
+ num_beam_groups: 1
153
+ num_beams: 1
154
+ num_return_sequences: 1
155
+ output_attentions: false
156
+ output_hidden_states: false
157
+ output_scores: false
158
+ pad_token_id: null
159
+ prefix: null
160
+ problem_type: null
161
+ pruned_heads: {}
162
+ random_rotation_angle: null
163
+ remove_invalid_values: false
164
+ repetition_penalty: 1.0
165
+ reproject_vision: false
166
+ return_dict: true
167
+ return_dict_in_generate: false
168
+ sep_token_id: null
169
+ state_dropout_prob: 0.0
170
+ suppress_tokens: null
171
+ task_specific_params: null
172
+ temperature: 1.0
173
+ tf_legacy_loss: false
174
+ tie_encoder_decoder: false
175
+ tie_word_embeddings: true
176
+ tokenizer_class: null
177
+ top_k: 50
178
+ top_p: 1.0
179
+ torch_dtype: null
180
+ torchscript: false
181
+ transformers_version: null
182
+ tune_diffusion_model: true
183
+ tune_llm: false
184
+ tune_projector: true
185
+ tune_visual: false
186
+ typical_p: 1.0
187
+ use_bfloat16: false
188
+ use_relative_action: true
189
+ training: !!python/object:gr00t.configs.training.training_config.TrainingConfig
190
+ add_rl_callback: false
191
+ assert_loss_less_than: null
192
+ batch_size: null
193
+ bf16: true
194
+ dataloader_num_workers: 2
195
+ ddp_bucket_cap_mb: 100
196
+ deepspeed_stage: 2
197
+ enable_open_loop_eval: false
198
+ enable_profiling: false
199
+ eval_batch_size: 2
200
+ eval_bf16: true
201
+ eval_set_split_ratio: 0.1
202
+ eval_steps: 500
203
+ eval_strategy: 'no'
204
+ experiment_name: null
205
+ fp16: false
206
+ global_batch_size: 32
207
+ gradient_accumulation_steps: 1
208
+ gradient_checkpointing: false
209
+ learning_rate: 0.0001
210
+ logging_steps: 10
211
+ lr_scheduler_type: cosine
212
+ max_concurrent_uploads: 2
213
+ max_grad_norm: 1.0
214
+ max_retries: 3
215
+ max_steps: 10000
216
+ num_gpus: 1
217
+ open_loop_eval_plot_indices: null
218
+ open_loop_eval_steps_per_traj: 100
219
+ open_loop_eval_traj_ids:
220
+ - 0
221
+ optim: adamw_torch
222
+ output_dir: /root/robotics/checkpoints/so101_finetune
223
+ remove_unused_columns: false
224
+ save_best_eval_metric_greater_is_better: true
225
+ save_best_eval_metric_name: ''
226
+ save_steps: 1000
227
+ save_total_limit: 5
228
+ save_vl_model: false
229
+ start_from_checkpoint: nvidia/GR00T-N1.6-3B
230
+ tf32: true
231
+ transformers_access_token: null
232
+ transformers_cache_dir: null
233
+ transformers_local_files_only: false
234
+ transformers_trust_remote_code: true
235
+ upload_checkpoints: false
236
+ upload_every: 1000
237
+ upload_last_n_checkpoints: 5
238
+ use_ddp: false
239
+ use_wandb: false
240
+ wandb_project: finetune-gr00t-n1d6
241
+ warmup_ratio: 0.05
242
+ warmup_steps: 0
243
+ weight_decay: 1.0e-05
experiment_cfg/dataset_statistics.json ADDED
@@ -0,0 +1,824 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "new_embodiment": {
3
+ "state": {
4
+ "single_arm": {
5
+ "min": [
6
+ -92.8849868774414,
7
+ -99.91742706298828,
8
+ -67.91249084472656,
9
+ 36.91042709350586,
10
+ -24.052331924438477
11
+ ],
12
+ "max": [
13
+ 95.126708984375,
14
+ 58.959537506103516,
15
+ 99.17958068847656,
16
+ 99.48074340820312,
17
+ 53.97517776489258
18
+ ],
19
+ "mean": [
20
+ 10.931507608543503,
21
+ -28.95046560557863,
22
+ 25.570738767073305,
23
+ 74.43297238341502,
24
+ 21.50083724758433
25
+ ],
26
+ "std": [
27
+ 34.69148018485696,
28
+ 54.16032436200401,
29
+ 53.17449461594266,
30
+ 10.341797307222885,
31
+ 9.371549591779347
32
+ ],
33
+ "q01": [
34
+ -22.624184165500633,
35
+ -99.40776729669031,
36
+ -31.21839759746301,
37
+ 59.67779062267487,
38
+ 10.834610455198732
39
+ ],
40
+ "q99": [
41
+ 75.05701412369808,
42
+ 37.91258107003083,
43
+ 96.31979150207671,
44
+ 93.7740276781866,
45
+ 32.91402871074016
46
+ ]
47
+ },
48
+ "gripper": {
49
+ "min": [
50
+ 0.5939124226570129
51
+ ],
52
+ "max": [
53
+ 53.74907302856445
54
+ ],
55
+ "mean": [
56
+ 14.45829335149272
57
+ ],
58
+ "std": [
59
+ 13.322700942014224
60
+ ],
61
+ "q01": [
62
+ 0.7675683326245213
63
+ ],
64
+ "q99": [
65
+ 35.03100730782007
66
+ ]
67
+ }
68
+ },
69
+ "action": {
70
+ "single_arm": {
71
+ "min": [
72
+ -92.92256164550781,
73
+ -100.0,
74
+ -70.09174346923828,
75
+ 35.5710563659668,
76
+ -24.621212005615234
77
+ ],
78
+ "max": [
79
+ 100.0,
80
+ 58.298927307128906,
81
+ 100.0,
82
+ 100.0,
83
+ 54.10353469848633
84
+ ],
85
+ "mean": [
86
+ 11.193427569556206,
87
+ -29.93287122603463,
88
+ 24.12164763572855,
89
+ 74.38122942695422,
90
+ 21.383208426144503
91
+ ],
92
+ "std": [
93
+ 34.745786671720836,
94
+ 53.651103408270856,
95
+ 53.678901499939,
96
+ 10.514202938600853,
97
+ 9.43512510432986
98
+ ],
99
+ "q01": [
100
+ -22.672492966741213,
101
+ -99.76953777785012,
102
+ -33.615861709127095,
103
+ 58.92763227034659,
104
+ 10.350923132809598
105
+ ],
106
+ "q99": [
107
+ 75.729203784993,
108
+ 37.37134504793819,
109
+ 96.04545314615326,
110
+ 94.28331749476017,
111
+ 33.11827648594648
112
+ ]
113
+ },
114
+ "gripper": {
115
+ "min": [
116
+ 0.0
117
+ ],
118
+ "max": [
119
+ 54.02658462524414
120
+ ],
121
+ "mean": [
122
+ 13.455937754916391
123
+ ],
124
+ "std": [
125
+ 13.554266512466993
126
+ ],
127
+ "q01": [
128
+ 0.3240959848558188
129
+ ],
130
+ "q99": [
131
+ 35.38169955279487
132
+ ]
133
+ }
134
+ },
135
+ "relative_action": {
136
+ "single_arm": {
137
+ "min": [
138
+ [
139
+ -13.138093948364258,
140
+ -24.029727935791016,
141
+ -22.93235206604004,
142
+ -11.51348876953125,
143
+ -9.146585464477539
144
+ ],
145
+ [
146
+ -16.451932907104492,
147
+ -27.415359497070312,
148
+ -27.307920455932617,
149
+ -13.431537628173828,
150
+ -11.360607147216797
151
+ ],
152
+ [
153
+ -19.76577377319336,
154
+ -30.80099105834961,
155
+ -31.59233283996582,
156
+ -15.75604248046875,
157
+ -13.443941116333008
158
+ ],
159
+ [
160
+ -22.763275146484375,
161
+ -35.26011657714844,
162
+ -35.81268310546875,
163
+ -18.371559143066406,
164
+ -15.456689834594727
165
+ ],
166
+ [
167
+ -25.711191177368164,
168
+ -38.645748138427734,
169
+ -40.188255310058594,
170
+ -20.89989471435547,
171
+ -17.287498474121094
172
+ ],
173
+ [
174
+ -28.70869255065918,
175
+ -42.03137969970703,
176
+ -44.40843963623047,
177
+ -23.409629821777344,
178
+ -18.964786529541016
179
+ ],
180
+ [
181
+ -31.63266944885254,
182
+ -45.334434509277344,
183
+ -48.628623962402344,
184
+ -25.91936492919922,
185
+ -20.416805267333984
186
+ ],
187
+ [
188
+ -34.38037872314453,
189
+ -48.55491256713867,
190
+ -52.665321350097656,
191
+ -28.256011962890625,
192
+ -21.691547393798828
193
+ ],
194
+ [
195
+ -37.10942459106445,
196
+ -51.6102409362793,
197
+ -56.58509826660156,
198
+ -30.522796630859375,
199
+ -22.827911376953125
200
+ ],
201
+ [
202
+ -39.54607391357422,
203
+ -54.665565490722656,
204
+ -60.32256317138672,
205
+ -32.772911071777344,
206
+ -24.081449508666992
207
+ ],
208
+ [
209
+ -41.877464294433594,
210
+ -57.6383171081543,
211
+ -63.99229049682617,
212
+ -34.93647003173828,
213
+ -25.423280715942383
214
+ ],
215
+ [
216
+ -44.43657684326172,
217
+ -60.4459114074707,
218
+ -67.57026672363281,
219
+ -36.92695617675781,
220
+ -26.611827850341797
221
+ ],
222
+ [
223
+ -47.35081481933594,
224
+ -63.253509521484375,
225
+ -70.85194396972656,
226
+ -38.91743469238281,
227
+ -27.700204849243164
228
+ ],
229
+ [
230
+ -50.17732238769531,
231
+ -65.73080444335938,
232
+ -73.89686584472656,
233
+ -40.821372985839844,
234
+ -28.706575393676758
235
+ ],
236
+ [
237
+ -52.92503356933594,
238
+ -67.8777847290039,
239
+ -77.38311004638672,
240
+ -42.6522331237793,
241
+ -29.56362533569336
242
+ ],
243
+ [
244
+ -55.5062141418457,
245
+ -69.94219970703125,
246
+ -80.57362365722656,
247
+ -44.38308334350586,
248
+ -30.332382202148438
249
+ ]
250
+ ],
251
+ "max": [
252
+ [
253
+ 19.939687728881836,
254
+ 24.938064575195312,
255
+ 20.57772445678711,
256
+ 11.438907623291016,
257
+ 9.896707534790039
258
+ ],
259
+ [
260
+ 24.715517044067383,
261
+ 29.727493286132812,
262
+ 25.83032989501953,
263
+ 13.862098693847656,
264
+ 12.51327896118164
265
+ ],
266
+ [
267
+ 29.686275482177734,
268
+ 34.434349060058594,
269
+ 31.243175506591797,
270
+ 16.01158905029297,
271
+ 15.129850387573242
272
+ ],
273
+ [
274
+ 34.267173767089844,
275
+ 38.89347839355469,
276
+ 36.34800720214844,
277
+ 17.944347381591797,
278
+ 17.679330825805664
279
+ ],
280
+ [
281
+ 38.84806823730469,
282
+ 43.10487365722656,
283
+ 41.39387893676758,
284
+ 19.688030242919922,
285
+ 20.161720275878906
286
+ ],
287
+ [
288
+ 43.13656997680664,
289
+ 47.15111541748047,
290
+ 46.40755081176758,
291
+ 21.245800018310547,
292
+ 22.442832946777344
293
+ ],
294
+ [
295
+ 47.13267135620117,
296
+ 51.032203674316406,
297
+ 51.14775085449219,
298
+ 22.370853424072266,
299
+ 24.72394561767578
300
+ ],
301
+ [
302
+ 50.83637237548828,
303
+ 55.57390594482422,
304
+ 55.73490524291992,
305
+ 23.692794799804688,
306
+ 26.87087631225586
307
+ ],
308
+ [
309
+ 54.76925277709961,
310
+ 59.785301208496094,
311
+ 60.29279327392578,
312
+ 25.250564575195312,
313
+ 29.01734161376953
314
+ ],
315
+ [
316
+ 58.862823486328125,
317
+ 64.07926940917969,
318
+ 64.66836547851562,
319
+ 26.732696533203125,
320
+ 30.962997436523438
321
+ ],
322
+ [
323
+ 62.77622985839844,
324
+ 68.12551879882812,
325
+ 69.11735534667969,
326
+ 28.11224365234375,
327
+ 32.70737838745117
328
+ ],
329
+ [
330
+ 66.67486572265625,
331
+ 72.00660705566406,
332
+ 73.97974395751953,
333
+ 29.1507568359375,
334
+ 34.348793029785156
335
+ ],
336
+ [
337
+ 70.25521087646484,
338
+ 75.8876953125,
339
+ 78.56690216064453,
340
+ 30.02259063720703,
341
+ 35.75771713256836
342
+ ],
343
+ [
344
+ 73.66903686523438,
345
+ 79.76879119873047,
346
+ 82.94247436523438,
347
+ 30.851463317871094,
348
+ 37.02033996582031
349
+ ],
350
+ [
351
+ 77.08033752441406,
352
+ 83.31957244873047,
353
+ 86.97917175292969,
354
+ 31.854061126708984,
355
+ 38.16089630126953
356
+ ],
357
+ [
358
+ 80.32763671875,
359
+ 86.7052001953125,
360
+ 90.9901123046875,
361
+ 32.98745346069336,
362
+ 38.898902893066406
363
+ ]
364
+ ],
365
+ "mean": [
366
+ [
367
+ 0.25989416241645813,
368
+ -0.9993920922279358,
369
+ -1.4722816944122314,
370
+ -0.0647861436009407,
371
+ -0.11705081909894943
372
+ ],
373
+ [
374
+ 0.2596963047981262,
375
+ -0.9994603991508484,
376
+ -1.4724633693695068,
377
+ -0.06435474008321762,
378
+ -0.11709708720445633
379
+ ],
380
+ [
381
+ 0.2594923675060272,
382
+ -0.9995259046554565,
383
+ -1.4726449251174927,
384
+ -0.0639190524816513,
385
+ -0.11713884025812149
386
+ ],
387
+ [
388
+ 0.2592769265174866,
389
+ -0.999592125415802,
390
+ -1.4728251695632935,
391
+ -0.06348296999931335,
392
+ -0.11717879772186279
393
+ ],
394
+ [
395
+ 0.2590578496456146,
396
+ -0.9996652603149414,
397
+ -1.4729840755462646,
398
+ -0.06305010616779327,
399
+ -0.11721114069223404
400
+ ],
401
+ [
402
+ 0.25883495807647705,
403
+ -0.9997338056564331,
404
+ -1.4731240272521973,
405
+ -0.06261725723743439,
406
+ -0.11723517626523972
407
+ ],
408
+ [
409
+ 0.2586185336112976,
410
+ -0.999806821346283,
411
+ -1.4732255935668945,
412
+ -0.06218778342008591,
413
+ -0.11726422607898712
414
+ ],
415
+ [
416
+ 0.2584228515625,
417
+ -0.9998774528503418,
418
+ -1.4733026027679443,
419
+ -0.06174850836396217,
420
+ -0.11728698760271072
421
+ ],
422
+ [
423
+ 0.25823941826820374,
424
+ -0.9999615550041199,
425
+ -1.4733269214630127,
426
+ -0.06129994988441467,
427
+ -0.11730773746967316
428
+ ],
429
+ [
430
+ 0.2580503821372986,
431
+ -1.0000407695770264,
432
+ -1.4732962846755981,
433
+ -0.060855790972709656,
434
+ -0.11732759326696396
435
+ ],
436
+ [
437
+ 0.2578616440296173,
438
+ -1.0001286268234253,
439
+ -1.4731963872909546,
440
+ -0.06041018292307854,
441
+ -0.11734972149133682
442
+ ],
443
+ [
444
+ 0.2576693892478943,
445
+ -1.0002046823501587,
446
+ -1.4730337858200073,
447
+ -0.05996293947100639,
448
+ -0.11736023426055908
449
+ ],
450
+ [
451
+ 0.25746458768844604,
452
+ -1.0002926588058472,
453
+ -1.472779631614685,
454
+ -0.05950615927577019,
455
+ -0.11737081408500671
456
+ ],
457
+ [
458
+ 0.25726476311683655,
459
+ -1.0003970861434937,
460
+ -1.4724504947662354,
461
+ -0.05905091017484665,
462
+ -0.11738365888595581
463
+ ],
464
+ [
465
+ 0.2570723593235016,
466
+ -1.000535249710083,
467
+ -1.472000002861023,
468
+ -0.05858457833528519,
469
+ -0.1174008920788765
470
+ ],
471
+ [
472
+ 0.25689461827278137,
473
+ -1.000710129737854,
474
+ -1.4714038372039795,
475
+ -0.05811181291937828,
476
+ -0.11742182075977325
477
+ ]
478
+ ],
479
+ "std": [
480
+ [
481
+ 2.881397008895874,
482
+ 4.358253479003906,
483
+ 4.0597052574157715,
484
+ 1.785101294517517,
485
+ 1.051520824432373
486
+ ],
487
+ [
488
+ 3.6468863487243652,
489
+ 5.419794082641602,
490
+ 5.050844669342041,
491
+ 2.2164688110351562,
492
+ 1.307966709136963
493
+ ],
494
+ [
495
+ 4.407171249389648,
496
+ 6.475194454193115,
497
+ 6.038036823272705,
498
+ 2.640221118927002,
499
+ 1.559531807899475
500
+ ],
501
+ [
502
+ 5.1601643562316895,
503
+ 7.520181655883789,
504
+ 7.017229080200195,
505
+ 3.0530614852905273,
506
+ 1.8045036792755127
507
+ ],
508
+ [
509
+ 5.904684066772461,
510
+ 8.552570343017578,
511
+ 7.986705303192139,
512
+ 3.453758716583252,
513
+ 2.0423495769500732
514
+ ],
515
+ [
516
+ 6.640153884887695,
517
+ 9.571252822875977,
518
+ 8.945037841796875,
519
+ 3.841803550720215,
520
+ 2.2731847763061523
521
+ ],
522
+ [
523
+ 7.366391658782959,
524
+ 10.575342178344727,
525
+ 9.891847610473633,
526
+ 4.21760892868042,
527
+ 2.4974586963653564
528
+ ],
529
+ [
530
+ 8.083230972290039,
531
+ 11.564557075500488,
532
+ 10.82669734954834,
533
+ 4.581320762634277,
534
+ 2.715571165084839
535
+ ],
536
+ [
537
+ 8.790854454040527,
538
+ 12.538786888122559,
539
+ 11.74976921081543,
540
+ 4.93326997756958,
541
+ 2.9281158447265625
542
+ ],
543
+ [
544
+ 9.489245414733887,
545
+ 13.498032569885254,
546
+ 12.661258697509766,
547
+ 5.274088382720947,
548
+ 3.1353917121887207
549
+ ],
550
+ [
551
+ 10.17880630493164,
552
+ 14.442769050598145,
553
+ 13.561197280883789,
554
+ 5.604133605957031,
555
+ 3.337749719619751
556
+ ],
557
+ [
558
+ 10.859707832336426,
559
+ 15.37321949005127,
560
+ 14.44973087310791,
561
+ 5.923854827880859,
562
+ 3.53542160987854
563
+ ],
564
+ [
565
+ 11.532102584838867,
566
+ 16.289947509765625,
567
+ 15.327227592468262,
568
+ 6.233618259429932,
569
+ 3.7285473346710205
570
+ ],
571
+ [
572
+ 12.196182250976562,
573
+ 17.19304847717285,
574
+ 16.19367790222168,
575
+ 6.533710956573486,
576
+ 3.91733717918396
577
+ ],
578
+ [
579
+ 12.85202693939209,
580
+ 18.083038330078125,
581
+ 17.04924201965332,
582
+ 6.824688911437988,
583
+ 4.101884841918945
584
+ ],
585
+ [
586
+ 13.499919891357422,
587
+ 18.960132598876953,
588
+ 17.89400291442871,
589
+ 7.106819152832031,
590
+ 4.2823028564453125
591
+ ]
592
+ ],
593
+ "q01": [
594
+ [
595
+ -7.606138229370117,
596
+ -11.486379623413086,
597
+ -14.471379280090332,
598
+ -6.149517059326172,
599
+ -3.963740825653076
600
+ ],
601
+ [
602
+ -9.732962608337402,
603
+ -14.120559692382812,
604
+ -17.71729278564453,
605
+ -7.77755880355835,
606
+ -4.950170993804932
607
+ ],
608
+ [
609
+ -11.841544151306152,
610
+ -16.68042755126953,
611
+ -21.018787384033203,
612
+ -9.203256607055664,
613
+ -5.891245365142822
614
+ ],
615
+ [
616
+ -13.932883262634277,
617
+ -19.075145721435547,
618
+ -24.12662696838379,
619
+ -10.755986213684082,
620
+ -6.858901500701904
621
+ ],
622
+ [
623
+ -15.951546669006348,
624
+ -21.55243682861328,
625
+ -27.221839904785156,
626
+ -12.136816024780273,
627
+ -7.71159553527832
628
+ ],
629
+ [
630
+ -17.917835235595703,
631
+ -23.864578247070312,
632
+ -30.226322174072266,
633
+ -13.59746265411377,
634
+ -8.585160255432129
635
+ ],
636
+ [
637
+ -19.894145965576172,
638
+ -26.176712036132812,
639
+ -33.26250457763672,
640
+ -14.911218643188477,
641
+ -9.40192985534668
642
+ ],
643
+ [
644
+ -21.84005355834961,
645
+ -28.40627670288086,
646
+ -36.22230911254883,
647
+ -16.14484405517578,
648
+ -10.22704029083252
649
+ ],
650
+ [
651
+ -23.756364822387695,
652
+ -30.47894859313965,
653
+ -39.17569351196289,
654
+ -17.362281799316406,
655
+ -10.890664100646973
656
+ ],
657
+ [
658
+ -25.539535522460938,
659
+ -32.617671966552734,
660
+ -42.06956481933594,
661
+ -18.493328094482422,
662
+ -11.642324447631836
663
+ ],
664
+ [
665
+ -27.20233917236328,
666
+ -34.59950256347656,
667
+ -44.80621337890625,
668
+ -19.489608764648438,
669
+ -12.426852226257324
670
+ ],
671
+ [
672
+ -28.937437057495117,
673
+ -36.589599609375,
674
+ -47.56462860107422,
675
+ -20.517410278320312,
676
+ -13.071046829223633
677
+ ],
678
+ [
679
+ -30.5144100189209,
680
+ -38.56317138671875,
681
+ -50.12369155883789,
682
+ -21.383867263793945,
683
+ -13.837430000305176
684
+ ],
685
+ [
686
+ -32.098697662353516,
687
+ -40.545005798339844,
688
+ -52.66397476196289,
689
+ -22.326982498168945,
690
+ -14.528752326965332
691
+ ],
692
+ [
693
+ -33.731502532958984,
694
+ -42.27910614013672,
695
+ -55.20830154418945,
696
+ -23.027145385742188,
697
+ -15.075759887695312
698
+ ],
699
+ [
700
+ -35.24055099487305,
701
+ -44.0214729309082,
702
+ -57.733943939208984,
703
+ -23.813722610473633,
704
+ -15.628026962280273
705
+ ]
706
+ ],
707
+ "q99": [
708
+ [
709
+ 9.87263011932373,
710
+ 13.625106811523438,
711
+ 11.465791702270508,
712
+ 5.390267848968506,
713
+ 3.593177556991577
714
+ ],
715
+ [
716
+ 12.4508695602417,
717
+ 17.175888061523438,
718
+ 14.815629959106445,
719
+ 6.840359687805176,
720
+ 4.55449914932251
721
+ ],
722
+ [
723
+ 15.073770523071289,
724
+ 20.64409637451172,
725
+ 18.112077713012695,
726
+ 8.252008438110352,
727
+ 5.4923787117004395
728
+ ],
729
+ [
730
+ 17.64455223083496,
731
+ 24.112302780151367,
732
+ 21.298742294311523,
733
+ 9.598461151123047,
734
+ 6.391921520233154
735
+ ],
736
+ [
737
+ 20.18633270263672,
738
+ 27.497934341430664,
739
+ 24.572296142578125,
740
+ 10.876214027404785,
741
+ 7.20913028717041
742
+ ],
743
+ [
744
+ 22.714122772216797,
745
+ 30.800987243652344,
746
+ 27.675636291503906,
747
+ 12.082145690917969,
748
+ 7.9814605712890625
749
+ ],
750
+ [
751
+ 25.180116653442383,
752
+ 34.10404586791992,
753
+ 30.77216911315918,
754
+ 13.277448654174805,
755
+ 8.723445892333984
756
+ ],
757
+ [
758
+ 27.6424503326416,
759
+ 37.241947174072266,
760
+ 33.81678009033203,
761
+ 14.368345260620117,
762
+ 9.479180335998535
763
+ ],
764
+ [
765
+ 30.071474075317383,
766
+ 40.379852294921875,
767
+ 36.97920227050781,
768
+ 15.452836990356445,
769
+ 10.219199180603027
770
+ ],
771
+ [
772
+ 32.50673294067383,
773
+ 43.4351806640625,
774
+ 39.76253128051758,
775
+ 16.37455940246582,
776
+ 10.921721458435059
777
+ ],
778
+ [
779
+ 34.83102798461914,
780
+ 46.407928466796875,
781
+ 42.77512741088867,
782
+ 17.416133880615234,
783
+ 11.71412181854248
784
+ ],
785
+ [
786
+ 37.22615432739258,
787
+ 49.29810333251953,
788
+ 45.5524787902832,
789
+ 18.36945915222168,
790
+ 12.46317195892334
791
+ ],
792
+ [
793
+ 39.55117416381836,
794
+ 52.270851135253906,
795
+ 48.24955749511719,
796
+ 19.23557472229004,
797
+ 13.178842544555664
798
+ ],
799
+ [
800
+ 41.80004119873047,
801
+ 54.91329574584961,
802
+ 50.78627395629883,
803
+ 20.06748390197754,
804
+ 13.868379592895508
805
+ ],
806
+ [
807
+ 43.950950622558594,
808
+ 57.63831329345703,
809
+ 53.41462707519531,
810
+ 20.768192291259766,
811
+ 14.467419624328613
812
+ ],
813
+ [
814
+ 46.199745178222656,
815
+ 60.36333465576172,
816
+ 56.08879470825195,
817
+ 21.49018096923828,
818
+ 15.075361251831055
819
+ ]
820
+ ]
821
+ }
822
+ }
823
+ }
824
+ }
experiment_cfg/final_model_config.json ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "Gr00tN1d6",
3
+ "model_dtype": "bfloat16",
4
+ "model_name": "nvidia/Eagle-Block2A-2B-v2",
5
+ "backbone_model_type": "eagle",
6
+ "model_revision": null,
7
+ "tune_top_llm_layers": 4,
8
+ "backbone_embedding_dim": 2048,
9
+ "tune_llm": false,
10
+ "tune_visual": false,
11
+ "select_layer": 16,
12
+ "reproject_vision": false,
13
+ "use_flash_attention": true,
14
+ "load_bf16": true,
15
+ "collator_overwrite_image_inputs": false,
16
+ "eagle_collator": true,
17
+ "backbone_trainable_params_fp32": true,
18
+ "apply_sincos_state_encoding": true,
19
+ "use_relative_action": true,
20
+ "max_state_dim": 128,
21
+ "max_action_dim": 128,
22
+ "action_horizon": 50,
23
+ "hidden_size": 1024,
24
+ "input_embedding_dim": 1536,
25
+ "add_pos_embed": true,
26
+ "attn_dropout": 0.2,
27
+ "use_vlln": true,
28
+ "max_seq_len": 1024,
29
+ "use_alternate_vl_dit": true,
30
+ "attend_text_every_n_blocks": 2,
31
+ "diffusion_model_cfg": {
32
+ "attention_head_dim": 48,
33
+ "dropout": 0.2,
34
+ "final_dropout": true,
35
+ "interleave_self_attention": true,
36
+ "norm_type": "ada_norm",
37
+ "num_attention_heads": 32,
38
+ "num_layers": 32,
39
+ "output_dim": 1024,
40
+ "positional_embeddings": null
41
+ },
42
+ "num_inference_timesteps": 4,
43
+ "noise_beta_alpha": 1.5,
44
+ "noise_beta_beta": 1.0,
45
+ "noise_s": 0.999,
46
+ "num_timestep_buckets": 1000,
47
+ "tune_projector": true,
48
+ "tune_diffusion_model": true,
49
+ "tune_vlln": true,
50
+ "state_dropout_prob": 0.0,
51
+ "state_additive_noise_scale": 0.0,
52
+ "max_num_embodiments": 32
53
+ }
experiment_cfg/final_processor_config.json ADDED
The diff for this file is too large to render. See raw diff