Hugging Face's logo

junbrro
/
gr1hp_loraalign_30000

Safetensors
RLDX-1
Model card Files
xet
 Community
junbrro commited on
Commit
d6cb48b
·
verified ·
1 Parent(s): 7bbf393

30k checkpoint (inference files)

Browse files
Files changed (13) hide show
  1. config.json +137 -0
  2. experiment_cfg/conf.yaml +408 -0
  3. experiment_cfg/config.yaml +399 -0
  4. experiment_cfg/dataset_statistics.json +272 -0
  5. experiment_cfg/final_model_config.json +207 -0
  6. experiment_cfg/final_processor_config.json +0 -0
  7. model-00001-of-00003.safetensors +3 -0
  8. model-00002-of-00003.safetensors +3 -0
  9. model-00003-of-00003.safetensors +3 -0
  10. model.safetensors.index.json +0 -0
  11. processor/embodiment_id.json +45 -0
  12. processor/processor_config.json +2963 -0
  13. processor/statistics.json +0 -0
config.json ADDED
@@ -0,0 +1,137 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "action_horizon": 16,
3
+ "action_model_lora_alpha": 32,
4
+ "action_model_lora_dropout": 0.0,
5
+ "action_model_lora_rank": 16,
6
+ "action_model_lora_target_modules": [
7
+ "vl_qkv",
8
+ "vl_proj",
9
+ "sa_qkv",
10
+ "sa_proj",
11
+ "p_qkv",
12
+ "p_proj",
13
+ "linear1",
14
+ "linear2"
15
+ ],
16
+ "action_model_use_lora": false,
17
+ "actlat_embodiment_id": "openarm_prq",
18
+ "actlat_mode": false,
19
+ "actlat_prq_stats_path": "",
20
+ "actlat_standardize": false,
21
+ "actlat_tokenizer_path": "",
22
+ "actlat_z_stats_path": "",
23
+ "anchor_dim_norm": "",
24
+ "anchor_dim_stats_path": "",
25
+ "anchor_human_tips_weight": 1.0,
26
+ "anchor_readout_asym": false,
27
+ "anchor_readout_dim": 24,
28
+ "anchor_readout_shuffle": false,
29
+ "anchor_readout_taps": "cog,d2,s3,s7",
30
+ "anchor_readout_tweight": true,
31
+ "anchor_readout_weight": 0.0,
32
+ "anchor_tips_dims": 15,
33
+ "architectures": [
34
+ "RLDX"
35
+ ],
36
+ "backbone_lora_alpha": 32,
37
+ "backbone_lora_dropout": 0.0,
38
+ "backbone_lora_num_layers": -1,
39
+ "backbone_lora_rank": 16,
40
+ "backbone_lora_target_modules": [
41
+ "q_proj",
42
+ "k_proj",
43
+ "v_proj",
44
+ "o_proj",
45
+ "gate_proj",
46
+ "up_proj",
47
+ "down_proj"
48
+ ],
49
+ "backbone_use_lora": true,
50
+ "color_jitter_params": null,
51
+ "conversation_image_first": false,
52
+ "diffusion_model_cfg": {
53
+ "action_model_max_seq_len": 512,
54
+ "attention_head_dim": 64,
55
+ "depth_multi_stream": 4,
56
+ "depth_single_stream": 8,
57
+ "dropout": 0.2,
58
+ "final_dropout": true,
59
+ "num_attention_heads": 24,
60
+ "output_dim": 1024,
61
+ "physics_dim": 0,
62
+ "positional_embeddings": "rope_sa_only",
63
+ "pre_norm": "layer_norm",
64
+ "qk_norm": "rms_norm",
65
+ "rope_theta": 10000.0,
66
+ "sa_dim": 1536,
67
+ "temb_type": "input_token",
68
+ "use_physics": false,
69
+ "use_swiglu": true,
70
+ "vl_dim": 4096
71
+ },
72
+ "dtype": "bfloat16",
73
+ "egopi_split_embodiment": false,
74
+ "embod_reg_gather": false,
75
+ "embod_reg_hand_bins": 0,
76
+ "embod_reg_hand_edges": "",
77
+ "embod_reg_layer": 2,
78
+ "embod_reg_mode": "",
79
+ "embod_reg_tbins": 0,
80
+ "embod_reg_vic_cov": 0.04,
81
+ "embod_reg_vic_var": 1.0,
82
+ "embod_reg_weight": 0.05,
83
+ "freeze_cog_tokens": false,
84
+ "freeze_modules": "",
85
+ "general_embodiment_train_ratio": 0,
86
+ "gr1hp_align_dim": 24,
87
+ "gr1hp_align_only": true,
88
+ "gr1hp_align_shuffle": false,
89
+ "gr1hp_align_weight": 0.2,
90
+ "gr1hp_split_embodiment": false,
91
+ "image_max_area": 65536,
92
+ "image_resize_m": 32,
93
+ "keep_full_lm": false,
94
+ "memory_cfg": {
95
+ "hidden_size": 4096,
96
+ "intermediate_size": 16384,
97
+ "max_position_embeddings": 32,
98
+ "num_attention_heads": 16,
99
+ "num_hidden_layers": 2,
100
+ "num_key_value_heads": 16,
101
+ "rms_norm_eps": 1e-05,
102
+ "use_causal_attn": true,
103
+ "use_rope": true
104
+ },
105
+ "model_name": "RLWRLD/RLDX-1-VLM",
106
+ "model_revision": null,
107
+ "model_type": "RLDX-1",
108
+ "n_cog_tokens": 64,
109
+ "physics_delta_indices": null,
110
+ "physics_dims": [],
111
+ "physics_keys": [],
112
+ "physics_use_flow_matching": true,
113
+ "random_crop_fraction": null,
114
+ "random_rotation_angle": null,
115
+ "rtc_inference_delay": 0,
116
+ "rtc_inference_exec_horizon": 0,
117
+ "rtc_inference_mode": "none",
118
+ "rtc_jacobian_steps_only": 3,
119
+ "rtc_training_max_delay": 0,
120
+ "stage_from_checkpoint": "",
121
+ "state_dropout_prob": 0.0,
122
+ "train_stage": "",
123
+ "transformers_version": "4.57.0",
124
+ "tune_diffusion_model": true,
125
+ "tune_llm": false,
126
+ "tune_projector": true,
127
+ "tune_top_llm_layers": 0,
128
+ "tune_visual": false,
129
+ "use_memory": false,
130
+ "use_motion": false,
131
+ "use_physics": false,
132
+ "use_video": true,
133
+ "video_length": 1,
134
+ "video_stride": 2,
135
+ "vlm_lang_loss": false,
136
+ "vlm_lang_weight": 0.5
137
+ }
experiment_cfg/conf.yaml ADDED
@@ -0,0 +1,408 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ load_config_path: null
2
+ model:
3
+ model_type: RLDX-1
4
+ model_dtype: bfloat16
5
+ model_name: RLWRLD/RLDX-1-VLM
6
+ backbone_model_type: vtc_qwen3_vl
7
+ model_revision: null
8
+ tune_top_llm_layers: 0
9
+ backbone_embedding_dim: 4096
10
+ tune_llm: false
11
+ tune_visual: false
12
+ select_layer: 18
13
+ reproject_vision: false
14
+ use_flash_attention: true
15
+ load_bf16: true
16
+ backbone_trainable_params_fp32: true
17
+ freeze_cog_tokens: false
18
+ keep_full_lm: false
19
+ vlm_lang_loss: false
20
+ vlm_lang_weight: 0.5
21
+ image_max_area: 65536
22
+ image_resize_m: 32
23
+ random_crop_fraction: null
24
+ random_rotation_angle: null
25
+ color_jitter_params: null
26
+ formalize_language: true
27
+ apply_sincos_state_encoding: false
28
+ use_relative_action: true
29
+ use_percentiles: true
30
+ conversation_image_first: false
31
+ max_state_dim: 64
32
+ max_action_dim: 64
33
+ action_horizon: 16
34
+ hidden_size: 1024
35
+ input_embedding_dim: 1536
36
+ general_embodiment_train_ratio: 0.0
37
+ add_pos_embed: true
38
+ attn_dropout: 0.2
39
+ use_vlln: false
40
+ max_seq_len: 1024
41
+ n_cog_tokens: 64
42
+ diffusion_model_cfg:
43
+ attention_head_dim: 64
44
+ depth_multi_stream: 4
45
+ depth_single_stream: 8
46
+ dropout: 0.2
47
+ final_dropout: true
48
+ num_attention_heads: 24
49
+ output_dim: 1024
50
+ positional_embeddings: rope_sa_only
51
+ rope_theta: 10000.0
52
+ temb_type: input_token
53
+ use_swiglu: true
54
+ action_model_max_seq_len: 512
55
+ pre_norm: layer_norm
56
+ qk_norm: rms_norm
57
+ num_inference_timesteps: 4
58
+ noise_beta_alpha: 1.5
59
+ noise_beta_beta: 1.0
60
+ noise_s: 0.999
61
+ num_timestep_buckets: 1000
62
+ tune_projector: true
63
+ tune_diffusion_model: true
64
+ tune_vlln: true
65
+ action_model_use_lora: false
66
+ action_model_lora_rank: 16
67
+ action_model_lora_alpha: 32
68
+ action_model_lora_dropout: 0.0
69
+ action_model_lora_target_modules:
70
+ - vl_qkv
71
+ - vl_proj
72
+ - sa_qkv
73
+ - sa_proj
74
+ - p_qkv
75
+ - p_proj
76
+ - linear1
77
+ - linear2
78
+ backbone_use_lora: true
79
+ backbone_lora_rank: 16
80
+ backbone_lora_alpha: 32
81
+ backbone_lora_dropout: 0.0
82
+ backbone_lora_num_layers: -1
83
+ backbone_lora_target_modules:
84
+ - q_proj
85
+ - k_proj
86
+ - v_proj
87
+ - o_proj
88
+ - gate_proj
89
+ - up_proj
90
+ - down_proj
91
+ state_dropout_prob: 0.0
92
+ state_additive_noise_scale: 0.0
93
+ rtc_training_max_delay: 0
94
+ rtc_inference_mode: none
95
+ rtc_inference_delay: 0
96
+ rtc_inference_exec_horizon: 0
97
+ rtc_jacobian_steps_only: 3
98
+ use_memory: false
99
+ memory_length: 4
100
+ memory_n_cog_tokens: null
101
+ concat_memory: false
102
+ memory_dropout_prob: 0.0
103
+ memory_stride: 16
104
+ memory_cfg:
105
+ hidden_size: 4096
106
+ intermediate_size: 16384
107
+ num_hidden_layers: 2
108
+ num_attention_heads: 16
109
+ num_key_value_heads: 16
110
+ max_position_embeddings: 32
111
+ rms_norm_eps: 1.0e-05
112
+ use_causal_attn: true
113
+ use_rope: true
114
+ use_motion: false
115
+ motion_insert_layer: 9
116
+ motion_d_hid: 512
117
+ motion_window:
118
+ - 5
119
+ - 9
120
+ - 9
121
+ motion_ext_chnls:
122
+ - 256
123
+ motion_int_chnls:
124
+ - 256
125
+ - 256
126
+ - 512
127
+ motion_corr_func: cosine
128
+ motion_n_encoders: 1
129
+ motion_use_layerscale: false
130
+ motion_layerscale_init: 1.0e-05
131
+ motion_use_layernorm: false
132
+ motion_use_syncbn: false
133
+ motion_injection_point: vision_encoder
134
+ motion_pool_type: avg
135
+ motion_drop: true
136
+ motion_gradient_check: false
137
+ motion_int_mode: lite
138
+ use_video: true
139
+ video_length: 1
140
+ video_stride: 2
141
+ use_physics: false
142
+ physics_keys: []
143
+ physics_dims: []
144
+ physics_loss_weight: 0.1
145
+ allow_missing_physics: false
146
+ physics_delta_indices: null
147
+ physics_use_flow_matching: true
148
+ physics_dropout_prob: 0.0
149
+ use_finger_latent: false
150
+ finger_wrist_mode: delta
151
+ finger_bimanual: false
152
+ finger_alignment_npz: ''
153
+ finger_fingertip_norm_npz: ''
154
+ finger_wrist_delta_stats_npz: ''
155
+ finger_wrist_abs_stats_npz: ''
156
+ embod_reg_mode: ''
157
+ embod_reg_weight: 0.05
158
+ embod_reg_layer: 2
159
+ embod_reg_gather: false
160
+ embod_reg_tbins: 0
161
+ embod_reg_vic_var: 1.0
162
+ embod_reg_vic_cov: 0.04
163
+ embod_reg_hand_bins: 0
164
+ embod_reg_hand_edges: ''
165
+ anchor_dim_norm: ''
166
+ anchor_dim_stats_path: ''
167
+ anchor_tips_dims: 15
168
+ anchor_human_tips_weight: 1.0
169
+ actlat_mode: false
170
+ actlat_tokenizer_path: ''
171
+ actlat_embodiment_id: openarm_prq
172
+ actlat_prq_stats_path: ''
173
+ actlat_standardize: false
174
+ actlat_z_stats_path: ''
175
+ anchor_readout_weight: 0.0
176
+ anchor_readout_taps: cog,d2,s3,s7
177
+ anchor_readout_dim: 24
178
+ anchor_readout_tweight: true
179
+ anchor_readout_shuffle: false
180
+ anchor_readout_asym: false
181
+ gr1hp_align_weight: 0.2
182
+ gr1hp_align_only: true
183
+ gr1hp_align_dim: 24
184
+ gr1hp_align_shuffle: false
185
+ geom_bn_mode: ''
186
+ geom_bn_weight: 0.0
187
+ geom_bn_dim: 24
188
+ geom_bn_asym: true
189
+ geom_bn_ema: 0.0
190
+ geom_bn_shuffle: false
191
+ geom_bn_lift_hidden: 0
192
+ geom_bn_route_warmup_steps: 0
193
+ geom_effort_dim: 0
194
+ geom_effort_weight: 0.0
195
+ egopi_split_embodiment: false
196
+ gr1hp_split_embodiment: false
197
+ train_stage: ''
198
+ stage_from_checkpoint: ''
199
+ freeze_modules: ''
200
+ max_num_embodiments: 36
201
+ data:
202
+ datasets:
203
+ - dataset_paths:
204
+ - /data/junhyeong/datasets/gr1/gr1_mg_gr00t_100
205
+ embodiment_tag: gr1_human_prq
206
+ mix_ratio: 0.5
207
+ dataset_type: physical_embodiment
208
+ val_dataset_path: null
209
+ egopi_source: null
210
+ egopi_fk_cache_dir: null
211
+ egopi_filter_path: null
212
+ egopi_stats_path: null
213
+ task_filter: null
214
+ hawor_source: null
215
+ hawor_fk_cache_dir: null
216
+ hawor_filter_path: null
217
+ hawor_human_h5: null
218
+ hawor_human_source_map: null
219
+ hawor_bimanual: false
220
+ gr1_prq_source: null
221
+ gr1_prq_cache_path: null
222
+ gr1_prq_stats_path: null
223
+ gr1hp_source: gr1
224
+ gr1hp_bridge_path: assets/gr1/human_gr1_bridge.npz
225
+ gr1hp_stats_path: assets/gr1/gr1_human_prq_stats.json
226
+ gr1hp_human_h5: null
227
+ gr1hp_human_source_map: null
228
+ gr1hp_hand: auto
229
+ gr1hp_subsample_episodes: 479
230
+ gr1hp_cache_path: assets/gr1/prq_cache/gr1.h5
231
+ gr1hp_dual_raw: false
232
+ gr1hp_raw_embodiment_tag: null
233
+ gr1hp_mano_anchor: false
234
+ video_key_map: null
235
+ lang_key_map: null
236
+ - dataset_paths:
237
+ - /data/junhyeong/datasets/rlwrld_human_lerobot
238
+ embodiment_tag: gr1_human_prq
239
+ mix_ratio: 0.5
240
+ dataset_type: physical_embodiment
241
+ val_dataset_path: null
242
+ egopi_source: null
243
+ egopi_fk_cache_dir: null
244
+ egopi_filter_path: null
245
+ egopi_stats_path: null
246
+ task_filter: null
247
+ hawor_source: null
248
+ hawor_fk_cache_dir: null
249
+ hawor_filter_path: null
250
+ hawor_human_h5: null
251
+ hawor_human_source_map: null
252
+ hawor_bimanual: false
253
+ gr1_prq_source: null
254
+ gr1_prq_cache_path: null
255
+ gr1_prq_stats_path: null
256
+ gr1hp_source: human
257
+ gr1hp_bridge_path: assets/gr1/human_gr1_bridge.npz
258
+ gr1hp_stats_path: assets/gr1/gr1_human_prq_stats.json
259
+ gr1hp_human_h5: /data/junhyeong/assets/rlwrld_final.h5
260
+ gr1hp_human_source_map: /data/junhyeong/datasets/rlwrld_human_lerobot/meta/source_map.csv
261
+ gr1hp_hand: auto
262
+ gr1hp_subsample_episodes: null
263
+ gr1hp_cache_path: assets/gr1/prq_cache/human.h5
264
+ gr1hp_dual_raw: false
265
+ gr1hp_raw_embodiment_tag: null
266
+ gr1hp_mano_anchor: true
267
+ video_key_map:
268
+ ego_view: camera_ego_right
269
+ lang_key_map:
270
+ human.coarse_action: human.task_description
271
+ modality_configs:
272
+ gr1_human_prq:
273
+ video:
274
+ delta_indices:
275
+ - 0
276
+ modality_keys:
277
+ - ego_view
278
+ sin_cos_embedding_keys: null
279
+ mean_std_embedding_keys: null
280
+ action_configs: null
281
+ state:
282
+ delta_indices:
283
+ - 0
284
+ modality_keys:
285
+ - wrist_pos
286
+ - wrist_rot6d
287
+ - hand_q
288
+ sin_cos_embedding_keys: null
289
+ mean_std_embedding_keys: null
290
+ action_configs: null
291
+ action:
292
+ delta_indices:
293
+ - 0
294
+ - 1
295
+ - 2
296
+ - 3
297
+ - 4
298
+ - 5
299
+ - 6
300
+ - 7
301
+ - 8
302
+ - 9
303
+ - 10
304
+ - 11
305
+ - 12
306
+ - 13
307
+ - 14
308
+ - 15
309
+ modality_keys:
310
+ - wrist_pos
311
+ - wrist_rot6d
312
+ - hand_q
313
+ sin_cos_embedding_keys: null
314
+ mean_std_embedding_keys: null
315
+ action_configs:
316
+ - rep: ABSOLUTE
317
+ type: NON_EEF
318
+ format: DEFAULT
319
+ state_key: null
320
+ - rep: ABSOLUTE
321
+ type: NON_EEF
322
+ format: DEFAULT
323
+ state_key: null
324
+ - rep: ABSOLUTE
325
+ type: NON_EEF
326
+ format: DEFAULT
327
+ state_key: null
328
+ language:
329
+ delta_indices:
330
+ - 0
331
+ modality_keys:
332
+ - annotation.human.coarse_action
333
+ sin_cos_embedding_keys: null
334
+ mean_std_embedding_keys: null
335
+ action_configs: null
336
+ dataset_mode: standard
337
+ download_cache: false
338
+ shard_size: 1024
339
+ episode_sampling_rate: 0.1
340
+ num_shards_per_epoch: 100000
341
+ override_pretraining_statistics: false
342
+ mode: single_turn
343
+ random_chop: 0.0
344
+ mock_dataset_mode: false
345
+ shuffle: true
346
+ seed: 42
347
+ multiprocessing_context: fork
348
+ allow_padding: true
349
+ subsample_ratio: 1.0
350
+ video_backend: torchcodec
351
+ training:
352
+ output_dir: /data/rlwrld-unified-checkpoints/junhyeong/checkpoints/gr1hp-loraalign-20260808
353
+ experiment_name: gr1hp-loraalign-20260808
354
+ max_steps: 30000
355
+ global_batch_size: 64
356
+ batch_size: null
357
+ gradient_accumulation_steps: 1
358
+ learning_rate: 0.0001
359
+ lr_scheduler_type: cosine
360
+ weight_decay: 1.0e-05
361
+ warmup_ratio: 0.05
362
+ warmup_steps: 0
363
+ max_grad_norm: 1.0
364
+ new_param_warmup_steps: 0
365
+ optim: adamw_torch_fused
366
+ start_from_checkpoint: /data/junhyeong/ckpt/RLDX-1-PT-IMG
367
+ tf32: true
368
+ fp16: false
369
+ bf16: true
370
+ eval_bf16: true
371
+ logging_steps: 10
372
+ save_steps: 5000
373
+ save_total_limit: 7
374
+ save_vl_model: false
375
+ upload_checkpoints: false
376
+ upload_every: 1000
377
+ upload_last_n_checkpoints: 5
378
+ max_concurrent_uploads: 2
379
+ eval_strategy: 'no'
380
+ eval_steps: 500
381
+ eval_set_split_ratio: 0.1
382
+ eval_batch_size: 2
383
+ save_best_eval_metric_name: ''
384
+ save_best_eval_metric_greater_is_better: true
385
+ deepspeed_stage: 2
386
+ gradient_checkpointing: false
387
+ transformers_trust_remote_code: true
388
+ transformers_local_files_only: false
389
+ transformers_cache_dir: null
390
+ transformers_access_token: null
391
+ use_ddp: false
392
+ ddp_bucket_cap_mb: 100
393
+ num_gpus: 4
394
+ dataloader_num_workers: 8
395
+ remove_unused_columns: false
396
+ use_wandb: true
397
+ wandb_project: rldx-gr1hp
398
+ enable_profiling: false
399
+ max_retries: 3
400
+ assert_loss_less_than: null
401
+ add_rl_callback: false
402
+ enable_open_loop_eval: false
403
+ open_loop_eval_traj_ids:
404
+ - 0
405
+ open_loop_eval_steps_per_traj: 100
406
+ open_loop_eval_plot_indices: null
407
+ max_steps: 30000
408
+ save_steps: 5000
experiment_cfg/config.yaml ADDED
@@ -0,0 +1,399 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ !!python/object:rldx.configs.base_config.Config
2
+ data: !!python/object:rldx.configs.data.data_config.DataConfig
3
+ allow_padding: true
4
+ dataset_mode: standard
5
+ datasets:
6
+ - !!python/object:rldx.configs.data.data_config.SingleDatasetConfig
7
+ dataset_paths:
8
+ - /data/junhyeong/datasets/gr1/gr1_mg_gr00t_100
9
+ dataset_type: physical_embodiment
10
+ egopi_filter_path: null
11
+ egopi_fk_cache_dir: null
12
+ egopi_source: null
13
+ egopi_stats_path: null
14
+ embodiment_tag: gr1_human_prq
15
+ gr1_prq_cache_path: null
16
+ gr1_prq_source: null
17
+ gr1_prq_stats_path: null
18
+ gr1hp_bridge_path: assets/gr1/human_gr1_bridge.npz
19
+ gr1hp_cache_path: assets/gr1/prq_cache/gr1.h5
20
+ gr1hp_dual_raw: false
21
+ gr1hp_hand: auto
22
+ gr1hp_human_h5: null
23
+ gr1hp_human_source_map: null
24
+ gr1hp_mano_anchor: false
25
+ gr1hp_raw_embodiment_tag: null
26
+ gr1hp_source: gr1
27
+ gr1hp_stats_path: assets/gr1/gr1_human_prq_stats.json
28
+ gr1hp_subsample_episodes: 479
29
+ hawor_bimanual: false
30
+ hawor_filter_path: null
31
+ hawor_fk_cache_dir: null
32
+ hawor_human_h5: null
33
+ hawor_human_source_map: null
34
+ hawor_source: null
35
+ lang_key_map: null
36
+ mix_ratio: 0.5
37
+ task_filter: null
38
+ val_dataset_path: null
39
+ video_key_map: null
40
+ - !!python/object:rldx.configs.data.data_config.SingleDatasetConfig
41
+ dataset_paths:
42
+ - /data/junhyeong/datasets/rlwrld_human_lerobot
43
+ dataset_type: physical_embodiment
44
+ egopi_filter_path: null
45
+ egopi_fk_cache_dir: null
46
+ egopi_source: null
47
+ egopi_stats_path: null
48
+ embodiment_tag: gr1_human_prq
49
+ gr1_prq_cache_path: null
50
+ gr1_prq_source: null
51
+ gr1_prq_stats_path: null
52
+ gr1hp_bridge_path: assets/gr1/human_gr1_bridge.npz
53
+ gr1hp_cache_path: assets/gr1/prq_cache/human.h5
54
+ gr1hp_dual_raw: false
55
+ gr1hp_hand: auto
56
+ gr1hp_human_h5: /data/junhyeong/assets/rlwrld_final.h5
57
+ gr1hp_human_source_map: /data/junhyeong/datasets/rlwrld_human_lerobot/meta/source_map.csv
58
+ gr1hp_mano_anchor: true
59
+ gr1hp_raw_embodiment_tag: null
60
+ gr1hp_source: human
61
+ gr1hp_stats_path: assets/gr1/gr1_human_prq_stats.json
62
+ gr1hp_subsample_episodes: null
63
+ hawor_bimanual: false
64
+ hawor_filter_path: null
65
+ hawor_fk_cache_dir: null
66
+ hawor_human_h5: null
67
+ hawor_human_source_map: null
68
+ hawor_source: null
69
+ lang_key_map:
70
+ human.coarse_action: human.task_description
71
+ mix_ratio: 0.5
72
+ task_filter: null
73
+ val_dataset_path: null
74
+ video_key_map:
75
+ ego_view: camera_ego_right
76
+ download_cache: false
77
+ episode_sampling_rate: 0.1
78
+ mock_dataset_mode: false
79
+ modality_configs:
80
+ gr1_human_prq:
81
+ action: !!python/object:rldx.data.types.ModalityConfig
82
+ action_configs:
83
+ - !!python/object:rldx.data.types.ActionConfig
84
+ format: &id001 !!python/object/apply:rldx.data.types.ActionFormat
85
+ - default
86
+ rep: &id002 !!python/object/apply:rldx.data.types.ActionRepresentation
87
+ - absolute
88
+ state_key: null
89
+ type: &id003 !!python/object/apply:rldx.data.types.ActionType
90
+ - non_eef
91
+ - !!python/object:rldx.data.types.ActionConfig
92
+ format: *id001
93
+ rep: *id002
94
+ state_key: null
95
+ type: *id003
96
+ - !!python/object:rldx.data.types.ActionConfig
97
+ format: *id001
98
+ rep: *id002
99
+ state_key: null
100
+ type: *id003
101
+ delta_indices:
102
+ - 0
103
+ - 1
104
+ - 2
105
+ - 3
106
+ - 4
107
+ - 5
108
+ - 6
109
+ - 7
110
+ - 8
111
+ - 9
112
+ - 10
113
+ - 11
114
+ - 12
115
+ - 13
116
+ - 14
117
+ - 15
118
+ mean_std_embedding_keys: null
119
+ modality_keys:
120
+ - wrist_pos
121
+ - wrist_rot6d
122
+ - hand_q
123
+ sin_cos_embedding_keys: null
124
+ language: !!python/object:rldx.data.types.ModalityConfig
125
+ action_configs: null
126
+ delta_indices:
127
+ - 0
128
+ mean_std_embedding_keys: null
129
+ modality_keys:
130
+ - annotation.human.coarse_action
131
+ sin_cos_embedding_keys: null
132
+ state: !!python/object:rldx.data.types.ModalityConfig
133
+ action_configs: null
134
+ delta_indices:
135
+ - 0
136
+ mean_std_embedding_keys: null
137
+ modality_keys:
138
+ - wrist_pos
139
+ - wrist_rot6d
140
+ - hand_q
141
+ sin_cos_embedding_keys: null
142
+ video: !!python/object:rldx.data.types.ModalityConfig
143
+ action_configs: null
144
+ delta_indices:
145
+ - 0
146
+ mean_std_embedding_keys: null
147
+ modality_keys:
148
+ - ego_view
149
+ sin_cos_embedding_keys: null
150
+ mode: single_turn
151
+ multiprocessing_context: fork
152
+ num_shards_per_epoch: 100000
153
+ override_pretraining_statistics: false
154
+ random_chop: 0.0
155
+ seed: 42
156
+ shard_size: 1024
157
+ shuffle: true
158
+ subsample_ratio: 1.0
159
+ video_backend: torchcodec
160
+ load_config_path: null
161
+ model: !!python/object:rldx.configs.model.rldx.RLDXConfig
162
+ _attn_implementation_internal: null
163
+ _commit_hash: null
164
+ _name_or_path: ''
165
+ _output_attentions: false
166
+ action_horizon: 16
167
+ action_model_lora_alpha: 32
168
+ action_model_lora_dropout: 0.0
169
+ action_model_lora_rank: 16
170
+ action_model_lora_target_modules:
171
+ - vl_qkv
172
+ - vl_proj
173
+ - sa_qkv
174
+ - sa_proj
175
+ - p_qkv
176
+ - p_proj
177
+ - linear1
178
+ - linear2
179
+ action_model_use_lora: false
180
+ actlat_embodiment_id: openarm_prq
181
+ actlat_mode: false
182
+ actlat_prq_stats_path: ''
183
+ actlat_standardize: false
184
+ actlat_tokenizer_path: ''
185
+ actlat_z_stats_path: ''
186
+ add_cross_attention: false
187
+ anchor_dim_norm: ''
188
+ anchor_dim_stats_path: ''
189
+ anchor_human_tips_weight: 1.0
190
+ anchor_readout_asym: false
191
+ anchor_readout_dim: 24
192
+ anchor_readout_shuffle: false
193
+ anchor_readout_taps: cog,d2,s3,s7
194
+ anchor_readout_tweight: true
195
+ anchor_readout_weight: 0.0
196
+ anchor_tips_dims: 15
197
+ architectures: null
198
+ backbone_lora_alpha: 32
199
+ backbone_lora_dropout: 0.0
200
+ backbone_lora_num_layers: -1
201
+ backbone_lora_rank: 16
202
+ backbone_lora_target_modules:
203
+ - q_proj
204
+ - k_proj
205
+ - v_proj
206
+ - o_proj
207
+ - gate_proj
208
+ - up_proj
209
+ - down_proj
210
+ backbone_use_lora: true
211
+ bad_words_ids: null
212
+ begin_suppress_tokens: null
213
+ bos_token_id: null
214
+ chunk_size_feed_forward: 0
215
+ color_jitter_params: null
216
+ conversation_image_first: false
217
+ cross_attention_hidden_size: null
218
+ decoder_start_token_id: null
219
+ diffusion_model_cfg:
220
+ action_model_max_seq_len: 512
221
+ attention_head_dim: 64
222
+ depth_multi_stream: 4
223
+ depth_single_stream: 8
224
+ dropout: 0.2
225
+ final_dropout: true
226
+ num_attention_heads: 24
227
+ output_dim: 1024
228
+ positional_embeddings: rope_sa_only
229
+ pre_norm: layer_norm
230
+ qk_norm: rms_norm
231
+ rope_theta: 10000.0
232
+ temb_type: input_token
233
+ use_swiglu: true
234
+ diversity_penalty: 0.0
235
+ do_sample: false
236
+ dtype: null
237
+ early_stopping: false
238
+ egopi_split_embodiment: false
239
+ embod_reg_gather: false
240
+ embod_reg_hand_bins: 0
241
+ embod_reg_hand_edges: ''
242
+ embod_reg_layer: 2
243
+ embod_reg_mode: ''
244
+ embod_reg_tbins: 0
245
+ embod_reg_vic_cov: 0.04
246
+ embod_reg_vic_var: 1.0
247
+ embod_reg_weight: 0.05
248
+ encoder_no_repeat_ngram_size: 0
249
+ eos_token_id: null
250
+ exponential_decay_length_penalty: null
251
+ finetuning_task: null
252
+ forced_bos_token_id: null
253
+ forced_eos_token_id: null
254
+ freeze_cog_tokens: false
255
+ freeze_modules: ''
256
+ general_embodiment_train_ratio: 0
257
+ gr1hp_align_dim: 24
258
+ gr1hp_align_only: true
259
+ gr1hp_align_shuffle: false
260
+ gr1hp_align_weight: 0.2
261
+ gr1hp_split_embodiment: false
262
+ id2label:
263
+ 0: LABEL_0
264
+ 1: LABEL_1
265
+ image_max_area: 65536
266
+ image_resize_m: 32
267
+ is_decoder: false
268
+ is_encoder_decoder: false
269
+ keep_full_lm: false
270
+ label2id:
271
+ LABEL_0: 0
272
+ LABEL_1: 1
273
+ length_penalty: 1.0
274
+ max_length: 20
275
+ memory_cfg:
276
+ hidden_size: 4096
277
+ intermediate_size: 16384
278
+ max_position_embeddings: 32
279
+ num_attention_heads: 16
280
+ num_hidden_layers: 2
281
+ num_key_value_heads: 16
282
+ rms_norm_eps: 1.0e-05
283
+ use_causal_attn: true
284
+ use_rope: true
285
+ min_length: 0
286
+ model_name: RLWRLD/RLDX-1-VLM
287
+ model_revision: null
288
+ model_type: RLDX-1
289
+ n_cog_tokens: 64
290
+ no_repeat_ngram_size: 0
291
+ num_beam_groups: 1
292
+ num_beams: 1
293
+ num_return_sequences: 1
294
+ output_hidden_states: false
295
+ output_scores: false
296
+ pad_token_id: null
297
+ physics_dims: []
298
+ physics_keys: []
299
+ prefix: null
300
+ problem_type: null
301
+ pruned_heads: {}
302
+ random_crop_fraction: null
303
+ random_rotation_angle: null
304
+ remove_invalid_values: false
305
+ repetition_penalty: 1.0
306
+ return_dict: true
307
+ return_dict_in_generate: false
308
+ rtc_inference_delay: 0
309
+ rtc_inference_exec_horizon: 0
310
+ rtc_inference_mode: none
311
+ rtc_jacobian_steps_only: 3
312
+ rtc_training_max_delay: 0
313
+ sep_token_id: null
314
+ stage_from_checkpoint: ''
315
+ state_dropout_prob: 0.0
316
+ suppress_tokens: null
317
+ task_specific_params: null
318
+ temperature: 1.0
319
+ tf_legacy_loss: false
320
+ tie_encoder_decoder: false
321
+ tie_word_embeddings: true
322
+ tokenizer_class: null
323
+ top_k: 50
324
+ top_p: 1.0
325
+ torchscript: false
326
+ train_stage: ''
327
+ transformers_version: null
328
+ tune_diffusion_model: true
329
+ tune_llm: false
330
+ tune_projector: true
331
+ tune_top_llm_layers: 0
332
+ tune_visual: false
333
+ typical_p: 1.0
334
+ use_bfloat16: false
335
+ use_memory: false
336
+ use_motion: false
337
+ use_physics: false
338
+ use_video: true
339
+ video_length: 1
340
+ video_stride: 2
341
+ vlm_lang_loss: false
342
+ vlm_lang_weight: 0.5
343
+ training: !!python/object:rldx.configs.training.training_config.TrainingConfig
344
+ add_rl_callback: false
345
+ assert_loss_less_than: null
346
+ backbone_lr: 0.0
347
+ batch_size: null
348
+ bf16: true
349
+ dataloader_num_workers: 8
350
+ ddp_bucket_cap_mb: 100
351
+ deepspeed_stage: 2
352
+ enable_open_loop_eval: false
353
+ enable_profiling: false
354
+ eval_batch_size: 2
355
+ eval_bf16: true
356
+ eval_set_split_ratio: 0.1
357
+ eval_steps: 500
358
+ eval_strategy: 'no'
359
+ experiment_name: gr1hp-loraalign-20260808
360
+ fp16: false
361
+ global_batch_size: 64
362
+ gradient_accumulation_steps: 1
363
+ gradient_checkpointing: false
364
+ learning_rate: 0.0001
365
+ logging_steps: 10
366
+ lr_scheduler_type: cosine
367
+ max_concurrent_uploads: 2
368
+ max_grad_norm: 1.0
369
+ max_retries: 3
370
+ max_steps: 30000
371
+ new_param_warmup_steps: 0
372
+ num_gpus: 4
373
+ open_loop_eval_plot_indices: null
374
+ open_loop_eval_steps_per_traj: 100
375
+ open_loop_eval_traj_ids:
376
+ - 0
377
+ optim: adamw_torch_fused
378
+ output_dir: /data/rlwrld-unified-checkpoints/junhyeong/checkpoints/gr1hp-loraalign-20260808
379
+ remove_unused_columns: false
380
+ save_best_eval_metric_greater_is_better: true
381
+ save_best_eval_metric_name: ''
382
+ save_steps: 5000
383
+ save_total_limit: 7
384
+ save_vl_model: false
385
+ start_from_checkpoint: /data/junhyeong/ckpt/RLDX-1-PT-IMG
386
+ tf32: true
387
+ transformers_access_token: null
388
+ transformers_cache_dir: null
389
+ transformers_local_files_only: false
390
+ transformers_trust_remote_code: true
391
+ upload_checkpoints: false
392
+ upload_every: 1000
393
+ upload_last_n_checkpoints: 5
394
+ use_ddp: false
395
+ use_wandb: true
396
+ wandb_project: rldx-gr1hp
397
+ warmup_ratio: 0.05
398
+ warmup_steps: 0
399
+ weight_decay: 1.0e-05
experiment_cfg/dataset_statistics.json ADDED
@@ -0,0 +1,272 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "gr1_human_prq": {
3
+ "state": {
4
+ "wrist_pos": {
5
+ "min": [
6
+ -0.12936705350875854,
7
+ -0.667879045009613,
8
+ -0.011580604128539562
9
+ ],
10
+ "max": [
11
+ 0.5296112895011902,
12
+ 0.12853454053401947,
13
+ 0.6460531949996948
14
+ ],
15
+ "mean": [
16
+ 0.29533956944942474,
17
+ -0.24158845096826553,
18
+ 0.25371725857257843
19
+ ],
20
+ "std": [
21
+ 0.09716614587744744,
22
+ 0.11296890982830084,
23
+ 0.07645092208772325
24
+ ],
25
+ "q01": [
26
+ 0.07942652076482773,
27
+ -0.5644937646389008,
28
+ 0.08935761481523515
29
+ ],
30
+ "q99": [
31
+ 0.48779279112815854,
32
+ 0.05798103630542755,
33
+ 0.5285348415374755
34
+ ]
35
+ },
36
+ "wrist_rot6d": {
37
+ "min": [
38
+ -0.9558911919593811,
39
+ -0.7119177579879761,
40
+ -0.6918424963951111,
41
+ -0.9924377799034119,
42
+ -0.7879054546356201,
43
+ -0.9999184608459473
44
+ ],
45
+ "max": [
46
+ 0.9835734963417053,
47
+ 0.9999964237213135,
48
+ 0.9999682903289795,
49
+ 0.9999796152114868,
50
+ 0.9999924302101135,
51
+ 0.6493740081787109
52
+ ],
53
+ "mean": [
54
+ -0.03983950708061457,
55
+ 0.6419177949428558,
56
+ 0.5731318891048431,
57
+ -0.026727666932856664,
58
+ 0.5000582784414291,
59
+ -0.604814887046814
60
+ ],
61
+ "std": [
62
+ 0.2847598909065845,
63
+ 0.28045088125818984,
64
+ 0.31326231949726213,
65
+ 0.45434662590667196,
66
+ 0.3377052674297327,
67
+ 0.2508940489966109
68
+ ],
69
+ "q01": [
70
+ -0.8073391282558441,
71
+ -0.28321926176548007,
72
+ -0.3514450192451477,
73
+ -0.8703053236007691,
74
+ -0.36040452182292937,
75
+ -0.9848381114006043
76
+ ],
77
+ "q99": [
78
+ 0.8525016677379608,
79
+ 0.9961881709098815,
80
+ 0.9932036983966828,
81
+ 0.8807504057884216,
82
+ 0.9969734346866608,
83
+ 0.25840330481529217
84
+ ]
85
+ },
86
+ "hand_q": {
87
+ "min": [
88
+ -0.2837759256362915,
89
+ -0.45191138982772827,
90
+ -0.26049521565437317,
91
+ -0.07302045077085495,
92
+ -0.17368702590465546,
93
+ 0.0
94
+ ],
95
+ "max": [
96
+ 1.5965672731399536,
97
+ 1.5700000524520874,
98
+ 1.6704024076461792,
99
+ 1.5760689973831177,
100
+ 1.1547459363937378,
101
+ 1.8795303106307983
102
+ ],
103
+ "mean": [
104
+ 0.5675551295280457,
105
+ 0.4572437107563019,
106
+ 0.4543878138065338,
107
+ 0.5510742962360382,
108
+ 0.14924978837370872,
109
+ 1.5305395722389221
110
+ ],
111
+ "std": [
112
+ 0.5927769574172551,
113
+ 0.4676759057931687,
114
+ 0.4129692615651924,
115
+ 0.4221353649124037,
116
+ 0.18591654007078529,
117
+ 0.3333518844437281
118
+ ],
119
+ "q01": [
120
+ -0.10000000149011612,
121
+ -0.10000000149011612,
122
+ -0.06524610489606858,
123
+ -0.0009640824561938644,
124
+ -0.17368702590465546,
125
+ 0.5725240767002105
126
+ ],
127
+ "q99": [
128
+ 1.5965672731399536,
129
+ 1.495900900363922,
130
+ 1.4788811016082763,
131
+ 1.4855934596061706,
132
+ 0.6264129543304442,
133
+ 1.8216639709472657
134
+ ]
135
+ }
136
+ },
137
+ "action": {
138
+ "wrist_pos": {
139
+ "min": [
140
+ -0.12936705350875854,
141
+ -0.667879045009613,
142
+ -0.011580604128539562
143
+ ],
144
+ "max": [
145
+ 0.5296112895011902,
146
+ 0.12853454053401947,
147
+ 0.6460531949996948
148
+ ],
149
+ "mean": [
150
+ 0.29538029432296753,
151
+ -0.24184688925743103,
152
+ 0.25390544533729553
153
+ ],
154
+ "std": [
155
+ 0.09732441397601972,
156
+ 0.11346623731458663,
157
+ 0.07663734341862564
158
+ ],
159
+ "q01": [
160
+ 0.07942652076482773,
161
+ -0.5666251635551453,
162
+ 0.08935761481523515
163
+ ],
164
+ "q99": [
165
+ 0.48779279112815854,
166
+ 0.058025030642747714,
167
+ 0.5291574490070342
168
+ ]
169
+ },
170
+ "wrist_rot6d": {
171
+ "min": [
172
+ -0.9558911919593811,
173
+ -0.7119177579879761,
174
+ -0.6918424963951111,
175
+ -0.9924377799034119,
176
+ -0.7879054546356201,
177
+ -0.9999184608459473
178
+ ],
179
+ "max": [
180
+ 0.9835734963417053,
181
+ 0.9999964237213135,
182
+ 0.9999682903289795,
183
+ 0.9999796152114868,
184
+ 0.9999924302101135,
185
+ 0.6493740081787109
186
+ ],
187
+ "mean": [
188
+ -0.03960566408932209,
189
+ 0.6426223814487457,
190
+ 0.5729012787342072,
191
+ -0.02567985962377861,
192
+ 0.49940645694732666,
193
+ -0.605372428894043
194
+ ],
195
+ "std": [
196
+ 0.2846284723210341,
197
+ 0.27962233902347716,
198
+ 0.31312880692036205,
199
+ 0.4549785151467235,
200
+ 0.33742383129815395,
201
+ 0.25019110492920466
202
+ ],
203
+ "q01": [
204
+ -0.8073391282558441,
205
+ -0.28380464553833007,
206
+ -0.3514450192451477,
207
+ -0.8703053236007691,
208
+ -0.36094334125518795,
209
+ -0.9848381114006043
210
+ ],
211
+ "q99": [
212
+ 0.8525016677379608,
213
+ 0.9961881709098815,
214
+ 0.9928955614566803,
215
+ 0.8855576586723325,
216
+ 0.9968215143680572,
217
+ 0.2598367834091184
218
+ ]
219
+ },
220
+ "hand_q": {
221
+ "min": [
222
+ -0.2837759256362915,
223
+ -0.45191138982772827,
224
+ -0.26049521565437317,
225
+ -0.07302045077085495,
226
+ -0.17368702590465546,
227
+ 0.0
228
+ ],
229
+ "max": [
230
+ 1.5965672731399536,
231
+ 1.5700000524520874,
232
+ 1.6704024076461792,
233
+ 1.5760689973831177,
234
+ 1.1547459363937378,
235
+ 1.8795303106307983
236
+ ],
237
+ "mean": [
238
+ 0.5677891075611115,
239
+ 0.4574934244155884,
240
+ 0.4546465426683426,
241
+ 0.5513266623020172,
242
+ 0.1490255482494831,
243
+ 1.5341996550559998
244
+ ],
245
+ "std": [
246
+ 0.5926638319686875,
247
+ 0.4675603543415329,
248
+ 0.4127946067721787,
249
+ 0.42194973501693317,
250
+ 0.18543004949987615,
251
+ 0.3262138163984361
252
+ ],
253
+ "q01": [
254
+ -0.10000000149011612,
255
+ -0.10000000149011612,
256
+ -0.0632270258665085,
257
+ -0.0009655289549846202,
258
+ -0.17368702590465546,
259
+ 0.6341138243675232
260
+ ],
261
+ "q99": [
262
+ 1.5965672731399536,
263
+ 1.496000726222992,
264
+ 1.4788811016082763,
265
+ 1.4856495332717896,
266
+ 0.6264129543304442,
267
+ 1.8220702290534974
268
+ ]
269
+ }
270
+ }
271
+ }
272
+ }
experiment_cfg/final_model_config.json ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model_type": "RLDX-1",
3
+ "model_dtype": "bfloat16",
4
+ "model_name": "RLWRLD/RLDX-1-VLM",
5
+ "backbone_model_type": "vtc_qwen3_vl",
6
+ "model_revision": null,
7
+ "tune_top_llm_layers": 0,
8
+ "backbone_embedding_dim": 4096,
9
+ "tune_llm": false,
10
+ "tune_visual": false,
11
+ "select_layer": 18,
12
+ "reproject_vision": false,
13
+ "use_flash_attention": true,
14
+ "load_bf16": true,
15
+ "backbone_trainable_params_fp32": true,
16
+ "freeze_cog_tokens": false,
17
+ "keep_full_lm": false,
18
+ "vlm_lang_loss": false,
19
+ "vlm_lang_weight": 0.5,
20
+ "image_max_area": 65536,
21
+ "image_resize_m": 32,
22
+ "apply_sincos_state_encoding": false,
23
+ "use_relative_action": true,
24
+ "use_percentiles": true,
25
+ "conversation_image_first": false,
26
+ "max_state_dim": 64,
27
+ "max_action_dim": 64,
28
+ "action_horizon": 16,
29
+ "hidden_size": 1024,
30
+ "input_embedding_dim": 1536,
31
+ "general_embodiment_train_ratio": 0,
32
+ "add_pos_embed": true,
33
+ "attn_dropout": 0.2,
34
+ "use_vlln": false,
35
+ "max_seq_len": 1024,
36
+ "n_cog_tokens": 64,
37
+ "diffusion_model_cfg": {
38
+ "attention_head_dim": 64,
39
+ "depth_multi_stream": 4,
40
+ "depth_single_stream": 8,
41
+ "dropout": 0.2,
42
+ "final_dropout": true,
43
+ "num_attention_heads": 24,
44
+ "output_dim": 1024,
45
+ "positional_embeddings": "rope_sa_only",
46
+ "rope_theta": 10000.0,
47
+ "temb_type": "input_token",
48
+ "use_swiglu": true,
49
+ "action_model_max_seq_len": 512,
50
+ "pre_norm": "layer_norm",
51
+ "qk_norm": "rms_norm",
52
+ "sa_dim": 1536,
53
+ "vl_dim": 4096,
54
+ "use_physics": false,
55
+ "physics_dim": 0
56
+ },
57
+ "num_inference_timesteps": 4,
58
+ "noise_beta_alpha": 1.5,
59
+ "noise_beta_beta": 1.0,
60
+ "noise_s": 0.999,
61
+ "num_timestep_buckets": 1000,
62
+ "tune_projector": true,
63
+ "tune_diffusion_model": true,
64
+ "tune_vlln": true,
65
+ "action_model_use_lora": false,
66
+ "action_model_lora_rank": 16,
67
+ "action_model_lora_alpha": 32,
68
+ "action_model_lora_dropout": 0.0,
69
+ "action_model_lora_target_modules": [
70
+ "vl_qkv",
71
+ "vl_proj",
72
+ "sa_qkv",
73
+ "sa_proj",
74
+ "p_qkv",
75
+ "p_proj",
76
+ "linear1",
77
+ "linear2"
78
+ ],
79
+ "backbone_use_lora": true,
80
+ "backbone_lora_rank": 16,
81
+ "backbone_lora_alpha": 32,
82
+ "backbone_lora_dropout": 0.0,
83
+ "backbone_lora_num_layers": -1,
84
+ "backbone_lora_target_modules": [
85
+ "q_proj",
86
+ "k_proj",
87
+ "v_proj",
88
+ "o_proj",
89
+ "gate_proj",
90
+ "up_proj",
91
+ "down_proj"
92
+ ],
93
+ "state_dropout_prob": 0.0,
94
+ "state_additive_noise_scale": 0.0,
95
+ "rtc_training_max_delay": 0,
96
+ "rtc_inference_mode": "none",
97
+ "rtc_inference_delay": 0,
98
+ "rtc_inference_exec_horizon": 0,
99
+ "rtc_jacobian_steps_only": 3,
100
+ "use_memory": false,
101
+ "memory_length": 4,
102
+ "memory_n_cog_tokens": null,
103
+ "concat_memory": false,
104
+ "memory_dropout_prob": 0.0,
105
+ "memory_stride": 16,
106
+ "memory_cfg": {
107
+ "hidden_size": 4096,
108
+ "intermediate_size": 16384,
109
+ "num_hidden_layers": 2,
110
+ "num_attention_heads": 16,
111
+ "num_key_value_heads": 16,
112
+ "max_position_embeddings": 32,
113
+ "rms_norm_eps": 1e-05,
114
+ "use_causal_attn": true,
115
+ "use_rope": true
116
+ },
117
+ "use_motion": false,
118
+ "motion_insert_layer": 9,
119
+ "motion_d_hid": 512,
120
+ "motion_window": [
121
+ 5,
122
+ 9,
123
+ 9
124
+ ],
125
+ "motion_ext_chnls": [
126
+ 256
127
+ ],
128
+ "motion_int_chnls": [
129
+ 256,
130
+ 256,
131
+ 512
132
+ ],
133
+ "motion_corr_func": "cosine",
134
+ "motion_n_encoders": 1,
135
+ "motion_use_layerscale": false,
136
+ "motion_layerscale_init": 1e-05,
137
+ "motion_use_layernorm": false,
138
+ "motion_use_syncbn": false,
139
+ "motion_injection_point": "vision_encoder",
140
+ "motion_pool_type": "avg",
141
+ "motion_drop": true,
142
+ "motion_gradient_check": false,
143
+ "motion_int_mode": "lite",
144
+ "use_video": true,
145
+ "video_length": 1,
146
+ "video_stride": 2,
147
+ "use_physics": false,
148
+ "physics_keys": [],
149
+ "physics_dims": [],
150
+ "physics_loss_weight": 0.1,
151
+ "allow_missing_physics": false,
152
+ "physics_delta_indices": null,
153
+ "physics_use_flow_matching": true,
154
+ "physics_dropout_prob": 0.0,
155
+ "use_finger_latent": false,
156
+ "finger_wrist_mode": "delta",
157
+ "finger_bimanual": false,
158
+ "finger_alignment_npz": "",
159
+ "finger_fingertip_norm_npz": "",
160
+ "finger_wrist_delta_stats_npz": "",
161
+ "finger_wrist_abs_stats_npz": "",
162
+ "embod_reg_mode": "",
163
+ "embod_reg_weight": 0.05,
164
+ "embod_reg_layer": 2,
165
+ "embod_reg_gather": false,
166
+ "embod_reg_tbins": 0,
167
+ "embod_reg_vic_var": 1.0,
168
+ "embod_reg_vic_cov": 0.04,
169
+ "embod_reg_hand_bins": 0,
170
+ "embod_reg_hand_edges": "",
171
+ "anchor_dim_norm": "",
172
+ "anchor_dim_stats_path": "",
173
+ "anchor_tips_dims": 15,
174
+ "anchor_human_tips_weight": 1.0,
175
+ "actlat_mode": false,
176
+ "actlat_tokenizer_path": "",
177
+ "actlat_embodiment_id": "openarm_prq",
178
+ "actlat_prq_stats_path": "",
179
+ "actlat_standardize": false,
180
+ "actlat_z_stats_path": "",
181
+ "anchor_readout_weight": 0.0,
182
+ "anchor_readout_taps": "cog,d2,s3,s7",
183
+ "anchor_readout_dim": 24,
184
+ "anchor_readout_tweight": true,
185
+ "anchor_readout_shuffle": false,
186
+ "anchor_readout_asym": false,
187
+ "gr1hp_align_weight": 0.2,
188
+ "gr1hp_align_only": true,
189
+ "gr1hp_align_dim": 24,
190
+ "gr1hp_align_shuffle": false,
191
+ "geom_bn_mode": "",
192
+ "geom_bn_weight": 0.0,
193
+ "geom_bn_dim": 24,
194
+ "geom_bn_asym": true,
195
+ "geom_bn_ema": 0.0,
196
+ "geom_bn_shuffle": false,
197
+ "geom_bn_lift_hidden": 0,
198
+ "geom_bn_route_warmup_steps": 0,
199
+ "geom_effort_dim": 0,
200
+ "geom_effort_weight": 0.0,
201
+ "egopi_split_embodiment": false,
202
+ "gr1hp_split_embodiment": false,
203
+ "train_stage": "",
204
+ "stage_from_checkpoint": "",
205
+ "freeze_modules": "",
206
+ "max_num_embodiments": 36
207
+ }
experiment_cfg/final_processor_config.json ADDED
The diff for this file is too large to render. See raw diff
 
model-00001-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f3479e59b9df643ee9c2019b2db84db454975883bde0f3d88a189a68934ddcd5
3
+ size 4928482688
model-00002-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fccf042abc0c77577f8bd88cd201af033971d7ab2e09a6331fa4ecbbb859d74d
3
+ size 4473941792
model-00003-of-00003.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c20750c9991150348096059236ed634339896361fef4b196b31af43f8c62bdbb
3
+ size 4467352456
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
processor/embodiment_id.json ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "general_embodiment": 0,
3
+ "fractal20220817_data": 1,
4
+ "kuka": 2,
5
+ "bridge_orig": 3,
6
+ "taco_play": 4,
7
+ "jaco_play": 5,
8
+ "berkeley_cable_routing": 6,
9
+ "roboturk": 7,
10
+ "viola": 8,
11
+ "berkeley_autolab_ur5": 9,
12
+ "toto": 10,
13
+ "language_table": 11,
14
+ "stanford_hydra_dataset_converted_externally_to_rlds": 12,
15
+ "austin_buds_dataset_converted_externally_to_rlds": 13,
16
+ "nyu_franka_play_dataset_converted_externally_to_rlds": 14,
17
+ "furniture_bench_dataset_converted_externally_to_rlds": 15,
18
+ "ucsd_kitchen_dataset_converted_externally_to_rlds": 16,
19
+ "austin_sailor_dataset_converted_externally_to_rlds": 17,
20
+ "austin_sirius_dataset_converted_externally_to_rlds": 18,
21
+ "dlr_edan_shared_control_converted_externally_to_rlds": 19,
22
+ "iamlab_cmu_pickup_insert_converted_externally_to_rlds": 20,
23
+ "utaustin_mutex": 21,
24
+ "berkeley_fanuc_manipulation": 22,
25
+ "cmu_stretch": 23,
26
+ "bc_z": 24,
27
+ "fmb_dataset": 25,
28
+ "dobbe": 26,
29
+ "droid": 27,
30
+ "agibot_dexhand": 28,
31
+ "agibot_gripper": 29,
32
+ "galaxea": 30,
33
+ "humanoid_everyday_g1": 31,
34
+ "humanoid_everyday_h1": 32,
35
+ "action_net": 33,
36
+ "neural_gr1": 34,
37
+ "new_embodiment": 35,
38
+ "egopi": 35,
39
+ "egopi_human": 11,
40
+ "hawor": 35,
41
+ "gr1_prq": 35,
42
+ "gr1_hawor": 35,
43
+ "gr1_human_prq": 35,
44
+ "gr1_human_prq_human": 12
45
+ }
processor/processor_config.json ADDED
@@ -0,0 +1,2963 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "processor_class": "RLDXProcessor",
3
+ "processor_kwargs": {
4
+ "modality_configs": {
5
+ "agibot_gripper": {
6
+ "video": {
7
+ "delta_indices": [
8
+ 0
9
+ ],
10
+ "modality_keys": [
11
+ "primary",
12
+ "wrist_left",
13
+ "wrist_right"
14
+ ],
15
+ "sin_cos_embedding_keys": null,
16
+ "mean_std_embedding_keys": null,
17
+ "action_configs": null
18
+ },
19
+ "state": {
20
+ "delta_indices": [
21
+ 0
22
+ ],
23
+ "modality_keys": [
24
+ "state"
25
+ ],
26
+ "sin_cos_embedding_keys": null,
27
+ "mean_std_embedding_keys": null,
28
+ "action_configs": null
29
+ },
30
+ "action": {
31
+ "delta_indices": [
32
+ 0,
33
+ 1,
34
+ 2,
35
+ 3,
36
+ 4,
37
+ 5,
38
+ 6,
39
+ 7,
40
+ 8,
41
+ 9,
42
+ 10,
43
+ 11,
44
+ 12,
45
+ 13,
46
+ 14,
47
+ 15
48
+ ],
49
+ "modality_keys": [
50
+ "action"
51
+ ],
52
+ "sin_cos_embedding_keys": null,
53
+ "mean_std_embedding_keys": null,
54
+ "action_configs": [
55
+ {
56
+ "rep": "ABSOLUTE",
57
+ "type": "NON_EEF",
58
+ "format": "DEFAULT",
59
+ "state_key": null
60
+ }
61
+ ]
62
+ },
63
+ "language": {
64
+ "delta_indices": [
65
+ 0
66
+ ],
67
+ "modality_keys": [
68
+ "annotation.human.action.task_description"
69
+ ],
70
+ "sin_cos_embedding_keys": null,
71
+ "mean_std_embedding_keys": null,
72
+ "action_configs": null
73
+ }
74
+ },
75
+ "dlr_edan_shared_control_converted_externally_to_rlds": {
76
+ "video": {
77
+ "delta_indices": [
78
+ 0
79
+ ],
80
+ "modality_keys": [
81
+ "primary"
82
+ ],
83
+ "sin_cos_embedding_keys": null,
84
+ "mean_std_embedding_keys": null,
85
+ "action_configs": null
86
+ },
87
+ "state": {
88
+ "delta_indices": [
89
+ 0
90
+ ],
91
+ "modality_keys": [
92
+ "end_effector_position",
93
+ "end_effector_rotation",
94
+ "gripper_position"
95
+ ],
96
+ "sin_cos_embedding_keys": null,
97
+ "mean_std_embedding_keys": null,
98
+ "action_configs": null
99
+ },
100
+ "action": {
101
+ "delta_indices": [
102
+ 0,
103
+ 1,
104
+ 2,
105
+ 3,
106
+ 4,
107
+ 5,
108
+ 6,
109
+ 7,
110
+ 8,
111
+ 9,
112
+ 10,
113
+ 11,
114
+ 12,
115
+ 13,
116
+ 14,
117
+ 15
118
+ ],
119
+ "modality_keys": [
120
+ "end_effector_position",
121
+ "end_effector_rotation",
122
+ "gripper_close"
123
+ ],
124
+ "sin_cos_embedding_keys": null,
125
+ "mean_std_embedding_keys": null,
126
+ "action_configs": [
127
+ {
128
+ "rep": "DELTA",
129
+ "type": "EEF",
130
+ "format": "DEFAULT",
131
+ "state_key": null
132
+ },
133
+ {
134
+ "rep": "DELTA",
135
+ "type": "EEF",
136
+ "format": "DEFAULT",
137
+ "state_key": null
138
+ },
139
+ {
140
+ "rep": "ABSOLUTE",
141
+ "type": "NON_EEF",
142
+ "format": "DEFAULT",
143
+ "state_key": null
144
+ }
145
+ ]
146
+ },
147
+ "language": {
148
+ "delta_indices": [
149
+ 0
150
+ ],
151
+ "modality_keys": [
152
+ "annotation.human.action.task_description"
153
+ ],
154
+ "sin_cos_embedding_keys": null,
155
+ "mean_std_embedding_keys": null,
156
+ "action_configs": null
157
+ }
158
+ },
159
+ "humanoid_everyday_g1": {
160
+ "video": {
161
+ "delta_indices": [
162
+ 0
163
+ ],
164
+ "modality_keys": [
165
+ "egocentric_resized"
166
+ ],
167
+ "sin_cos_embedding_keys": null,
168
+ "mean_std_embedding_keys": null,
169
+ "action_configs": null
170
+ },
171
+ "state": {
172
+ "delta_indices": [
173
+ 0
174
+ ],
175
+ "modality_keys": [
176
+ "left_arm",
177
+ "left_hand",
178
+ "right_arm",
179
+ "right_hand"
180
+ ],
181
+ "sin_cos_embedding_keys": null,
182
+ "mean_std_embedding_keys": null,
183
+ "action_configs": null
184
+ },
185
+ "action": {
186
+ "delta_indices": [
187
+ 0,
188
+ 1,
189
+ 2,
190
+ 3,
191
+ 4,
192
+ 5,
193
+ 6,
194
+ 7,
195
+ 8,
196
+ 9,
197
+ 10,
198
+ 11,
199
+ 12,
200
+ 13,
201
+ 14,
202
+ 15
203
+ ],
204
+ "modality_keys": [
205
+ "left_arm",
206
+ "left_hand",
207
+ "right_arm",
208
+ "right_hand"
209
+ ],
210
+ "sin_cos_embedding_keys": null,
211
+ "mean_std_embedding_keys": null,
212
+ "action_configs": [
213
+ {
214
+ "rep": "ABSOLUTE",
215
+ "type": "NON_EEF",
216
+ "format": "DEFAULT",
217
+ "state_key": null
218
+ },
219
+ {
220
+ "rep": "ABSOLUTE",
221
+ "type": "NON_EEF",
222
+ "format": "DEFAULT",
223
+ "state_key": null
224
+ },
225
+ {
226
+ "rep": "ABSOLUTE",
227
+ "type": "NON_EEF",
228
+ "format": "DEFAULT",
229
+ "state_key": null
230
+ },
231
+ {
232
+ "rep": "ABSOLUTE",
233
+ "type": "NON_EEF",
234
+ "format": "DEFAULT",
235
+ "state_key": null
236
+ }
237
+ ]
238
+ },
239
+ "language": {
240
+ "delta_indices": [
241
+ 0
242
+ ],
243
+ "modality_keys": [
244
+ "annotation.human.action.task_description"
245
+ ],
246
+ "sin_cos_embedding_keys": null,
247
+ "mean_std_embedding_keys": null,
248
+ "action_configs": null
249
+ }
250
+ },
251
+ "austin_sirius_dataset_converted_externally_to_rlds": {
252
+ "video": {
253
+ "delta_indices": [
254
+ 0
255
+ ],
256
+ "modality_keys": [
257
+ "primary",
258
+ "wrist"
259
+ ],
260
+ "sin_cos_embedding_keys": null,
261
+ "mean_std_embedding_keys": null,
262
+ "action_configs": null
263
+ },
264
+ "state": {
265
+ "delta_indices": [
266
+ 0
267
+ ],
268
+ "modality_keys": [
269
+ "end_effector_position",
270
+ "end_effector_rotation",
271
+ "gripper_position"
272
+ ],
273
+ "sin_cos_embedding_keys": null,
274
+ "mean_std_embedding_keys": null,
275
+ "action_configs": null
276
+ },
277
+ "action": {
278
+ "delta_indices": [
279
+ 0,
280
+ 1,
281
+ 2,
282
+ 3,
283
+ 4,
284
+ 5,
285
+ 6,
286
+ 7,
287
+ 8,
288
+ 9,
289
+ 10,
290
+ 11,
291
+ 12,
292
+ 13,
293
+ 14,
294
+ 15
295
+ ],
296
+ "modality_keys": [
297
+ "end_effector_position",
298
+ "end_effector_rotation",
299
+ "gripper_close"
300
+ ],
301
+ "sin_cos_embedding_keys": null,
302
+ "mean_std_embedding_keys": null,
303
+ "action_configs": [
304
+ {
305
+ "rep": "DELTA",
306
+ "type": "EEF",
307
+ "format": "DEFAULT",
308
+ "state_key": null
309
+ },
310
+ {
311
+ "rep": "DELTA",
312
+ "type": "EEF",
313
+ "format": "DEFAULT",
314
+ "state_key": null
315
+ },
316
+ {
317
+ "rep": "ABSOLUTE",
318
+ "type": "NON_EEF",
319
+ "format": "DEFAULT",
320
+ "state_key": null
321
+ }
322
+ ]
323
+ },
324
+ "language": {
325
+ "delta_indices": [
326
+ 0
327
+ ],
328
+ "modality_keys": [
329
+ "annotation.human.action.task_description"
330
+ ],
331
+ "sin_cos_embedding_keys": null,
332
+ "mean_std_embedding_keys": null,
333
+ "action_configs": null
334
+ }
335
+ },
336
+ "jaco_play": {
337
+ "video": {
338
+ "delta_indices": [
339
+ 0
340
+ ],
341
+ "modality_keys": [
342
+ "primary",
343
+ "wrist"
344
+ ],
345
+ "sin_cos_embedding_keys": null,
346
+ "mean_std_embedding_keys": null,
347
+ "action_configs": null
348
+ },
349
+ "state": {
350
+ "delta_indices": [
351
+ 0
352
+ ],
353
+ "modality_keys": [
354
+ "end_effector_position",
355
+ "end_effector_rotation",
356
+ "gripper_position"
357
+ ],
358
+ "sin_cos_embedding_keys": null,
359
+ "mean_std_embedding_keys": null,
360
+ "action_configs": null
361
+ },
362
+ "action": {
363
+ "delta_indices": [
364
+ 0,
365
+ 1,
366
+ 2,
367
+ 3,
368
+ 4,
369
+ 5,
370
+ 6,
371
+ 7,
372
+ 8,
373
+ 9,
374
+ 10,
375
+ 11,
376
+ 12,
377
+ 13,
378
+ 14,
379
+ 15
380
+ ],
381
+ "modality_keys": [
382
+ "end_effector_position",
383
+ "end_effector_rotation",
384
+ "gripper_close"
385
+ ],
386
+ "sin_cos_embedding_keys": null,
387
+ "mean_std_embedding_keys": null,
388
+ "action_configs": [
389
+ {
390
+ "rep": "DELTA",
391
+ "type": "EEF",
392
+ "format": "DEFAULT",
393
+ "state_key": null
394
+ },
395
+ {
396
+ "rep": "DELTA",
397
+ "type": "EEF",
398
+ "format": "DEFAULT",
399
+ "state_key": null
400
+ },
401
+ {
402
+ "rep": "ABSOLUTE",
403
+ "type": "NON_EEF",
404
+ "format": "DEFAULT",
405
+ "state_key": null
406
+ }
407
+ ]
408
+ },
409
+ "language": {
410
+ "delta_indices": [
411
+ 0
412
+ ],
413
+ "modality_keys": [
414
+ "annotation.human.action.task_description"
415
+ ],
416
+ "sin_cos_embedding_keys": null,
417
+ "mean_std_embedding_keys": null,
418
+ "action_configs": null
419
+ }
420
+ },
421
+ "cmu_stretch": {
422
+ "video": {
423
+ "delta_indices": [
424
+ 0
425
+ ],
426
+ "modality_keys": [
427
+ "primary"
428
+ ],
429
+ "sin_cos_embedding_keys": null,
430
+ "mean_std_embedding_keys": null,
431
+ "action_configs": null
432
+ },
433
+ "state": {
434
+ "delta_indices": [
435
+ 0
436
+ ],
437
+ "modality_keys": [
438
+ "end_effector_position",
439
+ "end_effector_rotation",
440
+ "gripper_position"
441
+ ],
442
+ "sin_cos_embedding_keys": null,
443
+ "mean_std_embedding_keys": null,
444
+ "action_configs": null
445
+ },
446
+ "action": {
447
+ "delta_indices": [
448
+ 0,
449
+ 1,
450
+ 2,
451
+ 3,
452
+ 4,
453
+ 5,
454
+ 6,
455
+ 7,
456
+ 8,
457
+ 9,
458
+ 10,
459
+ 11,
460
+ 12,
461
+ 13,
462
+ 14,
463
+ 15
464
+ ],
465
+ "modality_keys": [
466
+ "end_effector_position",
467
+ "end_effector_rotation",
468
+ "gripper_close"
469
+ ],
470
+ "sin_cos_embedding_keys": null,
471
+ "mean_std_embedding_keys": null,
472
+ "action_configs": [
473
+ {
474
+ "rep": "DELTA",
475
+ "type": "EEF",
476
+ "format": "DEFAULT",
477
+ "state_key": null
478
+ },
479
+ {
480
+ "rep": "DELTA",
481
+ "type": "EEF",
482
+ "format": "DEFAULT",
483
+ "state_key": null
484
+ },
485
+ {
486
+ "rep": "ABSOLUTE",
487
+ "type": "NON_EEF",
488
+ "format": "DEFAULT",
489
+ "state_key": null
490
+ }
491
+ ]
492
+ },
493
+ "language": {
494
+ "delta_indices": [
495
+ 0
496
+ ],
497
+ "modality_keys": [
498
+ "annotation.human.action.task_description"
499
+ ],
500
+ "sin_cos_embedding_keys": null,
501
+ "mean_std_embedding_keys": null,
502
+ "action_configs": null
503
+ }
504
+ },
505
+ "fractal20220817_data": {
506
+ "video": {
507
+ "delta_indices": [
508
+ 0
509
+ ],
510
+ "modality_keys": [
511
+ "primary"
512
+ ],
513
+ "sin_cos_embedding_keys": null,
514
+ "mean_std_embedding_keys": null,
515
+ "action_configs": null
516
+ },
517
+ "state": {
518
+ "delta_indices": [
519
+ 0
520
+ ],
521
+ "modality_keys": [
522
+ "end_effector_position",
523
+ "end_effector_rotation",
524
+ "gripper_position"
525
+ ],
526
+ "sin_cos_embedding_keys": null,
527
+ "mean_std_embedding_keys": null,
528
+ "action_configs": null
529
+ },
530
+ "action": {
531
+ "delta_indices": [
532
+ 0,
533
+ 1,
534
+ 2,
535
+ 3,
536
+ 4,
537
+ 5,
538
+ 6,
539
+ 7,
540
+ 8,
541
+ 9,
542
+ 10,
543
+ 11,
544
+ 12,
545
+ 13,
546
+ 14,
547
+ 15
548
+ ],
549
+ "modality_keys": [
550
+ "end_effector_position",
551
+ "end_effector_rotation",
552
+ "gripper_close"
553
+ ],
554
+ "sin_cos_embedding_keys": null,
555
+ "mean_std_embedding_keys": null,
556
+ "action_configs": [
557
+ {
558
+ "rep": "DELTA",
559
+ "type": "EEF",
560
+ "format": "DEFAULT",
561
+ "state_key": null
562
+ },
563
+ {
564
+ "rep": "DELTA",
565
+ "type": "EEF",
566
+ "format": "DEFAULT",
567
+ "state_key": null
568
+ },
569
+ {
570
+ "rep": "ABSOLUTE",
571
+ "type": "NON_EEF",
572
+ "format": "DEFAULT",
573
+ "state_key": null
574
+ }
575
+ ]
576
+ },
577
+ "language": {
578
+ "delta_indices": [
579
+ 0
580
+ ],
581
+ "modality_keys": [
582
+ "annotation.human.action.task_description"
583
+ ],
584
+ "sin_cos_embedding_keys": null,
585
+ "mean_std_embedding_keys": null,
586
+ "action_configs": null
587
+ }
588
+ },
589
+ "berkeley_autolab_ur5": {
590
+ "video": {
591
+ "delta_indices": [
592
+ 0
593
+ ],
594
+ "modality_keys": [
595
+ "primary",
596
+ "wrist"
597
+ ],
598
+ "sin_cos_embedding_keys": null,
599
+ "mean_std_embedding_keys": null,
600
+ "action_configs": null
601
+ },
602
+ "state": {
603
+ "delta_indices": [
604
+ 0
605
+ ],
606
+ "modality_keys": [
607
+ "end_effector_position",
608
+ "end_effector_rotation",
609
+ "gripper_position"
610
+ ],
611
+ "sin_cos_embedding_keys": null,
612
+ "mean_std_embedding_keys": null,
613
+ "action_configs": null
614
+ },
615
+ "action": {
616
+ "delta_indices": [
617
+ 0,
618
+ 1,
619
+ 2,
620
+ 3,
621
+ 4,
622
+ 5,
623
+ 6,
624
+ 7,
625
+ 8,
626
+ 9,
627
+ 10,
628
+ 11,
629
+ 12,
630
+ 13,
631
+ 14,
632
+ 15
633
+ ],
634
+ "modality_keys": [
635
+ "end_effector_position",
636
+ "end_effector_rotation",
637
+ "gripper_close"
638
+ ],
639
+ "sin_cos_embedding_keys": null,
640
+ "mean_std_embedding_keys": null,
641
+ "action_configs": [
642
+ {
643
+ "rep": "DELTA",
644
+ "type": "EEF",
645
+ "format": "DEFAULT",
646
+ "state_key": null
647
+ },
648
+ {
649
+ "rep": "DELTA",
650
+ "type": "EEF",
651
+ "format": "DEFAULT",
652
+ "state_key": null
653
+ },
654
+ {
655
+ "rep": "ABSOLUTE",
656
+ "type": "NON_EEF",
657
+ "format": "DEFAULT",
658
+ "state_key": null
659
+ }
660
+ ]
661
+ },
662
+ "language": {
663
+ "delta_indices": [
664
+ 0
665
+ ],
666
+ "modality_keys": [
667
+ "annotation.human.action.task_description"
668
+ ],
669
+ "sin_cos_embedding_keys": null,
670
+ "mean_std_embedding_keys": null,
671
+ "action_configs": null
672
+ }
673
+ },
674
+ "language_table": {
675
+ "video": {
676
+ "delta_indices": [
677
+ 0
678
+ ],
679
+ "modality_keys": [
680
+ "primary"
681
+ ],
682
+ "sin_cos_embedding_keys": null,
683
+ "mean_std_embedding_keys": null,
684
+ "action_configs": null
685
+ },
686
+ "state": {
687
+ "delta_indices": [
688
+ 0
689
+ ],
690
+ "modality_keys": [
691
+ "end_effector_position"
692
+ ],
693
+ "sin_cos_embedding_keys": null,
694
+ "mean_std_embedding_keys": null,
695
+ "action_configs": null
696
+ },
697
+ "action": {
698
+ "delta_indices": [
699
+ 0,
700
+ 1,
701
+ 2,
702
+ 3,
703
+ 4,
704
+ 5,
705
+ 6,
706
+ 7,
707
+ 8,
708
+ 9,
709
+ 10,
710
+ 11,
711
+ 12,
712
+ 13,
713
+ 14,
714
+ 15
715
+ ],
716
+ "modality_keys": [
717
+ "end_effector_position"
718
+ ],
719
+ "sin_cos_embedding_keys": null,
720
+ "mean_std_embedding_keys": null,
721
+ "action_configs": [
722
+ {
723
+ "rep": "DELTA",
724
+ "type": "EEF",
725
+ "format": "DEFAULT",
726
+ "state_key": null
727
+ }
728
+ ]
729
+ },
730
+ "language": {
731
+ "delta_indices": [
732
+ 0
733
+ ],
734
+ "modality_keys": [
735
+ "annotation.human.action.task_description"
736
+ ],
737
+ "sin_cos_embedding_keys": null,
738
+ "mean_std_embedding_keys": null,
739
+ "action_configs": null
740
+ }
741
+ },
742
+ "viola": {
743
+ "video": {
744
+ "delta_indices": [
745
+ 0
746
+ ],
747
+ "modality_keys": [
748
+ "primary",
749
+ "wrist"
750
+ ],
751
+ "sin_cos_embedding_keys": null,
752
+ "mean_std_embedding_keys": null,
753
+ "action_configs": null
754
+ },
755
+ "state": {
756
+ "delta_indices": [
757
+ 0
758
+ ],
759
+ "modality_keys": [
760
+ "joint_position",
761
+ "gripper_position"
762
+ ],
763
+ "sin_cos_embedding_keys": null,
764
+ "mean_std_embedding_keys": null,
765
+ "action_configs": null
766
+ },
767
+ "action": {
768
+ "delta_indices": [
769
+ 0,
770
+ 1,
771
+ 2,
772
+ 3,
773
+ 4,
774
+ 5,
775
+ 6,
776
+ 7,
777
+ 8,
778
+ 9,
779
+ 10,
780
+ 11,
781
+ 12,
782
+ 13,
783
+ 14,
784
+ 15
785
+ ],
786
+ "modality_keys": [
787
+ "end_effector_position",
788
+ "end_effector_rotation",
789
+ "gripper_close"
790
+ ],
791
+ "sin_cos_embedding_keys": null,
792
+ "mean_std_embedding_keys": null,
793
+ "action_configs": [
794
+ {
795
+ "rep": "DELTA",
796
+ "type": "EEF",
797
+ "format": "DEFAULT",
798
+ "state_key": null
799
+ },
800
+ {
801
+ "rep": "DELTA",
802
+ "type": "EEF",
803
+ "format": "DEFAULT",
804
+ "state_key": null
805
+ },
806
+ {
807
+ "rep": "ABSOLUTE",
808
+ "type": "NON_EEF",
809
+ "format": "DEFAULT",
810
+ "state_key": null
811
+ }
812
+ ]
813
+ },
814
+ "language": {
815
+ "delta_indices": [
816
+ 0
817
+ ],
818
+ "modality_keys": [
819
+ "annotation.human.action.task_description"
820
+ ],
821
+ "sin_cos_embedding_keys": null,
822
+ "mean_std_embedding_keys": null,
823
+ "action_configs": null
824
+ }
825
+ },
826
+ "furniture_bench_dataset_converted_externally_to_rlds": {
827
+ "video": {
828
+ "delta_indices": [
829
+ 0
830
+ ],
831
+ "modality_keys": [
832
+ "primary",
833
+ "wrist"
834
+ ],
835
+ "sin_cos_embedding_keys": null,
836
+ "mean_std_embedding_keys": null,
837
+ "action_configs": null
838
+ },
839
+ "state": {
840
+ "delta_indices": [
841
+ 0
842
+ ],
843
+ "modality_keys": [
844
+ "end_effector_position",
845
+ "end_effector_rotation",
846
+ "gripper_position"
847
+ ],
848
+ "sin_cos_embedding_keys": null,
849
+ "mean_std_embedding_keys": null,
850
+ "action_configs": null
851
+ },
852
+ "action": {
853
+ "delta_indices": [
854
+ 0,
855
+ 1,
856
+ 2,
857
+ 3,
858
+ 4,
859
+ 5,
860
+ 6,
861
+ 7,
862
+ 8,
863
+ 9,
864
+ 10,
865
+ 11,
866
+ 12,
867
+ 13,
868
+ 14,
869
+ 15
870
+ ],
871
+ "modality_keys": [
872
+ "end_effector_position",
873
+ "end_effector_rotation",
874
+ "gripper_close"
875
+ ],
876
+ "sin_cos_embedding_keys": null,
877
+ "mean_std_embedding_keys": null,
878
+ "action_configs": [
879
+ {
880
+ "rep": "DELTA",
881
+ "type": "EEF",
882
+ "format": "DEFAULT",
883
+ "state_key": null
884
+ },
885
+ {
886
+ "rep": "DELTA",
887
+ "type": "EEF",
888
+ "format": "DEFAULT",
889
+ "state_key": null
890
+ },
891
+ {
892
+ "rep": "ABSOLUTE",
893
+ "type": "NON_EEF",
894
+ "format": "DEFAULT",
895
+ "state_key": null
896
+ }
897
+ ]
898
+ },
899
+ "language": {
900
+ "delta_indices": [
901
+ 0
902
+ ],
903
+ "modality_keys": [
904
+ "annotation.human.action.task_description"
905
+ ],
906
+ "sin_cos_embedding_keys": null,
907
+ "mean_std_embedding_keys": null,
908
+ "action_configs": null
909
+ }
910
+ },
911
+ "agibot_dexhand": {
912
+ "video": {
913
+ "delta_indices": [
914
+ 0
915
+ ],
916
+ "modality_keys": [
917
+ "primary"
918
+ ],
919
+ "sin_cos_embedding_keys": null,
920
+ "mean_std_embedding_keys": null,
921
+ "action_configs": null
922
+ },
923
+ "state": {
924
+ "delta_indices": [
925
+ 0
926
+ ],
927
+ "modality_keys": [
928
+ "state"
929
+ ],
930
+ "sin_cos_embedding_keys": null,
931
+ "mean_std_embedding_keys": null,
932
+ "action_configs": null
933
+ },
934
+ "action": {
935
+ "delta_indices": [
936
+ 0,
937
+ 1,
938
+ 2,
939
+ 3,
940
+ 4,
941
+ 5,
942
+ 6,
943
+ 7,
944
+ 8,
945
+ 9,
946
+ 10,
947
+ 11,
948
+ 12,
949
+ 13,
950
+ 14,
951
+ 15
952
+ ],
953
+ "modality_keys": [
954
+ "action"
955
+ ],
956
+ "sin_cos_embedding_keys": null,
957
+ "mean_std_embedding_keys": null,
958
+ "action_configs": [
959
+ {
960
+ "rep": "ABSOLUTE",
961
+ "type": "NON_EEF",
962
+ "format": "DEFAULT",
963
+ "state_key": null
964
+ }
965
+ ]
966
+ },
967
+ "language": {
968
+ "delta_indices": [
969
+ 0
970
+ ],
971
+ "modality_keys": [
972
+ "annotation.human.action.task_description"
973
+ ],
974
+ "sin_cos_embedding_keys": null,
975
+ "mean_std_embedding_keys": null,
976
+ "action_configs": null
977
+ }
978
+ },
979
+ "taco_play": {
980
+ "video": {
981
+ "delta_indices": [
982
+ 0
983
+ ],
984
+ "modality_keys": [
985
+ "primary",
986
+ "wrist"
987
+ ],
988
+ "sin_cos_embedding_keys": null,
989
+ "mean_std_embedding_keys": null,
990
+ "action_configs": null
991
+ },
992
+ "state": {
993
+ "delta_indices": [
994
+ 0
995
+ ],
996
+ "modality_keys": [
997
+ "end_effector_position",
998
+ "end_effector_rotation",
999
+ "gripper_position"
1000
+ ],
1001
+ "sin_cos_embedding_keys": null,
1002
+ "mean_std_embedding_keys": null,
1003
+ "action_configs": null
1004
+ },
1005
+ "action": {
1006
+ "delta_indices": [
1007
+ 0,
1008
+ 1,
1009
+ 2,
1010
+ 3,
1011
+ 4,
1012
+ 5,
1013
+ 6,
1014
+ 7,
1015
+ 8,
1016
+ 9,
1017
+ 10,
1018
+ 11,
1019
+ 12,
1020
+ 13,
1021
+ 14,
1022
+ 15
1023
+ ],
1024
+ "modality_keys": [
1025
+ "end_effector_position",
1026
+ "end_effector_rotation",
1027
+ "gripper_close"
1028
+ ],
1029
+ "sin_cos_embedding_keys": null,
1030
+ "mean_std_embedding_keys": null,
1031
+ "action_configs": [
1032
+ {
1033
+ "rep": "DELTA",
1034
+ "type": "EEF",
1035
+ "format": "DEFAULT",
1036
+ "state_key": null
1037
+ },
1038
+ {
1039
+ "rep": "DELTA",
1040
+ "type": "EEF",
1041
+ "format": "DEFAULT",
1042
+ "state_key": null
1043
+ },
1044
+ {
1045
+ "rep": "ABSOLUTE",
1046
+ "type": "NON_EEF",
1047
+ "format": "DEFAULT",
1048
+ "state_key": null
1049
+ }
1050
+ ]
1051
+ },
1052
+ "language": {
1053
+ "delta_indices": [
1054
+ 0
1055
+ ],
1056
+ "modality_keys": [
1057
+ "annotation.human.action.task_description"
1058
+ ],
1059
+ "sin_cos_embedding_keys": null,
1060
+ "mean_std_embedding_keys": null,
1061
+ "action_configs": null
1062
+ }
1063
+ },
1064
+ "berkeley_fanuc_manipulation": {
1065
+ "video": {
1066
+ "delta_indices": [
1067
+ 0
1068
+ ],
1069
+ "modality_keys": [
1070
+ "primary",
1071
+ "wrist"
1072
+ ],
1073
+ "sin_cos_embedding_keys": null,
1074
+ "mean_std_embedding_keys": null,
1075
+ "action_configs": null
1076
+ },
1077
+ "state": {
1078
+ "delta_indices": [
1079
+ 0
1080
+ ],
1081
+ "modality_keys": [
1082
+ "joint_position",
1083
+ "gripper_position"
1084
+ ],
1085
+ "sin_cos_embedding_keys": null,
1086
+ "mean_std_embedding_keys": null,
1087
+ "action_configs": null
1088
+ },
1089
+ "action": {
1090
+ "delta_indices": [
1091
+ 0,
1092
+ 1,
1093
+ 2,
1094
+ 3,
1095
+ 4,
1096
+ 5,
1097
+ 6,
1098
+ 7,
1099
+ 8,
1100
+ 9,
1101
+ 10,
1102
+ 11,
1103
+ 12,
1104
+ 13,
1105
+ 14,
1106
+ 15
1107
+ ],
1108
+ "modality_keys": [
1109
+ "end_effector_position",
1110
+ "end_effector_rotation",
1111
+ "gripper_close"
1112
+ ],
1113
+ "sin_cos_embedding_keys": null,
1114
+ "mean_std_embedding_keys": null,
1115
+ "action_configs": [
1116
+ {
1117
+ "rep": "DELTA",
1118
+ "type": "EEF",
1119
+ "format": "DEFAULT",
1120
+ "state_key": null
1121
+ },
1122
+ {
1123
+ "rep": "DELTA",
1124
+ "type": "EEF",
1125
+ "format": "DEFAULT",
1126
+ "state_key": null
1127
+ },
1128
+ {
1129
+ "rep": "ABSOLUTE",
1130
+ "type": "NON_EEF",
1131
+ "format": "DEFAULT",
1132
+ "state_key": null
1133
+ }
1134
+ ]
1135
+ },
1136
+ "language": {
1137
+ "delta_indices": [
1138
+ 0
1139
+ ],
1140
+ "modality_keys": [
1141
+ "annotation.human.action.task_description"
1142
+ ],
1143
+ "sin_cos_embedding_keys": null,
1144
+ "mean_std_embedding_keys": null,
1145
+ "action_configs": null
1146
+ }
1147
+ },
1148
+ "bc_z": {
1149
+ "video": {
1150
+ "delta_indices": [
1151
+ 0
1152
+ ],
1153
+ "modality_keys": [
1154
+ "primary"
1155
+ ],
1156
+ "sin_cos_embedding_keys": null,
1157
+ "mean_std_embedding_keys": null,
1158
+ "action_configs": null
1159
+ },
1160
+ "state": {
1161
+ "delta_indices": [
1162
+ 0
1163
+ ],
1164
+ "modality_keys": [
1165
+ "end_effector_position",
1166
+ "end_effector_rotation",
1167
+ "gripper_position"
1168
+ ],
1169
+ "sin_cos_embedding_keys": null,
1170
+ "mean_std_embedding_keys": null,
1171
+ "action_configs": null
1172
+ },
1173
+ "action": {
1174
+ "delta_indices": [
1175
+ 0,
1176
+ 1,
1177
+ 2,
1178
+ 3,
1179
+ 4,
1180
+ 5,
1181
+ 6,
1182
+ 7,
1183
+ 8,
1184
+ 9,
1185
+ 10,
1186
+ 11,
1187
+ 12,
1188
+ 13,
1189
+ 14,
1190
+ 15
1191
+ ],
1192
+ "modality_keys": [
1193
+ "end_effector_position",
1194
+ "end_effector_rotation",
1195
+ "gripper_close"
1196
+ ],
1197
+ "sin_cos_embedding_keys": null,
1198
+ "mean_std_embedding_keys": null,
1199
+ "action_configs": [
1200
+ {
1201
+ "rep": "DELTA",
1202
+ "type": "EEF",
1203
+ "format": "DEFAULT",
1204
+ "state_key": null
1205
+ },
1206
+ {
1207
+ "rep": "DELTA",
1208
+ "type": "EEF",
1209
+ "format": "DEFAULT",
1210
+ "state_key": null
1211
+ },
1212
+ {
1213
+ "rep": "ABSOLUTE",
1214
+ "type": "NON_EEF",
1215
+ "format": "DEFAULT",
1216
+ "state_key": null
1217
+ }
1218
+ ]
1219
+ },
1220
+ "language": {
1221
+ "delta_indices": [
1222
+ 0
1223
+ ],
1224
+ "modality_keys": [
1225
+ "annotation.human.action.task_description"
1226
+ ],
1227
+ "sin_cos_embedding_keys": null,
1228
+ "mean_std_embedding_keys": null,
1229
+ "action_configs": null
1230
+ }
1231
+ },
1232
+ "toto": {
1233
+ "video": {
1234
+ "delta_indices": [
1235
+ 0
1236
+ ],
1237
+ "modality_keys": [
1238
+ "primary"
1239
+ ],
1240
+ "sin_cos_embedding_keys": null,
1241
+ "mean_std_embedding_keys": null,
1242
+ "action_configs": null
1243
+ },
1244
+ "state": {
1245
+ "delta_indices": [
1246
+ 0
1247
+ ],
1248
+ "modality_keys": [
1249
+ "joint_position",
1250
+ "gripper_position"
1251
+ ],
1252
+ "sin_cos_embedding_keys": null,
1253
+ "mean_std_embedding_keys": null,
1254
+ "action_configs": null
1255
+ },
1256
+ "action": {
1257
+ "delta_indices": [
1258
+ 0,
1259
+ 1,
1260
+ 2,
1261
+ 3,
1262
+ 4,
1263
+ 5,
1264
+ 6,
1265
+ 7,
1266
+ 8,
1267
+ 9,
1268
+ 10,
1269
+ 11,
1270
+ 12,
1271
+ 13,
1272
+ 14,
1273
+ 15
1274
+ ],
1275
+ "modality_keys": [
1276
+ "end_effector_position",
1277
+ "end_effector_rotation",
1278
+ "gripper_close"
1279
+ ],
1280
+ "sin_cos_embedding_keys": null,
1281
+ "mean_std_embedding_keys": null,
1282
+ "action_configs": [
1283
+ {
1284
+ "rep": "DELTA",
1285
+ "type": "EEF",
1286
+ "format": "DEFAULT",
1287
+ "state_key": null
1288
+ },
1289
+ {
1290
+ "rep": "DELTA",
1291
+ "type": "EEF",
1292
+ "format": "DEFAULT",
1293
+ "state_key": null
1294
+ },
1295
+ {
1296
+ "rep": "ABSOLUTE",
1297
+ "type": "NON_EEF",
1298
+ "format": "DEFAULT",
1299
+ "state_key": null
1300
+ }
1301
+ ]
1302
+ },
1303
+ "language": {
1304
+ "delta_indices": [
1305
+ 0
1306
+ ],
1307
+ "modality_keys": [
1308
+ "annotation.human.action.task_description"
1309
+ ],
1310
+ "sin_cos_embedding_keys": null,
1311
+ "mean_std_embedding_keys": null,
1312
+ "action_configs": null
1313
+ }
1314
+ },
1315
+ "dobbe": {
1316
+ "video": {
1317
+ "delta_indices": [
1318
+ 0
1319
+ ],
1320
+ "modality_keys": [
1321
+ "wrist"
1322
+ ],
1323
+ "sin_cos_embedding_keys": null,
1324
+ "mean_std_embedding_keys": null,
1325
+ "action_configs": null
1326
+ },
1327
+ "state": {
1328
+ "delta_indices": [
1329
+ 0
1330
+ ],
1331
+ "modality_keys": [
1332
+ "end_effector_position",
1333
+ "end_effector_rotation",
1334
+ "gripper_position"
1335
+ ],
1336
+ "sin_cos_embedding_keys": null,
1337
+ "mean_std_embedding_keys": null,
1338
+ "action_configs": null
1339
+ },
1340
+ "action": {
1341
+ "delta_indices": [
1342
+ 0,
1343
+ 1,
1344
+ 2,
1345
+ 3,
1346
+ 4,
1347
+ 5,
1348
+ 6,
1349
+ 7,
1350
+ 8,
1351
+ 9,
1352
+ 10,
1353
+ 11,
1354
+ 12,
1355
+ 13,
1356
+ 14,
1357
+ 15
1358
+ ],
1359
+ "modality_keys": [
1360
+ "end_effector_position",
1361
+ "end_effector_rotation",
1362
+ "gripper_close"
1363
+ ],
1364
+ "sin_cos_embedding_keys": null,
1365
+ "mean_std_embedding_keys": null,
1366
+ "action_configs": [
1367
+ {
1368
+ "rep": "DELTA",
1369
+ "type": "EEF",
1370
+ "format": "DEFAULT",
1371
+ "state_key": null
1372
+ },
1373
+ {
1374
+ "rep": "DELTA",
1375
+ "type": "EEF",
1376
+ "format": "DEFAULT",
1377
+ "state_key": null
1378
+ },
1379
+ {
1380
+ "rep": "ABSOLUTE",
1381
+ "type": "NON_EEF",
1382
+ "format": "DEFAULT",
1383
+ "state_key": null
1384
+ }
1385
+ ]
1386
+ },
1387
+ "language": {
1388
+ "delta_indices": [
1389
+ 0
1390
+ ],
1391
+ "modality_keys": [
1392
+ "annotation.human.action.task_description"
1393
+ ],
1394
+ "sin_cos_embedding_keys": null,
1395
+ "mean_std_embedding_keys": null,
1396
+ "action_configs": null
1397
+ }
1398
+ },
1399
+ "droid": {
1400
+ "video": {
1401
+ "delta_indices": [
1402
+ 0
1403
+ ],
1404
+ "modality_keys": [
1405
+ "primary",
1406
+ "secondary",
1407
+ "wrist"
1408
+ ],
1409
+ "sin_cos_embedding_keys": null,
1410
+ "mean_std_embedding_keys": null,
1411
+ "action_configs": null
1412
+ },
1413
+ "state": {
1414
+ "delta_indices": [
1415
+ 0
1416
+ ],
1417
+ "modality_keys": [
1418
+ "end_effector_position",
1419
+ "end_effector_rotation",
1420
+ "gripper_position"
1421
+ ],
1422
+ "sin_cos_embedding_keys": null,
1423
+ "mean_std_embedding_keys": null,
1424
+ "action_configs": null
1425
+ },
1426
+ "action": {
1427
+ "delta_indices": [
1428
+ 0,
1429
+ 1,
1430
+ 2,
1431
+ 3,
1432
+ 4,
1433
+ 5,
1434
+ 6,
1435
+ 7,
1436
+ 8,
1437
+ 9,
1438
+ 10,
1439
+ 11,
1440
+ 12,
1441
+ 13,
1442
+ 14,
1443
+ 15
1444
+ ],
1445
+ "modality_keys": [
1446
+ "end_effector_position",
1447
+ "end_effector_rotation",
1448
+ "gripper_close"
1449
+ ],
1450
+ "sin_cos_embedding_keys": null,
1451
+ "mean_std_embedding_keys": null,
1452
+ "action_configs": [
1453
+ {
1454
+ "rep": "DELTA",
1455
+ "type": "EEF",
1456
+ "format": "DEFAULT",
1457
+ "state_key": null
1458
+ },
1459
+ {
1460
+ "rep": "DELTA",
1461
+ "type": "EEF",
1462
+ "format": "DEFAULT",
1463
+ "state_key": null
1464
+ },
1465
+ {
1466
+ "rep": "ABSOLUTE",
1467
+ "type": "NON_EEF",
1468
+ "format": "DEFAULT",
1469
+ "state_key": null
1470
+ }
1471
+ ]
1472
+ },
1473
+ "language": {
1474
+ "delta_indices": [
1475
+ 0
1476
+ ],
1477
+ "modality_keys": [
1478
+ "annotation.human.action.task_description"
1479
+ ],
1480
+ "sin_cos_embedding_keys": null,
1481
+ "mean_std_embedding_keys": null,
1482
+ "action_configs": null
1483
+ }
1484
+ },
1485
+ "ucsd_kitchen_dataset_converted_externally_to_rlds": {
1486
+ "video": {
1487
+ "delta_indices": [
1488
+ 0
1489
+ ],
1490
+ "modality_keys": [
1491
+ "primary"
1492
+ ],
1493
+ "sin_cos_embedding_keys": null,
1494
+ "mean_std_embedding_keys": null,
1495
+ "action_configs": null
1496
+ },
1497
+ "state": {
1498
+ "delta_indices": [
1499
+ 0
1500
+ ],
1501
+ "modality_keys": [
1502
+ "joint_position"
1503
+ ],
1504
+ "sin_cos_embedding_keys": null,
1505
+ "mean_std_embedding_keys": null,
1506
+ "action_configs": null
1507
+ },
1508
+ "action": {
1509
+ "delta_indices": [
1510
+ 0,
1511
+ 1,
1512
+ 2,
1513
+ 3,
1514
+ 4,
1515
+ 5,
1516
+ 6,
1517
+ 7,
1518
+ 8,
1519
+ 9,
1520
+ 10,
1521
+ 11,
1522
+ 12,
1523
+ 13,
1524
+ 14,
1525
+ 15
1526
+ ],
1527
+ "modality_keys": [
1528
+ "end_effector_position",
1529
+ "end_effector_rotation",
1530
+ "gripper_close"
1531
+ ],
1532
+ "sin_cos_embedding_keys": null,
1533
+ "mean_std_embedding_keys": null,
1534
+ "action_configs": [
1535
+ {
1536
+ "rep": "DELTA",
1537
+ "type": "EEF",
1538
+ "format": "DEFAULT",
1539
+ "state_key": null
1540
+ },
1541
+ {
1542
+ "rep": "DELTA",
1543
+ "type": "EEF",
1544
+ "format": "DEFAULT",
1545
+ "state_key": null
1546
+ },
1547
+ {
1548
+ "rep": "ABSOLUTE",
1549
+ "type": "NON_EEF",
1550
+ "format": "DEFAULT",
1551
+ "state_key": null
1552
+ }
1553
+ ]
1554
+ },
1555
+ "language": {
1556
+ "delta_indices": [
1557
+ 0
1558
+ ],
1559
+ "modality_keys": [
1560
+ "annotation.human.action.task_description"
1561
+ ],
1562
+ "sin_cos_embedding_keys": null,
1563
+ "mean_std_embedding_keys": null,
1564
+ "action_configs": null
1565
+ }
1566
+ },
1567
+ "stanford_hydra_dataset_converted_externally_to_rlds": {
1568
+ "video": {
1569
+ "delta_indices": [
1570
+ 0
1571
+ ],
1572
+ "modality_keys": [
1573
+ "primary",
1574
+ "wrist"
1575
+ ],
1576
+ "sin_cos_embedding_keys": null,
1577
+ "mean_std_embedding_keys": null,
1578
+ "action_configs": null
1579
+ },
1580
+ "state": {
1581
+ "delta_indices": [
1582
+ 0
1583
+ ],
1584
+ "modality_keys": [
1585
+ "end_effector_position",
1586
+ "end_effector_rotation",
1587
+ "gripper_position"
1588
+ ],
1589
+ "sin_cos_embedding_keys": null,
1590
+ "mean_std_embedding_keys": null,
1591
+ "action_configs": null
1592
+ },
1593
+ "action": {
1594
+ "delta_indices": [
1595
+ 0,
1596
+ 1,
1597
+ 2,
1598
+ 3,
1599
+ 4,
1600
+ 5,
1601
+ 6,
1602
+ 7,
1603
+ 8,
1604
+ 9,
1605
+ 10,
1606
+ 11,
1607
+ 12,
1608
+ 13,
1609
+ 14,
1610
+ 15
1611
+ ],
1612
+ "modality_keys": [
1613
+ "end_effector_position",
1614
+ "end_effector_rotation",
1615
+ "gripper_close"
1616
+ ],
1617
+ "sin_cos_embedding_keys": null,
1618
+ "mean_std_embedding_keys": null,
1619
+ "action_configs": [
1620
+ {
1621
+ "rep": "DELTA",
1622
+ "type": "EEF",
1623
+ "format": "DEFAULT",
1624
+ "state_key": null
1625
+ },
1626
+ {
1627
+ "rep": "DELTA",
1628
+ "type": "EEF",
1629
+ "format": "DEFAULT",
1630
+ "state_key": null
1631
+ },
1632
+ {
1633
+ "rep": "ABSOLUTE",
1634
+ "type": "NON_EEF",
1635
+ "format": "DEFAULT",
1636
+ "state_key": null
1637
+ }
1638
+ ]
1639
+ },
1640
+ "language": {
1641
+ "delta_indices": [
1642
+ 0
1643
+ ],
1644
+ "modality_keys": [
1645
+ "annotation.human.action.task_description"
1646
+ ],
1647
+ "sin_cos_embedding_keys": null,
1648
+ "mean_std_embedding_keys": null,
1649
+ "action_configs": null
1650
+ }
1651
+ },
1652
+ "iamlab_cmu_pickup_insert_converted_externally_to_rlds": {
1653
+ "video": {
1654
+ "delta_indices": [
1655
+ 0
1656
+ ],
1657
+ "modality_keys": [
1658
+ "primary",
1659
+ "wrist"
1660
+ ],
1661
+ "sin_cos_embedding_keys": null,
1662
+ "mean_std_embedding_keys": null,
1663
+ "action_configs": null
1664
+ },
1665
+ "state": {
1666
+ "delta_indices": [
1667
+ 0
1668
+ ],
1669
+ "modality_keys": [
1670
+ "end_effector_position",
1671
+ "end_effector_rotation",
1672
+ "gripper_position"
1673
+ ],
1674
+ "sin_cos_embedding_keys": null,
1675
+ "mean_std_embedding_keys": null,
1676
+ "action_configs": null
1677
+ },
1678
+ "action": {
1679
+ "delta_indices": [
1680
+ 0,
1681
+ 1,
1682
+ 2,
1683
+ 3,
1684
+ 4,
1685
+ 5,
1686
+ 6,
1687
+ 7,
1688
+ 8,
1689
+ 9,
1690
+ 10,
1691
+ 11,
1692
+ 12,
1693
+ 13,
1694
+ 14,
1695
+ 15
1696
+ ],
1697
+ "modality_keys": [
1698
+ "end_effector_position",
1699
+ "end_effector_rotation",
1700
+ "gripper_close"
1701
+ ],
1702
+ "sin_cos_embedding_keys": null,
1703
+ "mean_std_embedding_keys": null,
1704
+ "action_configs": [
1705
+ {
1706
+ "rep": "DELTA",
1707
+ "type": "EEF",
1708
+ "format": "DEFAULT",
1709
+ "state_key": null
1710
+ },
1711
+ {
1712
+ "rep": "DELTA",
1713
+ "type": "EEF",
1714
+ "format": "DEFAULT",
1715
+ "state_key": null
1716
+ },
1717
+ {
1718
+ "rep": "ABSOLUTE",
1719
+ "type": "NON_EEF",
1720
+ "format": "DEFAULT",
1721
+ "state_key": null
1722
+ }
1723
+ ]
1724
+ },
1725
+ "language": {
1726
+ "delta_indices": [
1727
+ 0
1728
+ ],
1729
+ "modality_keys": [
1730
+ "annotation.human.action.task_description"
1731
+ ],
1732
+ "sin_cos_embedding_keys": null,
1733
+ "mean_std_embedding_keys": null,
1734
+ "action_configs": null
1735
+ }
1736
+ },
1737
+ "nyu_franka_play_dataset_converted_externally_to_rlds": {
1738
+ "video": {
1739
+ "delta_indices": [
1740
+ 0
1741
+ ],
1742
+ "modality_keys": [
1743
+ "primary",
1744
+ "secondary"
1745
+ ],
1746
+ "sin_cos_embedding_keys": null,
1747
+ "mean_std_embedding_keys": null,
1748
+ "action_configs": null
1749
+ },
1750
+ "state": {
1751
+ "delta_indices": [
1752
+ 0
1753
+ ],
1754
+ "modality_keys": [
1755
+ "end_effector_position",
1756
+ "end_effector_rotation",
1757
+ "gripper_position"
1758
+ ],
1759
+ "sin_cos_embedding_keys": null,
1760
+ "mean_std_embedding_keys": null,
1761
+ "action_configs": null
1762
+ },
1763
+ "action": {
1764
+ "delta_indices": [
1765
+ 0,
1766
+ 1,
1767
+ 2,
1768
+ 3,
1769
+ 4,
1770
+ 5,
1771
+ 6,
1772
+ 7,
1773
+ 8,
1774
+ 9,
1775
+ 10,
1776
+ 11,
1777
+ 12,
1778
+ 13,
1779
+ 14,
1780
+ 15
1781
+ ],
1782
+ "modality_keys": [
1783
+ "end_effector_position",
1784
+ "end_effector_rotation",
1785
+ "gripper_close"
1786
+ ],
1787
+ "sin_cos_embedding_keys": null,
1788
+ "mean_std_embedding_keys": null,
1789
+ "action_configs": [
1790
+ {
1791
+ "rep": "DELTA",
1792
+ "type": "EEF",
1793
+ "format": "DEFAULT",
1794
+ "state_key": null
1795
+ },
1796
+ {
1797
+ "rep": "DELTA",
1798
+ "type": "EEF",
1799
+ "format": "DEFAULT",
1800
+ "state_key": null
1801
+ },
1802
+ {
1803
+ "rep": "ABSOLUTE",
1804
+ "type": "NON_EEF",
1805
+ "format": "DEFAULT",
1806
+ "state_key": null
1807
+ }
1808
+ ]
1809
+ },
1810
+ "language": {
1811
+ "delta_indices": [
1812
+ 0
1813
+ ],
1814
+ "modality_keys": [
1815
+ "annotation.human.action.task_description"
1816
+ ],
1817
+ "sin_cos_embedding_keys": null,
1818
+ "mean_std_embedding_keys": null,
1819
+ "action_configs": null
1820
+ }
1821
+ },
1822
+ "austin_sailor_dataset_converted_externally_to_rlds": {
1823
+ "video": {
1824
+ "delta_indices": [
1825
+ 0
1826
+ ],
1827
+ "modality_keys": [
1828
+ "primary",
1829
+ "wrist"
1830
+ ],
1831
+ "sin_cos_embedding_keys": null,
1832
+ "mean_std_embedding_keys": null,
1833
+ "action_configs": null
1834
+ },
1835
+ "state": {
1836
+ "delta_indices": [
1837
+ 0
1838
+ ],
1839
+ "modality_keys": [
1840
+ "end_effector_position",
1841
+ "end_effector_rotation",
1842
+ "gripper_position"
1843
+ ],
1844
+ "sin_cos_embedding_keys": null,
1845
+ "mean_std_embedding_keys": null,
1846
+ "action_configs": null
1847
+ },
1848
+ "action": {
1849
+ "delta_indices": [
1850
+ 0,
1851
+ 1,
1852
+ 2,
1853
+ 3,
1854
+ 4,
1855
+ 5,
1856
+ 6,
1857
+ 7,
1858
+ 8,
1859
+ 9,
1860
+ 10,
1861
+ 11,
1862
+ 12,
1863
+ 13,
1864
+ 14,
1865
+ 15
1866
+ ],
1867
+ "modality_keys": [
1868
+ "end_effector_position",
1869
+ "end_effector_rotation",
1870
+ "gripper_close"
1871
+ ],
1872
+ "sin_cos_embedding_keys": null,
1873
+ "mean_std_embedding_keys": null,
1874
+ "action_configs": [
1875
+ {
1876
+ "rep": "DELTA",
1877
+ "type": "EEF",
1878
+ "format": "DEFAULT",
1879
+ "state_key": null
1880
+ },
1881
+ {
1882
+ "rep": "DELTA",
1883
+ "type": "EEF",
1884
+ "format": "DEFAULT",
1885
+ "state_key": null
1886
+ },
1887
+ {
1888
+ "rep": "ABSOLUTE",
1889
+ "type": "NON_EEF",
1890
+ "format": "DEFAULT",
1891
+ "state_key": null
1892
+ }
1893
+ ]
1894
+ },
1895
+ "language": {
1896
+ "delta_indices": [
1897
+ 0
1898
+ ],
1899
+ "modality_keys": [
1900
+ "annotation.human.action.task_description"
1901
+ ],
1902
+ "sin_cos_embedding_keys": null,
1903
+ "mean_std_embedding_keys": null,
1904
+ "action_configs": null
1905
+ }
1906
+ },
1907
+ "utaustin_mutex": {
1908
+ "video": {
1909
+ "delta_indices": [
1910
+ 0
1911
+ ],
1912
+ "modality_keys": [
1913
+ "primary",
1914
+ "wrist"
1915
+ ],
1916
+ "sin_cos_embedding_keys": null,
1917
+ "mean_std_embedding_keys": null,
1918
+ "action_configs": null
1919
+ },
1920
+ "state": {
1921
+ "delta_indices": [
1922
+ 0
1923
+ ],
1924
+ "modality_keys": [
1925
+ "joint_position",
1926
+ "gripper_position"
1927
+ ],
1928
+ "sin_cos_embedding_keys": null,
1929
+ "mean_std_embedding_keys": null,
1930
+ "action_configs": null
1931
+ },
1932
+ "action": {
1933
+ "delta_indices": [
1934
+ 0,
1935
+ 1,
1936
+ 2,
1937
+ 3,
1938
+ 4,
1939
+ 5,
1940
+ 6,
1941
+ 7,
1942
+ 8,
1943
+ 9,
1944
+ 10,
1945
+ 11,
1946
+ 12,
1947
+ 13,
1948
+ 14,
1949
+ 15
1950
+ ],
1951
+ "modality_keys": [
1952
+ "end_effector_position",
1953
+ "end_effector_rotation",
1954
+ "gripper_close"
1955
+ ],
1956
+ "sin_cos_embedding_keys": null,
1957
+ "mean_std_embedding_keys": null,
1958
+ "action_configs": [
1959
+ {
1960
+ "rep": "DELTA",
1961
+ "type": "EEF",
1962
+ "format": "DEFAULT",
1963
+ "state_key": null
1964
+ },
1965
+ {
1966
+ "rep": "DELTA",
1967
+ "type": "EEF",
1968
+ "format": "DEFAULT",
1969
+ "state_key": null
1970
+ },
1971
+ {
1972
+ "rep": "ABSOLUTE",
1973
+ "type": "NON_EEF",
1974
+ "format": "DEFAULT",
1975
+ "state_key": null
1976
+ }
1977
+ ]
1978
+ },
1979
+ "language": {
1980
+ "delta_indices": [
1981
+ 0
1982
+ ],
1983
+ "modality_keys": [
1984
+ "annotation.human.action.task_description"
1985
+ ],
1986
+ "sin_cos_embedding_keys": null,
1987
+ "mean_std_embedding_keys": null,
1988
+ "action_configs": null
1989
+ }
1990
+ },
1991
+ "fmb_dataset": {
1992
+ "video": {
1993
+ "delta_indices": [
1994
+ 0
1995
+ ],
1996
+ "modality_keys": [
1997
+ "primary",
1998
+ "secondary",
1999
+ "wrist"
2000
+ ],
2001
+ "sin_cos_embedding_keys": null,
2002
+ "mean_std_embedding_keys": null,
2003
+ "action_configs": null
2004
+ },
2005
+ "state": {
2006
+ "delta_indices": [
2007
+ 0
2008
+ ],
2009
+ "modality_keys": [
2010
+ "end_effector_position",
2011
+ "end_effector_rotation",
2012
+ "gripper_position"
2013
+ ],
2014
+ "sin_cos_embedding_keys": null,
2015
+ "mean_std_embedding_keys": null,
2016
+ "action_configs": null
2017
+ },
2018
+ "action": {
2019
+ "delta_indices": [
2020
+ 0,
2021
+ 1,
2022
+ 2,
2023
+ 3,
2024
+ 4,
2025
+ 5,
2026
+ 6,
2027
+ 7,
2028
+ 8,
2029
+ 9,
2030
+ 10,
2031
+ 11,
2032
+ 12,
2033
+ 13,
2034
+ 14,
2035
+ 15
2036
+ ],
2037
+ "modality_keys": [
2038
+ "end_effector_position",
2039
+ "end_effector_rotation",
2040
+ "gripper_close"
2041
+ ],
2042
+ "sin_cos_embedding_keys": null,
2043
+ "mean_std_embedding_keys": null,
2044
+ "action_configs": [
2045
+ {
2046
+ "rep": "DELTA",
2047
+ "type": "EEF",
2048
+ "format": "DEFAULT",
2049
+ "state_key": null
2050
+ },
2051
+ {
2052
+ "rep": "DELTA",
2053
+ "type": "EEF",
2054
+ "format": "DEFAULT",
2055
+ "state_key": null
2056
+ },
2057
+ {
2058
+ "rep": "ABSOLUTE",
2059
+ "type": "NON_EEF",
2060
+ "format": "DEFAULT",
2061
+ "state_key": null
2062
+ }
2063
+ ]
2064
+ },
2065
+ "language": {
2066
+ "delta_indices": [
2067
+ 0
2068
+ ],
2069
+ "modality_keys": [
2070
+ "annotation.human.action.task_description"
2071
+ ],
2072
+ "sin_cos_embedding_keys": null,
2073
+ "mean_std_embedding_keys": null,
2074
+ "action_configs": null
2075
+ }
2076
+ },
2077
+ "bridge_orig": {
2078
+ "video": {
2079
+ "delta_indices": [
2080
+ 0
2081
+ ],
2082
+ "modality_keys": [
2083
+ "primary",
2084
+ "secondary"
2085
+ ],
2086
+ "sin_cos_embedding_keys": null,
2087
+ "mean_std_embedding_keys": null,
2088
+ "action_configs": null
2089
+ },
2090
+ "state": {
2091
+ "delta_indices": [
2092
+ 0
2093
+ ],
2094
+ "modality_keys": [
2095
+ "end_effector_position",
2096
+ "end_effector_rotation",
2097
+ "gripper_position"
2098
+ ],
2099
+ "sin_cos_embedding_keys": null,
2100
+ "mean_std_embedding_keys": null,
2101
+ "action_configs": null
2102
+ },
2103
+ "action": {
2104
+ "delta_indices": [
2105
+ 0,
2106
+ 1,
2107
+ 2,
2108
+ 3,
2109
+ 4,
2110
+ 5,
2111
+ 6,
2112
+ 7,
2113
+ 8,
2114
+ 9,
2115
+ 10,
2116
+ 11,
2117
+ 12,
2118
+ 13,
2119
+ 14,
2120
+ 15
2121
+ ],
2122
+ "modality_keys": [
2123
+ "end_effector_position",
2124
+ "end_effector_rotation",
2125
+ "gripper_close"
2126
+ ],
2127
+ "sin_cos_embedding_keys": null,
2128
+ "mean_std_embedding_keys": null,
2129
+ "action_configs": [
2130
+ {
2131
+ "rep": "DELTA",
2132
+ "type": "EEF",
2133
+ "format": "DEFAULT",
2134
+ "state_key": null
2135
+ },
2136
+ {
2137
+ "rep": "DELTA",
2138
+ "type": "EEF",
2139
+ "format": "DEFAULT",
2140
+ "state_key": null
2141
+ },
2142
+ {
2143
+ "rep": "ABSOLUTE",
2144
+ "type": "NON_EEF",
2145
+ "format": "DEFAULT",
2146
+ "state_key": null
2147
+ }
2148
+ ]
2149
+ },
2150
+ "language": {
2151
+ "delta_indices": [
2152
+ 0
2153
+ ],
2154
+ "modality_keys": [
2155
+ "annotation.human.action.task_description"
2156
+ ],
2157
+ "sin_cos_embedding_keys": null,
2158
+ "mean_std_embedding_keys": null,
2159
+ "action_configs": null
2160
+ }
2161
+ },
2162
+ "austin_buds_dataset_converted_externally_to_rlds": {
2163
+ "video": {
2164
+ "delta_indices": [
2165
+ 0
2166
+ ],
2167
+ "modality_keys": [
2168
+ "primary",
2169
+ "wrist"
2170
+ ],
2171
+ "sin_cos_embedding_keys": null,
2172
+ "mean_std_embedding_keys": null,
2173
+ "action_configs": null
2174
+ },
2175
+ "state": {
2176
+ "delta_indices": [
2177
+ 0
2178
+ ],
2179
+ "modality_keys": [
2180
+ "joint_position",
2181
+ "gripper_position"
2182
+ ],
2183
+ "sin_cos_embedding_keys": null,
2184
+ "mean_std_embedding_keys": null,
2185
+ "action_configs": null
2186
+ },
2187
+ "action": {
2188
+ "delta_indices": [
2189
+ 0,
2190
+ 1,
2191
+ 2,
2192
+ 3,
2193
+ 4,
2194
+ 5,
2195
+ 6,
2196
+ 7,
2197
+ 8,
2198
+ 9,
2199
+ 10,
2200
+ 11,
2201
+ 12,
2202
+ 13,
2203
+ 14,
2204
+ 15
2205
+ ],
2206
+ "modality_keys": [
2207
+ "end_effector_position",
2208
+ "end_effector_rotation",
2209
+ "gripper_close"
2210
+ ],
2211
+ "sin_cos_embedding_keys": null,
2212
+ "mean_std_embedding_keys": null,
2213
+ "action_configs": [
2214
+ {
2215
+ "rep": "DELTA",
2216
+ "type": "EEF",
2217
+ "format": "DEFAULT",
2218
+ "state_key": null
2219
+ },
2220
+ {
2221
+ "rep": "DELTA",
2222
+ "type": "EEF",
2223
+ "format": "DEFAULT",
2224
+ "state_key": null
2225
+ },
2226
+ {
2227
+ "rep": "ABSOLUTE",
2228
+ "type": "NON_EEF",
2229
+ "format": "DEFAULT",
2230
+ "state_key": null
2231
+ }
2232
+ ]
2233
+ },
2234
+ "language": {
2235
+ "delta_indices": [
2236
+ 0
2237
+ ],
2238
+ "modality_keys": [
2239
+ "annotation.human.action.task_description"
2240
+ ],
2241
+ "sin_cos_embedding_keys": null,
2242
+ "mean_std_embedding_keys": null,
2243
+ "action_configs": null
2244
+ }
2245
+ },
2246
+ "neural_gr1": {
2247
+ "video": {
2248
+ "delta_indices": [
2249
+ 0
2250
+ ],
2251
+ "modality_keys": [
2252
+ "ego_view"
2253
+ ],
2254
+ "sin_cos_embedding_keys": null,
2255
+ "mean_std_embedding_keys": null,
2256
+ "action_configs": null
2257
+ },
2258
+ "state": {
2259
+ "delta_indices": [
2260
+ 0
2261
+ ],
2262
+ "modality_keys": [
2263
+ "left_arm",
2264
+ "left_hand",
2265
+ "left_leg",
2266
+ "neck",
2267
+ "right_arm",
2268
+ "right_hand",
2269
+ "right_leg",
2270
+ "waist"
2271
+ ],
2272
+ "sin_cos_embedding_keys": null,
2273
+ "mean_std_embedding_keys": null,
2274
+ "action_configs": null
2275
+ },
2276
+ "action": {
2277
+ "delta_indices": [
2278
+ 0,
2279
+ 1,
2280
+ 2,
2281
+ 3,
2282
+ 4,
2283
+ 5,
2284
+ 6,
2285
+ 7,
2286
+ 8,
2287
+ 9,
2288
+ 10,
2289
+ 11,
2290
+ 12,
2291
+ 13,
2292
+ 14,
2293
+ 15
2294
+ ],
2295
+ "modality_keys": [
2296
+ "left_arm",
2297
+ "left_hand",
2298
+ "left_leg",
2299
+ "neck",
2300
+ "right_arm",
2301
+ "right_hand",
2302
+ "right_leg",
2303
+ "waist"
2304
+ ],
2305
+ "sin_cos_embedding_keys": null,
2306
+ "mean_std_embedding_keys": null,
2307
+ "action_configs": [
2308
+ {
2309
+ "rep": "ABSOLUTE",
2310
+ "type": "NON_EEF",
2311
+ "format": "DEFAULT",
2312
+ "state_key": null
2313
+ },
2314
+ {
2315
+ "rep": "ABSOLUTE",
2316
+ "type": "NON_EEF",
2317
+ "format": "DEFAULT",
2318
+ "state_key": null
2319
+ },
2320
+ {
2321
+ "rep": "ABSOLUTE",
2322
+ "type": "NON_EEF",
2323
+ "format": "DEFAULT",
2324
+ "state_key": null
2325
+ },
2326
+ {
2327
+ "rep": "ABSOLUTE",
2328
+ "type": "NON_EEF",
2329
+ "format": "DEFAULT",
2330
+ "state_key": null
2331
+ },
2332
+ {
2333
+ "rep": "ABSOLUTE",
2334
+ "type": "NON_EEF",
2335
+ "format": "DEFAULT",
2336
+ "state_key": null
2337
+ },
2338
+ {
2339
+ "rep": "ABSOLUTE",
2340
+ "type": "NON_EEF",
2341
+ "format": "DEFAULT",
2342
+ "state_key": null
2343
+ },
2344
+ {
2345
+ "rep": "ABSOLUTE",
2346
+ "type": "NON_EEF",
2347
+ "format": "DEFAULT",
2348
+ "state_key": null
2349
+ },
2350
+ {
2351
+ "rep": "ABSOLUTE",
2352
+ "type": "NON_EEF",
2353
+ "format": "DEFAULT",
2354
+ "state_key": null
2355
+ }
2356
+ ]
2357
+ },
2358
+ "language": {
2359
+ "delta_indices": [
2360
+ 0
2361
+ ],
2362
+ "modality_keys": [
2363
+ "annotation.human.action.task_description"
2364
+ ],
2365
+ "sin_cos_embedding_keys": null,
2366
+ "mean_std_embedding_keys": null,
2367
+ "action_configs": null
2368
+ }
2369
+ },
2370
+ "galaxea": {
2371
+ "video": {
2372
+ "delta_indices": [
2373
+ 0
2374
+ ],
2375
+ "modality_keys": [
2376
+ "primary",
2377
+ "wrist_left",
2378
+ "wrist_right"
2379
+ ],
2380
+ "sin_cos_embedding_keys": null,
2381
+ "mean_std_embedding_keys": null,
2382
+ "action_configs": null
2383
+ },
2384
+ "state": {
2385
+ "delta_indices": [
2386
+ 0
2387
+ ],
2388
+ "modality_keys": [
2389
+ "state"
2390
+ ],
2391
+ "sin_cos_embedding_keys": null,
2392
+ "mean_std_embedding_keys": null,
2393
+ "action_configs": null
2394
+ },
2395
+ "action": {
2396
+ "delta_indices": [
2397
+ 0,
2398
+ 1,
2399
+ 2,
2400
+ 3,
2401
+ 4,
2402
+ 5,
2403
+ 6,
2404
+ 7,
2405
+ 8,
2406
+ 9,
2407
+ 10,
2408
+ 11,
2409
+ 12,
2410
+ 13,
2411
+ 14,
2412
+ 15
2413
+ ],
2414
+ "modality_keys": [
2415
+ "action"
2416
+ ],
2417
+ "sin_cos_embedding_keys": null,
2418
+ "mean_std_embedding_keys": null,
2419
+ "action_configs": [
2420
+ {
2421
+ "rep": "ABSOLUTE",
2422
+ "type": "NON_EEF",
2423
+ "format": "DEFAULT",
2424
+ "state_key": null
2425
+ }
2426
+ ]
2427
+ },
2428
+ "language": {
2429
+ "delta_indices": [
2430
+ 0
2431
+ ],
2432
+ "modality_keys": [
2433
+ "annotation.human.action.task_description"
2434
+ ],
2435
+ "sin_cos_embedding_keys": null,
2436
+ "mean_std_embedding_keys": null,
2437
+ "action_configs": null
2438
+ }
2439
+ },
2440
+ "roboturk": {
2441
+ "video": {
2442
+ "delta_indices": [
2443
+ 0
2444
+ ],
2445
+ "modality_keys": [
2446
+ "primary"
2447
+ ],
2448
+ "sin_cos_embedding_keys": null,
2449
+ "mean_std_embedding_keys": null,
2450
+ "action_configs": null
2451
+ },
2452
+ "state": {
2453
+ "delta_indices": [
2454
+ 0
2455
+ ],
2456
+ "modality_keys": [
2457
+ "none"
2458
+ ],
2459
+ "sin_cos_embedding_keys": null,
2460
+ "mean_std_embedding_keys": null,
2461
+ "action_configs": null
2462
+ },
2463
+ "action": {
2464
+ "delta_indices": [
2465
+ 0,
2466
+ 1,
2467
+ 2,
2468
+ 3,
2469
+ 4,
2470
+ 5,
2471
+ 6,
2472
+ 7,
2473
+ 8,
2474
+ 9,
2475
+ 10,
2476
+ 11,
2477
+ 12,
2478
+ 13,
2479
+ 14,
2480
+ 15
2481
+ ],
2482
+ "modality_keys": [
2483
+ "end_effector_position",
2484
+ "end_effector_rotation",
2485
+ "gripper_close"
2486
+ ],
2487
+ "sin_cos_embedding_keys": null,
2488
+ "mean_std_embedding_keys": null,
2489
+ "action_configs": [
2490
+ {
2491
+ "rep": "DELTA",
2492
+ "type": "EEF",
2493
+ "format": "DEFAULT",
2494
+ "state_key": null
2495
+ },
2496
+ {
2497
+ "rep": "DELTA",
2498
+ "type": "EEF",
2499
+ "format": "DEFAULT",
2500
+ "state_key": null
2501
+ },
2502
+ {
2503
+ "rep": "ABSOLUTE",
2504
+ "type": "NON_EEF",
2505
+ "format": "DEFAULT",
2506
+ "state_key": null
2507
+ }
2508
+ ]
2509
+ },
2510
+ "language": {
2511
+ "delta_indices": [
2512
+ 0
2513
+ ],
2514
+ "modality_keys": [
2515
+ "annotation.human.action.task_description"
2516
+ ],
2517
+ "sin_cos_embedding_keys": null,
2518
+ "mean_std_embedding_keys": null,
2519
+ "action_configs": null
2520
+ }
2521
+ },
2522
+ "berkeley_cable_routing": {
2523
+ "video": {
2524
+ "delta_indices": [
2525
+ 0
2526
+ ],
2527
+ "modality_keys": [
2528
+ "primary",
2529
+ "secondary",
2530
+ "wrist"
2531
+ ],
2532
+ "sin_cos_embedding_keys": null,
2533
+ "mean_std_embedding_keys": null,
2534
+ "action_configs": null
2535
+ },
2536
+ "state": {
2537
+ "delta_indices": [
2538
+ 0
2539
+ ],
2540
+ "modality_keys": [
2541
+ "joint_position"
2542
+ ],
2543
+ "sin_cos_embedding_keys": null,
2544
+ "mean_std_embedding_keys": null,
2545
+ "action_configs": null
2546
+ },
2547
+ "action": {
2548
+ "delta_indices": [
2549
+ 0,
2550
+ 1,
2551
+ 2,
2552
+ 3,
2553
+ 4,
2554
+ 5,
2555
+ 6,
2556
+ 7,
2557
+ 8,
2558
+ 9,
2559
+ 10,
2560
+ 11,
2561
+ 12,
2562
+ 13,
2563
+ 14,
2564
+ 15
2565
+ ],
2566
+ "modality_keys": [
2567
+ "end_effector_position",
2568
+ "end_effector_rotation",
2569
+ "gripper_close"
2570
+ ],
2571
+ "sin_cos_embedding_keys": null,
2572
+ "mean_std_embedding_keys": null,
2573
+ "action_configs": [
2574
+ {
2575
+ "rep": "DELTA",
2576
+ "type": "EEF",
2577
+ "format": "DEFAULT",
2578
+ "state_key": null
2579
+ },
2580
+ {
2581
+ "rep": "DELTA",
2582
+ "type": "EEF",
2583
+ "format": "DEFAULT",
2584
+ "state_key": null
2585
+ },
2586
+ {
2587
+ "rep": "ABSOLUTE",
2588
+ "type": "NON_EEF",
2589
+ "format": "DEFAULT",
2590
+ "state_key": null
2591
+ }
2592
+ ]
2593
+ },
2594
+ "language": {
2595
+ "delta_indices": [
2596
+ 0
2597
+ ],
2598
+ "modality_keys": [
2599
+ "annotation.human.action.task_description"
2600
+ ],
2601
+ "sin_cos_embedding_keys": null,
2602
+ "mean_std_embedding_keys": null,
2603
+ "action_configs": null
2604
+ }
2605
+ },
2606
+ "action_net": {
2607
+ "video": {
2608
+ "delta_indices": [
2609
+ 0
2610
+ ],
2611
+ "modality_keys": [
2612
+ "primary"
2613
+ ],
2614
+ "sin_cos_embedding_keys": null,
2615
+ "mean_std_embedding_keys": null,
2616
+ "action_configs": null
2617
+ },
2618
+ "state": {
2619
+ "delta_indices": [
2620
+ 0
2621
+ ],
2622
+ "modality_keys": [
2623
+ "state"
2624
+ ],
2625
+ "sin_cos_embedding_keys": null,
2626
+ "mean_std_embedding_keys": null,
2627
+ "action_configs": null
2628
+ },
2629
+ "action": {
2630
+ "delta_indices": [
2631
+ 0,
2632
+ 1,
2633
+ 2,
2634
+ 3,
2635
+ 4,
2636
+ 5,
2637
+ 6,
2638
+ 7,
2639
+ 8,
2640
+ 9,
2641
+ 10,
2642
+ 11,
2643
+ 12,
2644
+ 13,
2645
+ 14,
2646
+ 15
2647
+ ],
2648
+ "modality_keys": [
2649
+ "action"
2650
+ ],
2651
+ "sin_cos_embedding_keys": null,
2652
+ "mean_std_embedding_keys": null,
2653
+ "action_configs": [
2654
+ {
2655
+ "rep": "ABSOLUTE",
2656
+ "type": "NON_EEF",
2657
+ "format": "DEFAULT",
2658
+ "state_key": null
2659
+ }
2660
+ ]
2661
+ },
2662
+ "language": {
2663
+ "delta_indices": [
2664
+ 0
2665
+ ],
2666
+ "modality_keys": [
2667
+ "annotation.human.action.task_description"
2668
+ ],
2669
+ "sin_cos_embedding_keys": null,
2670
+ "mean_std_embedding_keys": null,
2671
+ "action_configs": null
2672
+ }
2673
+ },
2674
+ "kuka": {
2675
+ "video": {
2676
+ "delta_indices": [
2677
+ 0
2678
+ ],
2679
+ "modality_keys": [
2680
+ "primary"
2681
+ ],
2682
+ "sin_cos_embedding_keys": null,
2683
+ "mean_std_embedding_keys": null,
2684
+ "action_configs": null
2685
+ },
2686
+ "state": {
2687
+ "delta_indices": [
2688
+ 0
2689
+ ],
2690
+ "modality_keys": [
2691
+ "end_effector_position",
2692
+ "end_effector_rotation",
2693
+ "gripper_position"
2694
+ ],
2695
+ "sin_cos_embedding_keys": null,
2696
+ "mean_std_embedding_keys": null,
2697
+ "action_configs": null
2698
+ },
2699
+ "action": {
2700
+ "delta_indices": [
2701
+ 0,
2702
+ 1,
2703
+ 2,
2704
+ 3,
2705
+ 4,
2706
+ 5,
2707
+ 6,
2708
+ 7,
2709
+ 8,
2710
+ 9,
2711
+ 10,
2712
+ 11,
2713
+ 12,
2714
+ 13,
2715
+ 14,
2716
+ 15
2717
+ ],
2718
+ "modality_keys": [
2719
+ "end_effector_position",
2720
+ "end_effector_rotation",
2721
+ "gripper_close"
2722
+ ],
2723
+ "sin_cos_embedding_keys": null,
2724
+ "mean_std_embedding_keys": null,
2725
+ "action_configs": [
2726
+ {
2727
+ "rep": "DELTA",
2728
+ "type": "EEF",
2729
+ "format": "DEFAULT",
2730
+ "state_key": null
2731
+ },
2732
+ {
2733
+ "rep": "DELTA",
2734
+ "type": "EEF",
2735
+ "format": "DEFAULT",
2736
+ "state_key": null
2737
+ },
2738
+ {
2739
+ "rep": "ABSOLUTE",
2740
+ "type": "NON_EEF",
2741
+ "format": "DEFAULT",
2742
+ "state_key": null
2743
+ }
2744
+ ]
2745
+ },
2746
+ "language": {
2747
+ "delta_indices": [
2748
+ 0
2749
+ ],
2750
+ "modality_keys": [
2751
+ "annotation.human.action.task_description"
2752
+ ],
2753
+ "sin_cos_embedding_keys": null,
2754
+ "mean_std_embedding_keys": null,
2755
+ "action_configs": null
2756
+ }
2757
+ },
2758
+ "humanoid_everyday_h1": {
2759
+ "video": {
2760
+ "delta_indices": [
2761
+ 0
2762
+ ],
2763
+ "modality_keys": [
2764
+ "egocentric_resized"
2765
+ ],
2766
+ "sin_cos_embedding_keys": null,
2767
+ "mean_std_embedding_keys": null,
2768
+ "action_configs": null
2769
+ },
2770
+ "state": {
2771
+ "delta_indices": [
2772
+ 0
2773
+ ],
2774
+ "modality_keys": [
2775
+ "left_arm",
2776
+ "left_hand",
2777
+ "right_arm",
2778
+ "right_hand"
2779
+ ],
2780
+ "sin_cos_embedding_keys": null,
2781
+ "mean_std_embedding_keys": null,
2782
+ "action_configs": null
2783
+ },
2784
+ "action": {
2785
+ "delta_indices": [
2786
+ 0,
2787
+ 1,
2788
+ 2,
2789
+ 3,
2790
+ 4,
2791
+ 5,
2792
+ 6,
2793
+ 7,
2794
+ 8,
2795
+ 9,
2796
+ 10,
2797
+ 11,
2798
+ 12,
2799
+ 13,
2800
+ 14,
2801
+ 15
2802
+ ],
2803
+ "modality_keys": [
2804
+ "left_arm",
2805
+ "left_hand",
2806
+ "right_arm",
2807
+ "right_hand"
2808
+ ],
2809
+ "sin_cos_embedding_keys": null,
2810
+ "mean_std_embedding_keys": null,
2811
+ "action_configs": [
2812
+ {
2813
+ "rep": "ABSOLUTE",
2814
+ "type": "NON_EEF",
2815
+ "format": "DEFAULT",
2816
+ "state_key": null
2817
+ },
2818
+ {
2819
+ "rep": "ABSOLUTE",
2820
+ "type": "NON_EEF",
2821
+ "format": "DEFAULT",
2822
+ "state_key": null
2823
+ },
2824
+ {
2825
+ "rep": "ABSOLUTE",
2826
+ "type": "NON_EEF",
2827
+ "format": "DEFAULT",
2828
+ "state_key": null
2829
+ },
2830
+ {
2831
+ "rep": "ABSOLUTE",
2832
+ "type": "NON_EEF",
2833
+ "format": "DEFAULT",
2834
+ "state_key": null
2835
+ }
2836
+ ]
2837
+ },
2838
+ "language": {
2839
+ "delta_indices": [
2840
+ 0
2841
+ ],
2842
+ "modality_keys": [
2843
+ "annotation.human.action.task_description"
2844
+ ],
2845
+ "sin_cos_embedding_keys": null,
2846
+ "mean_std_embedding_keys": null,
2847
+ "action_configs": null
2848
+ }
2849
+ },
2850
+ "gr1_human_prq": {
2851
+ "video": {
2852
+ "delta_indices": [
2853
+ 0
2854
+ ],
2855
+ "modality_keys": [
2856
+ "ego_view"
2857
+ ],
2858
+ "sin_cos_embedding_keys": null,
2859
+ "mean_std_embedding_keys": null,
2860
+ "action_configs": null
2861
+ },
2862
+ "state": {
2863
+ "delta_indices": [
2864
+ 0
2865
+ ],
2866
+ "modality_keys": [
2867
+ "wrist_pos",
2868
+ "wrist_rot6d",
2869
+ "hand_q"
2870
+ ],
2871
+ "sin_cos_embedding_keys": null,
2872
+ "mean_std_embedding_keys": null,
2873
+ "action_configs": null
2874
+ },
2875
+ "action": {
2876
+ "delta_indices": [
2877
+ 0,
2878
+ 1,
2879
+ 2,
2880
+ 3,
2881
+ 4,
2882
+ 5,
2883
+ 6,
2884
+ 7,
2885
+ 8,
2886
+ 9,
2887
+ 10,
2888
+ 11,
2889
+ 12,
2890
+ 13,
2891
+ 14,
2892
+ 15
2893
+ ],
2894
+ "modality_keys": [
2895
+ "wrist_pos",
2896
+ "wrist_rot6d",
2897
+ "hand_q"
2898
+ ],
2899
+ "sin_cos_embedding_keys": null,
2900
+ "mean_std_embedding_keys": null,
2901
+ "action_configs": [
2902
+ {
2903
+ "rep": "ABSOLUTE",
2904
+ "type": "NON_EEF",
2905
+ "format": "DEFAULT",
2906
+ "state_key": null
2907
+ },
2908
+ {
2909
+ "rep": "ABSOLUTE",
2910
+ "type": "NON_EEF",
2911
+ "format": "DEFAULT",
2912
+ "state_key": null
2913
+ },
2914
+ {
2915
+ "rep": "ABSOLUTE",
2916
+ "type": "NON_EEF",
2917
+ "format": "DEFAULT",
2918
+ "state_key": null
2919
+ }
2920
+ ]
2921
+ },
2922
+ "language": {
2923
+ "delta_indices": [
2924
+ 0
2925
+ ],
2926
+ "modality_keys": [
2927
+ "annotation.human.coarse_action"
2928
+ ],
2929
+ "sin_cos_embedding_keys": null,
2930
+ "mean_std_embedding_keys": null,
2931
+ "action_configs": null
2932
+ }
2933
+ }
2934
+ },
2935
+ "image_max_area": 65536,
2936
+ "image_resize_m": 32,
2937
+ "random_crop_fraction": null,
2938
+ "random_rotation_angle": null,
2939
+ "color_jitter_params": {
2940
+ "brightness": 0.3,
2941
+ "contrast": 0.4,
2942
+ "saturation": 0.5,
2943
+ "hue": 0.08
2944
+ },
2945
+ "model_name": "RLWRLD/RLDX-1-VLM",
2946
+ "model_type": "vtc_qwen3_vl",
2947
+ "formalize_language": true,
2948
+ "max_state_dim": 64,
2949
+ "max_action_dim": 64,
2950
+ "max_action_horizon": 16,
2951
+ "use_percentiles": true,
2952
+ "clip_outliers": true,
2953
+ "apply_sincos_state_encoding": false,
2954
+ "use_relative_action": true,
2955
+ "memory_length": 1,
2956
+ "general_embodiment_train_ratio": 0,
2957
+ "allow_missing_physics": false,
2958
+ "physics_keys": null,
2959
+ "physics_dims": null,
2960
+ "conversation_image_first": false,
2961
+ "vlm_lang_loss": false
2962
+ }
2963
+ }
processor/statistics.json ADDED
The diff for this file is too large to render. See raw diff