{ "processor_class": "Gr00tN1d6Processor", "processor_kwargs": { "modality_configs": { "behavior_r1_pro": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "observation.images.rgb.head_256_256", "observation.images.rgb.left_wrist_256_256", "observation.images.rgb.right_wrist_256_256" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "robot_pos", "robot_ori_cos", "robot_ori_sin", "robot_2d_ori", "robot_2d_ori_cos", "robot_2d_ori_sin", "robot_lin_vel", "robot_ang_vel", "arm_left_qpos", "arm_left_qpos_sin", "arm_left_qpos_cos", "eef_left_pos", "eef_left_quat", "gripper_left_qpos", "arm_right_qpos", "arm_right_qpos_sin", "arm_right_qpos_cos", "eef_right_pos", "eef_right_quat", "gripper_right_qpos", "trunk_qpos" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 ], "modality_keys": [ "base", "torso", "left_arm", "left_gripper", "right_arm", "right_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "RELATIVE", "type": "NON_EEF", "format": "DEFAULT", "state_key": "trunk_qpos", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "RELATIVE", "type": "NON_EEF", "format": "DEFAULT", "state_key": "arm_left_qpos", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "RELATIVE", "type": "NON_EEF", "format": "DEFAULT", "state_key": "arm_right_qpos", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.human.coarse_action" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "gr1": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "ego_view_bg_crop_pad_res256_freq20" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "left_arm", "right_arm", "left_hand", "right_hand", "waist" ], "sin_cos_embedding_keys": [ "left_arm", "right_arm", "left_hand", "right_hand", "waist" ], "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ], "modality_keys": [ "left_arm", "right_arm", "left_hand", "right_hand", "waist" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "RELATIVE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "RELATIVE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "RELATIVE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "RELATIVE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "task" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "robocasa_panda_omron": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "res256_image_side_0", "res256_image_side_1", "res256_image_wrist_0" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "end_effector_position_relative", "end_effector_rotation_relative", "gripper_qpos", "base_position", "base_rotation" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ], "modality_keys": [ "end_effector_position", "end_effector_rotation", "gripper_close", "base_motion", "control_mode" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.human.action.task_description" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "cmr_versius": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "left_pose", "left_gripper", "right_pose", "right_gripper", "translation_scaling", "rotation_scaling", "hapticengaged_left", "hapticengaged_right" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "left_pose", "left_gripper", "right_pose", "right_gripper" ], "min_max_embedding_keys": null, "pass_through_keys": [ "translation_scaling", "rotation_scaling", "hapticengaged_left", "hapticengaged_right" ], "action_configs": null }, "action": { "delta_indices": [ 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100 ], "modality_keys": [ "left_pose", "left_gripper", "right_pose", "right_gripper", "hapticengaged_left", "hapticengaged_right" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": [ "hapticengaged_left", "hapticengaged_right" ], "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "left_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": "translation_scaling", "rotation_scaling_key": "rotation_scaling", "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": "left_gripper", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": true }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "right_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": "translation_scaling", "rotation_scaling_key": "rotation_scaling", "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": "right_gripper", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": true }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "skip", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "skip", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.human.task_description" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "ucsd_dvrk": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "camera_left" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "psm_retraction_pose", "psm_retraction_gripper", "psm_cutter_pose", "psm_cutter_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "psm_retraction_pose", "psm_retraction_gripper", "psm_cutter_pose", "psm_cutter_gripper" ], "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 ], "modality_keys": [ "psm_retraction_pose", "psm_retraction_gripper", "psm_cutter_pose", "psm_cutter_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm_retraction_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "wxyz", "reference_rotation_format": "quat", "reference_quat_order": "wxyz", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm_cutter_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "wxyz", "reference_rotation_format": "quat", "reference_quat_order": "wxyz", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "task" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "jhu_imerse_dvrk": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope_left", "wrist_left", "wrist_right" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm1_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm2_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.human.task_description" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "obuda_dvrk": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope_left", "wrist_left", "wrist_right" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm1_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm2_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "task" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "stanford_dvrk_real": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope_left" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm1_pose", "normalization_type": "percentile", "input_rotation_format": "euler", "input_quat_order": "xyzw", "reference_rotation_format": "euler", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm2_pose", "normalization_type": "percentile", "input_rotation_format": "euler", "input_quat_order": "xyzw", "reference_rotation_format": "euler", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "task" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "tud_tundra_ur5e": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "laparoscope_left" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "joint_position", "eef_pose" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "joint_position" ], "min_max_embedding_keys": null, "pass_through_keys": [ "eef_pose" ], "action_configs": null }, "action": { "delta_indices": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 ], "modality_keys": [ "eef_pose", "gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "eef_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "task" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "jhu_lscr_dvrk_smarts": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope_left", "camera_side_view" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm1_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm2_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.task" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "jhu_imerse_dvrk_mono": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope_left" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm1_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm2_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.human.task_description" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "rob_surgical_bitrack": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "left_pose", "right_pose", "aux_pose" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "left_pose", "right_pose", "aux_pose" ], "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ], "modality_keys": [ "left_pose", "right_pose", "aux_pose" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "left_pose", "normalization_type": "percentile", "input_rotation_format": "euler", "input_quat_order": "xyzw", "reference_rotation_format": "euler", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "right_pose", "normalization_type": "percentile", "input_rotation_format": "euler", "input_quat_order": "xyzw", "reference_rotation_format": "euler", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "aux_pose", "normalization_type": "percentile", "input_rotation_format": "euler", "input_quat_order": "xyzw", "reference_rotation_format": "euler", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.instruction" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "turin_mitic_ex_vivo": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope_left" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "psm1_joints", "psm2_joints", "psm1_pose", "psm2_pose" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "psm1_joints", "psm2_joints" ], "min_max_embedding_keys": null, "pass_through_keys": [ "psm1_pose", "psm2_pose" ], "action_configs": null }, "action": { "delta_indices": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 ], "modality_keys": [ "psm1_pose", "psm2_pose" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm1_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm2_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.instruction" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "ustc_torin_tuodao": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope_left" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "left_joints", "right_joints", "left_pose", "right_pose" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "left_joints", "right_joints" ], "min_max_embedding_keys": null, "pass_through_keys": [ "left_pose", "right_pose" ], "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ], "modality_keys": [ "left_pose", "left_gripper", "right_pose", "right_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "left_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "right_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.instruction" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "hamlyn_dvrk_30hz": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope", "wrist_left", "wrist_right" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "left_arm_pose", "left_arm_gripper", "right_arm_pose", "right_arm_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "left_arm_pose", "left_arm_gripper", "right_arm_pose", "right_arm_gripper" ], "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ], "modality_keys": [ "left_arm_pose", "left_arm_gripper", "right_arm_pose", "right_arm_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "left_arm_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "wxyz", "reference_rotation_format": "quat", "reference_quat_order": "wxyz", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "right_arm_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "wxyz", "reference_rotation_format": "quat", "reference_quat_order": "wxyz", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "task" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "ucb_dvrk": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "camera_left" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "psm1_joints", "psm1_gripper", "psm2_joints", "psm2_gripper", "psm1_pose", "psm2_pose" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "psm1_joints", "psm1_gripper", "psm2_joints", "psm2_gripper" ], "min_max_embedding_keys": null, "pass_through_keys": [ "psm1_pose", "psm2_pose" ], "action_configs": null }, "action": { "delta_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ], "modality_keys": [ "psm1_pose", "psm1_gripper", "psm2_pose", "psm2_gripper" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm1_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "psm2_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false }, { "rep": "ABSOLUTE", "type": "NON_EEF", "format": "DEFAULT", "state_key": null, "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "rot6d", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "task" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } }, "jhu_imerse_star_il": { "video": { "delta_indices": [ 0 ], "modality_keys": [ "endoscope_left", "wrist_left" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null }, "state": { "delta_indices": [ 0 ], "modality_keys": [ "kuka_joint_pos", "endo360_joint_pos", "kuka_pose" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": [ "kuka_joint_pos", "endo360_joint_pos" ], "min_max_embedding_keys": null, "pass_through_keys": [ "kuka_pose" ], "action_configs": null }, "action": { "delta_indices": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 ], "modality_keys": [ "kuka_pose" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": [ { "rep": "REL_XYZ_ROT6D", "type": "EEF", "format": "XYZ_ROT6D", "state_key": "kuka_pose", "normalization_type": "percentile", "input_rotation_format": "quat", "input_quat_order": "xyzw", "reference_rotation_format": "quat", "reference_quat_order": "xyzw", "translation_scaling_key": null, "rotation_scaling_key": null, "hold_through_clutch": false } ] }, "language": { "delta_indices": [ 0 ], "modality_keys": [ "annotation.human.task_description" ], "sin_cos_embedding_keys": null, "mean_std_embedding_keys": null, "min_max_embedding_keys": null, "pass_through_keys": null, "action_configs": null } } }, "image_crop_size": [ 224, 392 ], "image_target_size": [ 236, 414 ], "use_albumentations": false, "random_rotation_angle": 5, "color_jitter_params": { "brightness": 0.12, "contrast": 0.15, "saturation": 0.15, "hue": 0.02 }, "shortest_image_edge": null, "crop_fraction": null, "model_name": "nvidia/Eagle-Block2A-2B-v2", "model_type": "eagle", "formalize_language": true, "max_state_dim": 128, "max_action_dim": 128, "max_action_horizon": 50, "use_percentiles": false, "clip_outliers": true, "apply_sincos_state_encoding": true, "use_relative_action": true } }