diff --git a/RKDv2.1/DA_MeanPool+LP/default/RKD v2.1 DA MF.yaml b/RKDv2.1/DA_MeanPool+LP/default/RKD v2.1 DA MF.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68dba94249cad9339b0131f46e7ccef7286d463e --- /dev/null +++ b/RKDv2.1/DA_MeanPool+LP/default/RKD v2.1 DA MF.yaml @@ -0,0 +1,62 @@ +experiment_name: RKD v2.1 DA, Student:MeanPool + LP +policy_type: groot_MGRKD +base_model_path: ${HOME}/groot_robocasa/Atomic26_baseline/checkpoint-60000 +output_root: ${HOME}/groot_robocasa/robocasa_v2/RKDv2.1/DA_MeanPool+LP +dataset: + dataset_soup: my_atomic26_human +training: + num_gpus: 1 + batch_size: 128 + seed: 42 +model: + rkd_student_reconstruction_head_pooling: mean +phases: +- name: phase2_mgrkd_da_student_meanpool_only + max_steps: 30000 + save_steps: 0 + trainable: + preset: processing_line_only + tune_llm: false + tune_visual: false + tune_projector: false + tune_diffusion_model: false + losses: + rkd_enabled: true + rkd_fm_loss_weight: 0.0 + rkd_loss_weight: 1.0 + rkd_relation_mode: flatten + rkd_loss_type: distance_angle + rkd_teacher_source: action_encoder + rkd_action_encoder_projector_enabled: true + rkd_action_encoder_projector_dim: 512 + rkd_action_encoder_projector_pooling: flatten + rkd_student_reconstruction_head_enabled: true + rkd_student_reconstruction_head_dim: 512 + rkd_student_reconstruction_head_hidden_dim: 512 + rkd_distance_loss_weight: 1.0 + rkd_angle_loss_weight: 2.0 + rkd_exclude_diagonal: true +- name: phase3_fm_mgrkd_da_student_meanpool_fixed_0p5 + max_steps: 30000 + save_steps: 0 + trainable: + tune_llm: false + tune_visual: false + tune_projector: true + tune_diffusion_model: true + losses: + rkd_enabled: true + rkd_fm_loss_weight: 1.0 + rkd_loss_weight: 0.5 + rkd_relation_mode: flatten + rkd_loss_type: distance_angle + rkd_teacher_source: action_encoder + rkd_action_encoder_projector_enabled: true + rkd_action_encoder_projector_dim: 512 + rkd_action_encoder_projector_pooling: flatten + rkd_student_reconstruction_head_enabled: true + rkd_student_reconstruction_head_dim: 512 + rkd_student_reconstruction_head_hidden_dim: 512 + rkd_distance_loss_weight: 1.0 + rkd_angle_loss_weight: 2.0 + rkd_exclude_diagonal: true diff --git a/RKDv2.1/DA_MeanPool+LP/default/phase2/experiment_cfg/metadata.json b/RKDv2.1/DA_MeanPool+LP/default/phase2/experiment_cfg/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..300eb070963337016e16465f50ba8351ea2eb25c --- /dev/null +++ b/RKDv2.1/DA_MeanPool+LP/default/phase2/experiment_cfg/metadata.json @@ -0,0 +1,431 @@ +{ + "new_embodiment": { + "statistics": { + "state": { + "base_position": { + "max": [ + 7.3139495849609375, + 0.4876587688922882, + 0.7196521759033203 + ], + "min": [ + -4.94293737411499, + -6.890198230743408, + 0.6996827125549316 + ], + "mean": [ + 2.749288365724937, + -2.0455216239909983, + 0.700532551020533 + ], + "std": [ + 1.6786295897046262, + 1.312897969434244, + 0.00133751758906258 + ], + "q01": [ + -1.4241907881148037, + -5.1716547935950885, + 0.6999670898768614 + ], + "q99": [ + 6.198111545831555, + -0.5873531334104489, + 0.7038833014242587 + ] + }, + "base_rotation": { + "max": [ + 0.0, + 0.0, + 1.0, + 1.0 + ], + "min": [ + 0.0, + 0.0, + -1.0, + 0.0 + ], + "mean": [ + 0.0, + 0.0, + 0.23084999587450247, + 0.6238431088581847 + ], + "std": [ + 0.0, + 0.0, + 0.6556404371644047, + 0.3572252105732042 + ], + "q01": [ + 0.0, + 0.0, + -0.9999999999999999, + 1.7482354593273349e-06 + ], + "q99": [ + 0.0, + 0.0, + 0.9999999999999999, + 0.9999999999999999 + ] + }, + "end_effector_position_relative": { + "max": [ + 0.9014528393745422, + 0.8003877401351929, + 0.9829942584037781 + ], + "min": [ + -0.37471598386764526, + -0.8472502827644348, + -0.25070279836654663 + ], + "mean": [ + 0.28667332601863704, + -0.038315473170876704, + 0.45974658906777444 + ], + "std": [ + 0.17067058828305154, + 0.23569952037784195, + 0.21950702354181487 + ], + "q01": [ + 0.015172948963784228, + -0.44450502063220615, + 0.23314444103329338 + ], + "q99": [ + 0.5609069689572412, + 0.38454179128009547, + 0.7028102736473852 + ] + }, + "end_effector_rotation_relative": { + "max": [ + 0.9999998807907104, + 0.9984455108642578, + 0.9434727430343628, + 0.9062229990959167 + ], + "min": [ + -0.9999930262565613, + -0.9988337755203247, + -0.9618149995803833, + 2.1872274658107926e-07 + ], + "mean": [ + -0.25672297401336, + 0.024425335806687216, + -0.08740977164483847, + 0.16608739644018397 + ], + "std": [ + 0.7932585208392383, + 0.3102260195580416, + 0.3772472576315148, + 0.17451959300158773 + ], + "q01": [ + -0.99379006987882, + -0.5478102050038828, + -0.6101777952971636, + 0.002274085014134748 + ], + "q99": [ + 0.8804856038896288, + 0.5910611092873037, + 0.5216058042993562, + 0.5231965974012255 + ] + }, + "gripper_qpos": { + "max": [ + 0.055869169533252716, + 0.010916369967162609 + ], + "min": [ + -0.011436971835792065, + -0.05664053186774254 + ], + "mean": [ + 0.031651551516052555, + -0.03161481387920421 + ], + "std": [ + 0.013119769222217526, + 0.01306078225192402 + ], + "q01": [ + 0.0060999246446777336, + -0.04062601653964198 + ], + "q99": [ + 0.04054891621517283, + -0.0059163567113456345 + ] + } + }, + "action": { + "base_motion": { + "max": [ + 1.0, + 1.0, + 1.0, + 0.0 + ], + "min": [ + -1.0, + -1.0, + -1.0, + 0.0 + ], + "mean": [ + 0.008144896753718373, + -0.00018893135146078263, + -0.0008739062727221845, + 0.0 + ], + "std": [ + 0.11030045424364411, + 0.10148082570313594, + 0.08975698373467506, + 0.0 + ], + "q01": [ + -0.07287373067581518, + -0.09446994199569948, + -0.07702818259249572, + 0.0 + ], + "q99": [ + 0.04485485414407898, + 0.09626941605965177, + 0.07610665906122742, + 0.0 + ] + }, + "control_mode": { + "max": [ + 1.0 + ], + "min": [ + -1.0 + ], + "mean": [ + -0.9221974313425939 + ], + "std": [ + 0.38670194342612674 + ], + "q01": [ + -1.0 + ], + "q99": [ + -0.384693946883099 + ] + }, + "end_effector_position": { + "max": [ + 1.0, + 1.0, + 1.0 + ], + "min": [ + -1.0, + -1.0, + -1.0 + ], + "mean": [ + 0.01976129923017491, + -0.020870631344490034, + -0.05993476517349181 + ], + "std": [ + 0.4347024990804053, + 0.4221662976569997, + 0.3845506843044066 + ], + "q01": [ + -0.8835792939767807, + -0.9280919251713778, + -0.783361071470956 + ], + "q99": [ + 0.7622084438449307, + 0.8625125252609077, + 0.7470974753250706 + ] + }, + "end_effector_rotation": { + "max": [ + 1.0, + 1.0, + 1.0 + ], + "min": [ + -1.0, + -1.0, + -1.0 + ], + "mean": [ + 0.008089673068544335, + -0.026498254627927348, + 0.0029083551743059005 + ], + "std": [ + 0.11216734503733773, + 0.13206002613798629, + 0.12615870412692864 + ], + "q01": [ + -0.2814627972846672, + -0.4342386516115439, + -0.34489755628340574 + ], + "q99": [ + 0.31879353266939386, + 0.28411797683220563, + 0.3597718671669894 + ] + }, + "gripper_close": { + "max": [ + 1.0 + ], + "min": [ + -1.0 + ], + "mean": [ + -0.3824228874769045 + ], + "std": [ + 0.9240015096469786 + ], + "q01": [ + -1.0 + ], + "q99": [ + 0.5597272305813592 + ] + } + } + }, + "modalities": { + "video": { + "robot0_eye_in_hand": { + "resolution": [ + 256, + 256 + ], + "channels": 3, + "fps": 20.0 + }, + "robot0_agentview_left": { + "resolution": [ + 256, + 256 + ], + "channels": 3, + "fps": 20.0 + }, + "robot0_agentview_right": { + "resolution": [ + 256, + 256 + ], + "channels": 3, + "fps": 20.0 + } + }, + "state": { + "base_position": { + "absolute": true, + "rotation_type": null, + "shape": [ + 3 + ], + "continuous": true + }, + "base_rotation": { + "absolute": true, + "rotation_type": "quaternion", + "shape": [ + 4 + ], + "continuous": true + }, + "end_effector_position_relative": { + "absolute": true, + "rotation_type": null, + "shape": [ + 3 + ], + "continuous": true + }, + "end_effector_rotation_relative": { + "absolute": true, + "rotation_type": "quaternion", + "shape": [ + 4 + ], + "continuous": true + }, + "gripper_qpos": { + "absolute": true, + "rotation_type": null, + "shape": [ + 2 + ], + "continuous": true + } + }, + "action": { + "base_motion": { + "absolute": true, + "rotation_type": null, + "shape": [ + 4 + ], + "continuous": true + }, + "control_mode": { + "absolute": true, + "rotation_type": null, + "shape": [ + 1 + ], + "continuous": true + }, + "end_effector_position": { + "absolute": true, + "rotation_type": null, + "shape": [ + 3 + ], + "continuous": true + }, + "end_effector_rotation": { + "absolute": true, + "rotation_type": "axis_angle", + "shape": [ + 3 + ], + "continuous": true + }, + "gripper_close": { + "absolute": true, + "rotation_type": null, + "shape": [ + 1 + ], + "continuous": true + } + } + }, + "embodiment_tag": "new_embodiment" + } +} \ No newline at end of file diff --git a/RKDv2.1/DA_MeanPool+LP/default/resolved_config.yaml b/RKDv2.1/DA_MeanPool+LP/default/resolved_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83926636f501ee58fb4ac8125ec83ae2b72ba8b9 --- /dev/null +++ b/RKDv2.1/DA_MeanPool+LP/default/resolved_config.yaml @@ -0,0 +1,63 @@ +experiment_name: RKD v2.1 DA, Student:MeanPool + LP +policy_type: groot_MGRKD +base_model_path: ${HOME}/groot_robocasa/Atomic26_baseline/checkpoint-60000 +output_root: ${HOME}/groot_robocasa/robocasa_v2/RKDv2.1/DA_MeanPool+LP +dataset: + dataset_soup: my_atomic26_human +training: + num_gpus: 1 + batch_size: 128 + seed: 42 +model: + rkd_student_reconstruction_head_pooling: mean +phases: +- name: phase2_mgrkd_da_student_meanpool_only + max_steps: 30000 + save_steps: 0 + trainable: + preset: processing_line_only + tune_llm: false + tune_visual: false + tune_projector: false + tune_diffusion_model: false + losses: + rkd_enabled: true + rkd_fm_loss_weight: 0.0 + rkd_loss_weight: 1.0 + rkd_relation_mode: flatten + rkd_loss_type: distance_angle + rkd_teacher_source: action_encoder + rkd_action_encoder_projector_enabled: true + rkd_action_encoder_projector_dim: 512 + rkd_action_encoder_projector_pooling: flatten + rkd_student_reconstruction_head_enabled: true + rkd_student_reconstruction_head_dim: 512 + rkd_student_reconstruction_head_hidden_dim: 512 + rkd_distance_loss_weight: 1.0 + rkd_angle_loss_weight: 2.0 + rkd_exclude_diagonal: true +- name: phase3_fm_mgrkd_da_student_meanpool_fixed_0p5 + max_steps: 30000 + save_steps: 0 + trainable: + tune_llm: false + tune_visual: false + tune_projector: true + tune_diffusion_model: true + losses: + rkd_enabled: true + rkd_fm_loss_weight: 1.0 + rkd_loss_weight: 0.5 + rkd_relation_mode: flatten + rkd_loss_type: distance_angle + rkd_teacher_source: action_encoder + rkd_action_encoder_projector_enabled: true + rkd_action_encoder_projector_dim: 512 + rkd_action_encoder_projector_pooling: flatten + rkd_student_reconstruction_head_enabled: true + rkd_student_reconstruction_head_dim: 512 + rkd_student_reconstruction_head_hidden_dim: 512 + rkd_distance_loss_weight: 1.0 + rkd_angle_loss_weight: 2.0 + rkd_exclude_diagonal: true +resolved_sweep: {} diff --git a/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/RKD v2.1 KL-DTW APHead ComponentProbMix.yaml b/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/RKD v2.1 KL-DTW APHead ComponentProbMix.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c8b57358cdd5427afe9e3878cc95dd0cc41f33c --- /dev/null +++ b/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/RKD v2.1 KL-DTW APHead ComponentProbMix.yaml @@ -0,0 +1,72 @@ +experiment_name: RKD v2.1 KL-DTW ComponentProbMix, Student:APHead +policy_type: groot_MGRKD +base_model_path: ${HOME}/groot_robocasa/Atomic26_baseline/checkpoint-60000 +output_root: ${HOME}/groot_robocasa/robocasa_v2/RKDv2.1/KL_DTW_APHead_ComponentProbMix +dataset: + dataset_soup: my_atomic26_human +training: + num_gpus: 2 + batch_size: 64 + seed: 42 +model: + rkd_student_reconstruction_head_pooling: attention +phases: +- name: phase2_mgrkd_kl_dtw_aphead_component_prob_mix_only + max_steps: 30000 + save_steps: 0 + trainable: + preset: processing_line_only + tune_llm: false + tune_visual: false + tune_projector: false + tune_diffusion_model: false + losses: + rkd_enabled: true + rkd_fm_loss_weight: 0.0 + rkd_loss_weight: 1.0 + rkd_relation_mode: flatten + rkd_loss_type: kl + rkd_teacher_source: dtw_action + rkd_action_temp: 0.1 + rkd_vlm_temp: 0.1 + rkd_action_encoder_projector_enabled: false + rkd_raw_action_projector_enabled: false + rkd_student_reconstruction_head_enabled: true + rkd_student_reconstruction_head_dim: 512 + rkd_student_reconstruction_head_hidden_dim: 512 + rkd_dtw_mix_mode: component_prob_mix + rkd_dtw_pos_weight: 1.0 + rkd_dtw_rot_weight: 1.0 + rkd_dtw_gripper_weight: 0.5 + rkd_dtw_normalize_components: true + rkd_dtw_timing_log_interval: 1000 + rkd_exclude_diagonal: true +- name: phase3_fm_mgrkd_kl_dtw_aphead_component_prob_mix_fixed_0p5 + max_steps: 30000 + save_steps: 0 + trainable: + tune_llm: false + tune_visual: false + tune_projector: true + tune_diffusion_model: true + losses: + rkd_enabled: true + rkd_fm_loss_weight: 1.0 + rkd_loss_weight: 0.5 + rkd_relation_mode: flatten + rkd_loss_type: kl + rkd_teacher_source: dtw_action + rkd_action_temp: 0.1 + rkd_vlm_temp: 0.1 + rkd_action_encoder_projector_enabled: false + rkd_raw_action_projector_enabled: false + rkd_student_reconstruction_head_enabled: true + rkd_student_reconstruction_head_dim: 512 + rkd_student_reconstruction_head_hidden_dim: 512 + rkd_dtw_mix_mode: component_prob_mix + rkd_dtw_pos_weight: 1.0 + rkd_dtw_rot_weight: 1.0 + rkd_dtw_gripper_weight: 0.5 + rkd_dtw_normalize_components: true + rkd_dtw_timing_log_interval: 1000 + rkd_exclude_diagonal: true diff --git a/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/phase2/experiment_cfg/metadata.json b/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/phase2/experiment_cfg/metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..300eb070963337016e16465f50ba8351ea2eb25c --- /dev/null +++ b/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/phase2/experiment_cfg/metadata.json @@ -0,0 +1,431 @@ +{ + "new_embodiment": { + "statistics": { + "state": { + "base_position": { + "max": [ + 7.3139495849609375, + 0.4876587688922882, + 0.7196521759033203 + ], + "min": [ + -4.94293737411499, + -6.890198230743408, + 0.6996827125549316 + ], + "mean": [ + 2.749288365724937, + -2.0455216239909983, + 0.700532551020533 + ], + "std": [ + 1.6786295897046262, + 1.312897969434244, + 0.00133751758906258 + ], + "q01": [ + -1.4241907881148037, + -5.1716547935950885, + 0.6999670898768614 + ], + "q99": [ + 6.198111545831555, + -0.5873531334104489, + 0.7038833014242587 + ] + }, + "base_rotation": { + "max": [ + 0.0, + 0.0, + 1.0, + 1.0 + ], + "min": [ + 0.0, + 0.0, + -1.0, + 0.0 + ], + "mean": [ + 0.0, + 0.0, + 0.23084999587450247, + 0.6238431088581847 + ], + "std": [ + 0.0, + 0.0, + 0.6556404371644047, + 0.3572252105732042 + ], + "q01": [ + 0.0, + 0.0, + -0.9999999999999999, + 1.7482354593273349e-06 + ], + "q99": [ + 0.0, + 0.0, + 0.9999999999999999, + 0.9999999999999999 + ] + }, + "end_effector_position_relative": { + "max": [ + 0.9014528393745422, + 0.8003877401351929, + 0.9829942584037781 + ], + "min": [ + -0.37471598386764526, + -0.8472502827644348, + -0.25070279836654663 + ], + "mean": [ + 0.28667332601863704, + -0.038315473170876704, + 0.45974658906777444 + ], + "std": [ + 0.17067058828305154, + 0.23569952037784195, + 0.21950702354181487 + ], + "q01": [ + 0.015172948963784228, + -0.44450502063220615, + 0.23314444103329338 + ], + "q99": [ + 0.5609069689572412, + 0.38454179128009547, + 0.7028102736473852 + ] + }, + "end_effector_rotation_relative": { + "max": [ + 0.9999998807907104, + 0.9984455108642578, + 0.9434727430343628, + 0.9062229990959167 + ], + "min": [ + -0.9999930262565613, + -0.9988337755203247, + -0.9618149995803833, + 2.1872274658107926e-07 + ], + "mean": [ + -0.25672297401336, + 0.024425335806687216, + -0.08740977164483847, + 0.16608739644018397 + ], + "std": [ + 0.7932585208392383, + 0.3102260195580416, + 0.3772472576315148, + 0.17451959300158773 + ], + "q01": [ + -0.99379006987882, + -0.5478102050038828, + -0.6101777952971636, + 0.002274085014134748 + ], + "q99": [ + 0.8804856038896288, + 0.5910611092873037, + 0.5216058042993562, + 0.5231965974012255 + ] + }, + "gripper_qpos": { + "max": [ + 0.055869169533252716, + 0.010916369967162609 + ], + "min": [ + -0.011436971835792065, + -0.05664053186774254 + ], + "mean": [ + 0.031651551516052555, + -0.03161481387920421 + ], + "std": [ + 0.013119769222217526, + 0.01306078225192402 + ], + "q01": [ + 0.0060999246446777336, + -0.04062601653964198 + ], + "q99": [ + 0.04054891621517283, + -0.0059163567113456345 + ] + } + }, + "action": { + "base_motion": { + "max": [ + 1.0, + 1.0, + 1.0, + 0.0 + ], + "min": [ + -1.0, + -1.0, + -1.0, + 0.0 + ], + "mean": [ + 0.008144896753718373, + -0.00018893135146078263, + -0.0008739062727221845, + 0.0 + ], + "std": [ + 0.11030045424364411, + 0.10148082570313594, + 0.08975698373467506, + 0.0 + ], + "q01": [ + -0.07287373067581518, + -0.09446994199569948, + -0.07702818259249572, + 0.0 + ], + "q99": [ + 0.04485485414407898, + 0.09626941605965177, + 0.07610665906122742, + 0.0 + ] + }, + "control_mode": { + "max": [ + 1.0 + ], + "min": [ + -1.0 + ], + "mean": [ + -0.9221974313425939 + ], + "std": [ + 0.38670194342612674 + ], + "q01": [ + -1.0 + ], + "q99": [ + -0.384693946883099 + ] + }, + "end_effector_position": { + "max": [ + 1.0, + 1.0, + 1.0 + ], + "min": [ + -1.0, + -1.0, + -1.0 + ], + "mean": [ + 0.01976129923017491, + -0.020870631344490034, + -0.05993476517349181 + ], + "std": [ + 0.4347024990804053, + 0.4221662976569997, + 0.3845506843044066 + ], + "q01": [ + -0.8835792939767807, + -0.9280919251713778, + -0.783361071470956 + ], + "q99": [ + 0.7622084438449307, + 0.8625125252609077, + 0.7470974753250706 + ] + }, + "end_effector_rotation": { + "max": [ + 1.0, + 1.0, + 1.0 + ], + "min": [ + -1.0, + -1.0, + -1.0 + ], + "mean": [ + 0.008089673068544335, + -0.026498254627927348, + 0.0029083551743059005 + ], + "std": [ + 0.11216734503733773, + 0.13206002613798629, + 0.12615870412692864 + ], + "q01": [ + -0.2814627972846672, + -0.4342386516115439, + -0.34489755628340574 + ], + "q99": [ + 0.31879353266939386, + 0.28411797683220563, + 0.3597718671669894 + ] + }, + "gripper_close": { + "max": [ + 1.0 + ], + "min": [ + -1.0 + ], + "mean": [ + -0.3824228874769045 + ], + "std": [ + 0.9240015096469786 + ], + "q01": [ + -1.0 + ], + "q99": [ + 0.5597272305813592 + ] + } + } + }, + "modalities": { + "video": { + "robot0_eye_in_hand": { + "resolution": [ + 256, + 256 + ], + "channels": 3, + "fps": 20.0 + }, + "robot0_agentview_left": { + "resolution": [ + 256, + 256 + ], + "channels": 3, + "fps": 20.0 + }, + "robot0_agentview_right": { + "resolution": [ + 256, + 256 + ], + "channels": 3, + "fps": 20.0 + } + }, + "state": { + "base_position": { + "absolute": true, + "rotation_type": null, + "shape": [ + 3 + ], + "continuous": true + }, + "base_rotation": { + "absolute": true, + "rotation_type": "quaternion", + "shape": [ + 4 + ], + "continuous": true + }, + "end_effector_position_relative": { + "absolute": true, + "rotation_type": null, + "shape": [ + 3 + ], + "continuous": true + }, + "end_effector_rotation_relative": { + "absolute": true, + "rotation_type": "quaternion", + "shape": [ + 4 + ], + "continuous": true + }, + "gripper_qpos": { + "absolute": true, + "rotation_type": null, + "shape": [ + 2 + ], + "continuous": true + } + }, + "action": { + "base_motion": { + "absolute": true, + "rotation_type": null, + "shape": [ + 4 + ], + "continuous": true + }, + "control_mode": { + "absolute": true, + "rotation_type": null, + "shape": [ + 1 + ], + "continuous": true + }, + "end_effector_position": { + "absolute": true, + "rotation_type": null, + "shape": [ + 3 + ], + "continuous": true + }, + "end_effector_rotation": { + "absolute": true, + "rotation_type": "axis_angle", + "shape": [ + 3 + ], + "continuous": true + }, + "gripper_close": { + "absolute": true, + "rotation_type": null, + "shape": [ + 1 + ], + "continuous": true + } + } + }, + "embodiment_tag": "new_embodiment" + } +} \ No newline at end of file diff --git a/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/resolved_config.yaml b/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/resolved_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96c7c9717429a70a6098ae36728592411b920535 --- /dev/null +++ b/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/resolved_config.yaml @@ -0,0 +1,73 @@ +experiment_name: RKD v2.1 KL-DTW ComponentProbMix, Student:APHead +policy_type: groot_MGRKD +base_model_path: ${HOME}/groot_robocasa/Atomic26_baseline/checkpoint-60000 +output_root: ${HOME}/groot_robocasa/robocasa_v2/RKDv2.1/KL_DTW_APHead_ComponentProbMix +dataset: + dataset_soup: my_atomic26_human +training: + num_gpus: 2 + batch_size: 64 + seed: 42 +model: + rkd_student_reconstruction_head_pooling: attention +phases: +- name: phase2_mgrkd_kl_dtw_aphead_component_prob_mix_only + max_steps: 30000 + save_steps: 0 + trainable: + preset: processing_line_only + tune_llm: false + tune_visual: false + tune_projector: false + tune_diffusion_model: false + losses: + rkd_enabled: true + rkd_fm_loss_weight: 0.0 + rkd_loss_weight: 1.0 + rkd_relation_mode: flatten + rkd_loss_type: kl + rkd_teacher_source: dtw_action + rkd_action_temp: 0.1 + rkd_vlm_temp: 0.1 + rkd_action_encoder_projector_enabled: false + rkd_raw_action_projector_enabled: false + rkd_student_reconstruction_head_enabled: true + rkd_student_reconstruction_head_dim: 512 + rkd_student_reconstruction_head_hidden_dim: 512 + rkd_dtw_mix_mode: component_prob_mix + rkd_dtw_pos_weight: 1.0 + rkd_dtw_rot_weight: 1.0 + rkd_dtw_gripper_weight: 0.5 + rkd_dtw_normalize_components: true + rkd_dtw_timing_log_interval: 1000 + rkd_exclude_diagonal: true +- name: phase3_fm_mgrkd_kl_dtw_aphead_component_prob_mix_fixed_0p5 + max_steps: 30000 + save_steps: 0 + trainable: + tune_llm: false + tune_visual: false + tune_projector: true + tune_diffusion_model: true + losses: + rkd_enabled: true + rkd_fm_loss_weight: 1.0 + rkd_loss_weight: 0.5 + rkd_relation_mode: flatten + rkd_loss_type: kl + rkd_teacher_source: dtw_action + rkd_action_temp: 0.1 + rkd_vlm_temp: 0.1 + rkd_action_encoder_projector_enabled: false + rkd_raw_action_projector_enabled: false + rkd_student_reconstruction_head_enabled: true + rkd_student_reconstruction_head_dim: 512 + rkd_student_reconstruction_head_hidden_dim: 512 + rkd_dtw_mix_mode: component_prob_mix + rkd_dtw_pos_weight: 1.0 + rkd_dtw_rot_weight: 1.0 + rkd_dtw_gripper_weight: 0.5 + rkd_dtw_normalize_components: true + rkd_dtw_timing_log_interval: 1000 + rkd_exclude_diagonal: true +resolved_sweep: {} diff --git a/RKDv2.1/KL_DTW_APHead_ComponentProbMix/launch_logs/train_gpu2-3_bs64_numgpu2_20260625_153201.log b/RKDv2.1/KL_DTW_APHead_ComponentProbMix/launch_logs/train_gpu2-3_bs64_numgpu2_20260625_153201.log new file mode 100644 index 0000000000000000000000000000000000000000..e2ee7ed4725b6235d103237c485eecf787f90b44 --- /dev/null +++ b/RKDv2.1/KL_DTW_APHead_ComponentProbMix/launch_logs/train_gpu2-3_bs64_numgpu2_20260625_153201.log @@ -0,0 +1,683 @@ +[robosuite WARNING] No private macro file found! (macros.py:57) +[robosuite WARNING] It is recommended to use a private macro file (macros.py:58) +[robosuite WARNING] To setup, run: python /home/ext_minje/clvla/benchmarks/robocasa_v2/robosuite/robosuite/scripts/setup_macros.py (macros.py:59) +[robosuite WARNING] Could not import robosuite_models. Some robots may not be available. If you want to use these robots, please install robosuite_models from source (https://github.com/ARISE-Initiative/robosuite_models) or through pip install. (__init__.py:30) +[robosuite WARNING] Could not load the mink-based whole-body IK. Make sure you install related import properly (e.g. pip install mink==0.0.5), otherwise you will not be able to use the default IK controller setting for GR1 robot. (__init__.py:40) +WARNING: mimicgen environments not imported since mimicgen is not installed! +/home/ext_minje/miniforge3/envs/robocasa/lib/python3.11/site-packages/albumentations/__init__.py:13: UserWarning: A new version of Albumentations is available: 2.0.8 (you have 1.4.18). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1. + check_for_updates() +`use_fast` is set to `True` but the image processor class does not have a fast version. Falling back to the slow version. + +================================================== +GR00T FINE-TUNING CONFIGURATION: +================================================== +config: /home/ext_minje/clvla/benchmarks/robocasa_v2/experiment_cfg/Experiments_v2/RKD v2.1 KL-DTW APHead ComponentProbMix.yaml +dataset_soup: None +output_dir: /tmp/gr00t +output_root: None +data_config: panda_omron +batch_size: 64 +max_steps: 300000 +num_gpus: 2 +save_steps: 20000 +run_name: None +save_total_limit: 100 +seed: 42 +base_model_path: nvidia/GR00T-N1.5-3B +tune_llm: False +tune_visual: False +tune_projector: True +tune_diffusion_model: True +resume: False +learning_rate: 3e-05 +weight_decay: 1e-05 +warmup_ratio: 0.05 +lora_rank: 0 +lora_alpha: 16 +lora_dropout: 0.1 +lora_full_model: False +dataloader_num_workers: 8 +report_to: wandb +embodiment_tag: new_embodiment +video_backend: opencv +balance_dataset_weights: True +balance_trajectory_weights: True +ds_weights_alpha: 0.4 +================================================== + +Using 2 GPUs +Running torchrun command: ['/home/ext_minje/miniforge3/envs/robocasa/bin/python', '-m', 'torch.distributed.run', '--standalone', '--nproc_per_node=2', '--nnodes=1', '/home/ext_minje/clvla/benchmarks/robocasa_v2/my_scripts/gr00t_finetune.py', '--config', '/home/ext_minje/clvla/benchmarks/robocasa_v2/experiment_cfg/Experiments_v2/RKD v2.1 KL-DTW APHead ComponentProbMix.yaml', '--batch-size', '64', '--num-gpus', '2', '--seed', '42'] + +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[robosuite WARNING] No private macro file found! (macros.py:57) +[robosuite WARNING] It is recommended to use a private macro file (macros.py:58) +[robosuite WARNING] To setup, run: python /home/ext_minje/clvla/benchmarks/robocasa_v2/robosuite/robosuite/scripts/setup_macros.py (macros.py:59) +[robosuite WARNING] No private macro file found! (macros.py:57) +[robosuite WARNING] It is recommended to use a private macro file (macros.py:58) +[robosuite WARNING] To setup, run: python /home/ext_minje/clvla/benchmarks/robocasa_v2/robosuite/robosuite/scripts/setup_macros.py (macros.py:59) +[robosuite WARNING] Could not import robosuite_models. Some robots may not be available. If you want to use these robots, please install robosuite_models from source (https://github.com/ARISE-Initiative/robosuite_models) or through pip install. (__init__.py:30) +[robosuite WARNING] Could not load the mink-based whole-body IK. Make sure you install related import properly (e.g. pip install mink==0.0.5), otherwise you will not be able to use the default IK controller setting for GR1 robot. (__init__.py:40) +[robosuite WARNING] Could not import robosuite_models. Some robots may not be available. If you want to use these robots, please install robosuite_models from source (https://github.com/ARISE-Initiative/robosuite_models) or through pip install. (__init__.py:30) +[robosuite WARNING] Could not load the mink-based whole-body IK. Make sure you install related import properly (e.g. pip install mink==0.0.5), otherwise you will not be able to use the default IK controller setting for GR1 robot. (__init__.py:40) +WARNING: mimicgen environments not imported since mimicgen is not installed! +WARNING: mimicgen environments not imported since mimicgen is not installed! +/home/ext_minje/miniforge3/envs/robocasa/lib/python3.11/site-packages/albumentations/__init__.py:13: UserWarning: A new version of Albumentations is available: 2.0.8 (you have 1.4.18). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1. + check_for_updates() +/home/ext_minje/miniforge3/envs/robocasa/lib/python3.11/site-packages/albumentations/__init__.py:13: UserWarning: A new version of Albumentations is available: 2.0.8 (you have 1.4.18). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1. + check_for_updates() +`use_fast` is set to `True` but the image processor class does not have a fast version. Falling back to the slow version. +`use_fast` is set to `True` but the image processor class does not have a fast version. Falling back to the slow version. + +================================================== +GR00T FINE-TUNING CONFIGURATION: +================================================== +config: /home/ext_minje/clvla/benchmarks/robocasa_v2/experiment_cfg/Experiments_v2/RKD v2.1 KL-DTW APHead ComponentProbMix.yaml +dataset_soup: None +output_dir: /tmp/gr00t +output_root: None +data_config: panda_omron +batch_size: 64 +max_steps: 300000 +num_gpus: 2 +save_steps: 20000 +run_name: None +save_total_limit: 100 +seed: 42 +base_model_path: nvidia/GR00T-N1.5-3B +tune_llm: False +tune_visual: False +tune_projector: True +tune_diffusion_model: True +resume: False +learning_rate: 3e-05 +weight_decay: 1e-05 +warmup_ratio: 0.05 +lora_rank: 0 +lora_alpha: 16 +lora_dropout: 0.1 +lora_full_model: False +dataloader_num_workers: 8 +report_to: wandb +embodiment_tag: new_embodiment +video_backend: opencv +balance_dataset_weights: True +balance_trajectory_weights: True +ds_weights_alpha: 0.4 +================================================== + +Using 2 GPUs + +================================================================================ +Starting sweep branch: default +Sweep vars: {} +================================================================================ + +-------------------------------------------------------------------------------- +Running phase 1: phase2_mgrkd_kl_dtw_aphead_component_prob_mix_only +Policy type: groot_MGRKD +Base model path: /home/ext_minje/groot_robocasa/Atomic26_baseline/checkpoint-60000 +Output dir: /home/ext_minje/groot_robocasa/robocasa_v2/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/phase2 +Trainable preset: processing_line_only +Policy overrides: {'rkd_student_reconstruction_head_pooling': 'attention', 'rkd_enabled': True, 'rkd_fm_loss_weight': 0.0, 'rkd_loss_weight': 1.0, 'rkd_relation_mode': 'flatten', 'rkd_loss_type': 'kl', 'rkd_teacher_source': 'dtw_action', 'rkd_action_temp': 0.1, 'rkd_vlm_temp': 0.1, 'rkd_action_encoder_projector_enabled': False, 'rkd_raw_action_projector_enabled': False, 'rkd_student_reconstruction_head_enabled': True, 'rkd_student_reconstruction_head_dim': 512, 'rkd_student_reconstruction_head_hidden_dim': 512, 'rkd_dtw_mix_mode': 'component_prob_mix', 'rkd_dtw_pos_weight': 1.0, 'rkd_dtw_rot_weight': 1.0, 'rkd_dtw_gripper_weight': 0.5, 'rkd_dtw_normalize_components': True, 'rkd_dtw_timing_log_interval': 1000, 'rkd_exclude_diagonal': True} +-------------------------------------------------------------------------------- + +[{'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/CloseBlenderLid/20250822/lerobot', 'horizon': 900, 'filter_key': '100_demos', 'task': 'CloseBlenderLid', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/CloseFridge/20250819/lerobot', 'horizon': 900, 'filter_key': '100_demos', 'task': 'CloseFridge', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/CloseToasterOvenDoor/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'CloseToasterOvenDoor', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/CoffeeSetupMug/20250819/lerobot', 'horizon': 600, 'filter_key': '100_demos', 'task': 'CoffeeSetupMug', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/NavigateKitchen/20250821/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'NavigateKitchen', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/OpenCabinet/20250819/lerobot', 'horizon': 1050, 'filter_key': '100_demos', 'task': 'OpenCabinet', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/OpenDrawer/20250819/lerobot', 'horizon': 750, 'filter_key': '100_demos', 'task': 'OpenDrawer', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/OpenStandMixerHead/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'OpenStandMixerHead', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceCounterToCabinet/20250819/lerobot', 'horizon': 750, 'filter_key': '100_demos', 'task': 'PickPlaceCounterToCabinet', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceCounterToStove/20250819/lerobot', 'horizon': 600, 'filter_key': '100_demos', 'task': 'PickPlaceCounterToStove', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceDrawerToCounter/20250820/lerobot', 'horizon': 750, 'filter_key': '100_demos', 'task': 'PickPlaceDrawerToCounter', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceSinkToCounter/20250819/lerobot', 'horizon': 900, 'filter_key': '100_demos', 'task': 'PickPlaceSinkToCounter', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceToasterToCounter/20250819/lerobot', 'horizon': 600, 'filter_key': '100_demos', 'task': 'PickPlaceToasterToCounter', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceCabinetToCounter/20250819/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'PickPlaceCabinetToCounter', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOnElectricKettle/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'TurnOnElectricKettle', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOnMicrowave/20250819/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'TurnOnMicrowave', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOffMicrowave/20250819/lerobot', 'horizon': 300, 'filter_key': '100_demos', 'task': 'TurnOffMicrowave', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/OpenElectricKettleLid/20250820/lerobot', 'horizon': 300, 'filter_key': '100_demos', 'task': 'OpenElectricKettleLid', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/SlideDishwasherRack/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'SlideDishwasherRack', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOnSinkFaucet/20250819/lerobot', 'horizon': 600, 'filter_key': '100_demos', 'task': 'TurnOnSinkFaucet', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnSinkSpout/20250820/lerobot', 'horizon': 300, 'filter_key': '100_demos', 'task': 'TurnSinkSpout', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOffSinkFaucet/20250819/lerobot', 'horizon': 300, 'filter_key': '100_demos', 'task': 'TurnOffSinkFaucet', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/AdjustWaterTemperature/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'AdjustWaterTemperature', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOffStove/20250819/lerobot', 'horizon': 750, 'filter_key': '100_demos', 'task': 'TurnOffStove', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOnStove/20250819/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'TurnOnStove', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/AdjustToasterOvenTemperature/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'AdjustToasterOvenTemperature', 'split': 'pretrain', 'source': 'human'}] +Using 100 subset demos for filter_key: 100_demos +/home/ext_minje/clvla/benchmarks/robocasa_v2/Isaac-GR00T/gr00t/data/transform/state_action.py:257: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + self.statistics[key] = torch.tensor(value) +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT + +================================================== +GR00T FINE-TUNING CONFIGURATION: +================================================== +config: /home/ext_minje/clvla/benchmarks/robocasa_v2/experiment_cfg/Experiments_v2/RKD v2.1 KL-DTW APHead ComponentProbMix.yaml +dataset_soup: None +output_dir: /tmp/gr00t +output_root: None +data_config: panda_omron +batch_size: 64 +max_steps: 300000 +num_gpus: 2 +save_steps: 20000 +run_name: None +save_total_limit: 100 +seed: 42 +base_model_path: nvidia/GR00T-N1.5-3B +tune_llm: False +tune_visual: False +tune_projector: True +tune_diffusion_model: True +resume: False +learning_rate: 3e-05 +weight_decay: 1e-05 +warmup_ratio: 0.05 +lora_rank: 0 +lora_alpha: 16 +lora_dropout: 0.1 +lora_full_model: False +dataloader_num_workers: 8 +report_to: wandb +embodiment_tag: new_embodiment +video_backend: opencv +balance_dataset_weights: True +balance_trajectory_weights: True +ds_weights_alpha: 0.4 +================================================== + +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +dataset weights: [1. 0.88494669 0.76443286 0.83888607 0.73785464 1.00501644 + 0.80029956 0.66065525 0.83967491 0.83515002 0.95047759 0.86791692 + 0.88329477 0.78510653 0.64381843 0.67404766 0.69697218 0.60443684 + 0.78448103 0.83447486 0.61146215 0.64384058 0.79545025 0.93845856 + 0.75517122 0.7973985 ] +Using 2 GPUs + +================================================================================ +Starting sweep branch: default +Sweep vars: {} +================================================================================ + +-------------------------------------------------------------------------------- +Running phase 1: phase2_mgrkd_kl_dtw_aphead_component_prob_mix_only +Policy type: groot_MGRKD +Base model path: /home/ext_minje/groot_robocasa/Atomic26_baseline/checkpoint-60000 +Output dir: /home/ext_minje/groot_robocasa/robocasa_v2/RKDv2.1/KL_DTW_APHead_ComponentProbMix/default/phase2 +Trainable preset: processing_line_only +Policy overrides: {'rkd_student_reconstruction_head_pooling': 'attention', 'rkd_enabled': True, 'rkd_fm_loss_weight': 0.0, 'rkd_loss_weight': 1.0, 'rkd_relation_mode': 'flatten', 'rkd_loss_type': 'kl', 'rkd_teacher_source': 'dtw_action', 'rkd_action_temp': 0.1, 'rkd_vlm_temp': 0.1, 'rkd_action_encoder_projector_enabled': False, 'rkd_raw_action_projector_enabled': False, 'rkd_student_reconstruction_head_enabled': True, 'rkd_student_reconstruction_head_dim': 512, 'rkd_student_reconstruction_head_hidden_dim': 512, 'rkd_dtw_mix_mode': 'component_prob_mix', 'rkd_dtw_pos_weight': 1.0, 'rkd_dtw_rot_weight': 1.0, 'rkd_dtw_gripper_weight': 0.5, 'rkd_dtw_normalize_components': True, 'rkd_dtw_timing_log_interval': 1000, 'rkd_exclude_diagonal': True} +-------------------------------------------------------------------------------- + +Loaded 26 datasets +[{'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/CloseBlenderLid/20250822/lerobot', 'horizon': 900, 'filter_key': '100_demos', 'task': 'CloseBlenderLid', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/CloseFridge/20250819/lerobot', 'horizon': 900, 'filter_key': '100_demos', 'task': 'CloseFridge', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/CloseToasterOvenDoor/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'CloseToasterOvenDoor', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/CoffeeSetupMug/20250819/lerobot', 'horizon': 600, 'filter_key': '100_demos', 'task': 'CoffeeSetupMug', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/NavigateKitchen/20250821/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'NavigateKitchen', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/OpenCabinet/20250819/lerobot', 'horizon': 1050, 'filter_key': '100_demos', 'task': 'OpenCabinet', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/OpenDrawer/20250819/lerobot', 'horizon': 750, 'filter_key': '100_demos', 'task': 'OpenDrawer', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/OpenStandMixerHead/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'OpenStandMixerHead', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceCounterToCabinet/20250819/lerobot', 'horizon': 750, 'filter_key': '100_demos', 'task': 'PickPlaceCounterToCabinet', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceCounterToStove/20250819/lerobot', 'horizon': 600, 'filter_key': '100_demos', 'task': 'PickPlaceCounterToStove', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceDrawerToCounter/20250820/lerobot', 'horizon': 750, 'filter_key': '100_demos', 'task': 'PickPlaceDrawerToCounter', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceSinkToCounter/20250819/lerobot', 'horizon': 900, 'filter_key': '100_demos', 'task': 'PickPlaceSinkToCounter', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceToasterToCounter/20250819/lerobot', 'horizon': 600, 'filter_key': '100_demos', 'task': 'PickPlaceToasterToCounter', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/PickPlaceCabinetToCounter/20250819/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'PickPlaceCabinetToCounter', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOnElectricKettle/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'TurnOnElectricKettle', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOnMicrowave/20250819/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'TurnOnMicrowave', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOffMicrowave/20250819/lerobot', 'horizon': 300, 'filter_key': '100_demos', 'task': 'TurnOffMicrowave', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/OpenElectricKettleLid/20250820/lerobot', 'horizon': 300, 'filter_key': '100_demos', 'task': 'OpenElectricKettleLid', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/SlideDishwasherRack/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'SlideDishwasherRack', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOnSinkFaucet/20250819/lerobot', 'horizon': 600, 'filter_key': '100_demos', 'task': 'TurnOnSinkFaucet', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnSinkSpout/20250820/lerobot', 'horizon': 300, 'filter_key': '100_demos', 'task': 'TurnSinkSpout', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOffSinkFaucet/20250819/lerobot', 'horizon': 300, 'filter_key': '100_demos', 'task': 'TurnOffSinkFaucet', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/AdjustWaterTemperature/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'AdjustWaterTemperature', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOffStove/20250819/lerobot', 'horizon': 750, 'filter_key': '100_demos', 'task': 'TurnOffStove', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/TurnOnStove/20250819/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'TurnOnStove', 'split': 'pretrain', 'source': 'human'}, {'path': '/home/ext_minje/clvla/benchmarks/robocasa_v2/robocasa/datasets/v1.0/pretrain/atomic/AdjustToasterOvenTemperature/20250820/lerobot', 'horizon': 450, 'filter_key': '100_demos', 'task': 'AdjustToasterOvenTemperature', 'split': 'pretrain', 'source': 'human'}] +Using 100 subset demos for filter_key: 100_demos +/home/ext_minje/clvla/benchmarks/robocasa_v2/Isaac-GR00T/gr00t/data/transform/state_action.py:257: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + self.statistics[key] = torch.tensor(value) +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Loading pretrained dual brain from /home/ext_minje/groot_robocasa/Atomic26_baseline/checkpoint-60000 +Tune backbone vision tower: False +Tune backbone LLM: False +Tune action head projector: False +Tune action head DiT: False +Model not found or avail in the huggingface hub. Loading from local path: /home/ext_minje/groot_robocasa/Atomic26_baseline/checkpoint-60000 +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +Using 100 subset demos for filter_key: 100_demos +Initialized dataset lerobot with EmbodimentTag.NEW_EMBODIMENT +dataset weights: [1. 0.88494669 0.76443286 0.83888607 0.73785464 1.00501644 + 0.80029956 0.66065525 0.83967491 0.83515002 0.95047759 0.86791692 + 0.88329477 0.78510653 0.64381843 0.67404766 0.69697218 0.60443684 + 0.78448103 0.83447486 0.61146215 0.64384058 0.79545025 0.93845856 + 0.75517122 0.7973985 ] +Loaded 26 datasets +Loading pretrained dual brain from /home/ext_minje/groot_robocasa/Atomic26_baseline/checkpoint-60000 +Tune backbone vision tower: False +Tune backbone LLM: False +Tune action head projector: False +Tune action head DiT: False +Model not found or avail in the huggingface hub. Loading from local path: /home/ext_minje/groot_robocasa/Atomic26_baseline/checkpoint-60000 +Tune backbone llm: False +Tune backbone visual: True +Total number of DiT parameters: 550386688 +Tune backbone llm: False +Tune backbone visual: True +Total number of DiT parameters: 550386688 +Total number of SelfAttentionTransformer parameters: 201433088 +Tune action head projector: True +Tune action head diffusion model: True +/home/ext_minje/miniforge3/envs/robocasa/lib/python3.11/site-packages/torch/nn/modules/transformer.py:382: UserWarning: enable_nested_tensor is True, but self.use_nested_tensor is False because encoder_layer.norm_first was True + warnings.warn( + Loading checkpoint shards: 0%| | 0/2 [00:00