diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/action_space_manifest.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/action_space_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d3561af110f523f790a58730abbd57a87db28b05 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/action_space_manifest.json @@ -0,0 +1,144 @@ +{ + "schema_version": 2, + "action_space": "ee_abs_rot6d", + "action_views": [ + "ee_abs_rot6d" + ], + "source_feature": "action.ee_abs_rot6d", + "canonical_feature": "action", + "dtype": "float32", + "shape": [ + 10 + ], + "names": [ + "x", + "y", + "z", + "r_col0_x", + "r_col0_y", + "r_col0_z", + "r_col1_x", + "r_col1_y", + "r_col1_z", + "gripper" + ], + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "dataset_total_frames": 218367, + "dataset_total_episodes": 4930, + "dataset_fps": 10, + "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4", + "stats": { + "min": [ + -0.21784618496894836, + -0.4474811851978302, + 0.1702737659215927, + -0.9999986290931702, + -0.9999991655349731, + -0.9999999403953552, + -0.9999990463256836, + -0.9999998211860657, + -0.9999988079071045, + 0.0 + ], + "max": [ + 0.6283698678016663, + 0.4420371949672699, + 1.051027774810791, + 0.9999999403953552, + 0.9999995231628418, + 1.0, + 0.9999995231628418, + 1.0, + 0.9999996423721313, + 0.03999999910593033 + ], + "mean": [ + 0.4040228037735655, + -0.05247618696989652, + 0.6993459771218912, + 0.07011111991957117, + -0.03520558904460197, + 0.14470242846953205, + 0.05491166606739672, + -0.19073058893378805, + -0.06988117661871228, + 0.021167300266524632 + ], + "std": [ + 0.11861927913357073, + 0.11318943557281885, + 0.17849160646987516, + 0.4812528791886511, + 0.7487196258736938, + 0.4251126804030219, + 0.47797229807769137, + 0.6267695346407992, + 0.5782954306815554, + 0.01996590431050817 + ], + "count": [ + 218367 + ], + "q01": [ + 0.23988881827972064, + -0.10387461883034811, + 0.6297243923476871, + -0.43622841955353464, + -0.6885486527617446, + -0.14103555388039019, + -0.4327210063987595, + -0.9304790915716145, + -0.3465462258490417, + -1.000000013351432e-10 + ], + "q10": [ + 0.3042306436755216, + -0.09609991620622624, + 0.6394063072781145, + -0.3351306352998669, + -0.5751963015002924, + -0.08490621283307866, + -0.33849701676224275, + -0.8340557712529717, + -0.29760290790884514, + -1.000000013351432e-10 + ], + "q50": [ + 0.4160225263528291, + -0.06267459152989589, + 0.6811777491060801, + 0.0543536902230377, + -0.14012251058819217, + 0.04772394280004727, + 0.03838119138716219, + -0.14865169094195635, + -0.2073069401172425, + 0.020772594934870346 + ], + "q90": [ + 0.46641008755679864, + 0.020233143101890064, + 0.8095135405259374, + 0.48172401592474906, + 0.7398737043070319, + 0.5970841215636613, + 0.4674752875999555, + 0.36281754777672676, + 0.5033361883834496, + 0.039998362213352394 + ], + "q99": [ + 0.47320723171849244, + 0.04114262476135705, + 0.8467967259270389, + 0.5867781409980988, + 0.8529584859257228, + 0.7953936054341789, + 0.5768269858732034, + 0.49353349323389695, + 0.7168914440776504, + 0.03999983541667277 + ] + } +} diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/action_space_manifest.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/action_space_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d3561af110f523f790a58730abbd57a87db28b05 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/action_space_manifest.json @@ -0,0 +1,144 @@ +{ + "schema_version": 2, + "action_space": "ee_abs_rot6d", + "action_views": [ + "ee_abs_rot6d" + ], + "source_feature": "action.ee_abs_rot6d", + "canonical_feature": "action", + "dtype": "float32", + "shape": [ + 10 + ], + "names": [ + "x", + "y", + "z", + "r_col0_x", + "r_col0_y", + "r_col0_z", + "r_col1_x", + "r_col1_y", + "r_col1_z", + "gripper" + ], + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "dataset_total_frames": 218367, + "dataset_total_episodes": 4930, + "dataset_fps": 10, + "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4", + "stats": { + "min": [ + -0.21784618496894836, + -0.4474811851978302, + 0.1702737659215927, + -0.9999986290931702, + -0.9999991655349731, + -0.9999999403953552, + -0.9999990463256836, + -0.9999998211860657, + -0.9999988079071045, + 0.0 + ], + "max": [ + 0.6283698678016663, + 0.4420371949672699, + 1.051027774810791, + 0.9999999403953552, + 0.9999995231628418, + 1.0, + 0.9999995231628418, + 1.0, + 0.9999996423721313, + 0.03999999910593033 + ], + "mean": [ + 0.4040228037735655, + -0.05247618696989652, + 0.6993459771218912, + 0.07011111991957117, + -0.03520558904460197, + 0.14470242846953205, + 0.05491166606739672, + -0.19073058893378805, + -0.06988117661871228, + 0.021167300266524632 + ], + "std": [ + 0.11861927913357073, + 0.11318943557281885, + 0.17849160646987516, + 0.4812528791886511, + 0.7487196258736938, + 0.4251126804030219, + 0.47797229807769137, + 0.6267695346407992, + 0.5782954306815554, + 0.01996590431050817 + ], + "count": [ + 218367 + ], + "q01": [ + 0.23988881827972064, + -0.10387461883034811, + 0.6297243923476871, + -0.43622841955353464, + -0.6885486527617446, + -0.14103555388039019, + -0.4327210063987595, + -0.9304790915716145, + -0.3465462258490417, + -1.000000013351432e-10 + ], + "q10": [ + 0.3042306436755216, + -0.09609991620622624, + 0.6394063072781145, + -0.3351306352998669, + -0.5751963015002924, + -0.08490621283307866, + -0.33849701676224275, + -0.8340557712529717, + -0.29760290790884514, + -1.000000013351432e-10 + ], + "q50": [ + 0.4160225263528291, + -0.06267459152989589, + 0.6811777491060801, + 0.0543536902230377, + -0.14012251058819217, + 0.04772394280004727, + 0.03838119138716219, + -0.14865169094195635, + -0.2073069401172425, + 0.020772594934870346 + ], + "q90": [ + 0.46641008755679864, + 0.020233143101890064, + 0.8095135405259374, + 0.48172401592474906, + 0.7398737043070319, + 0.5970841215636613, + 0.4674752875999555, + 0.36281754777672676, + 0.5033361883834496, + 0.039998362213352394 + ], + "q99": [ + 0.47320723171849244, + 0.04114262476135705, + 0.8467967259270389, + 0.5867781409980988, + 0.8529584859257228, + 0.7953936054341789, + 0.5768269858732034, + 0.49353349323389695, + 0.7168914440776504, + 0.03999983541667277 + ] + } +} diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/policy_postprocessor.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..4bb07c3b3b6c5e094251664271eb9f04152fe46f --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/policy_postprocessor.json @@ -0,0 +1,23 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "groot_action_unpack_unnormalize_v2", + "config": { + "env_action_dim": 10, + "normalize_min_max": true, + "clip_normalized_action": true, + "libero_gripper_action": false, + "libero_gripper_binarize": true + }, + "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/policy_preprocessor.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..d750c500708922d95b6fd0d98467a7bad5196d5d --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/policy_preprocessor.json @@ -0,0 +1,78 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "groot_n1_7_pack_inputs_v1", + "config": { + "state_horizon": 1, + "action_horizon": 16, + "valid_action_horizon": 16, + "video_horizon": null, + "max_state_dim": 132, + "max_action_dim": 132, + "language_key": "task", + "formalize_language": true, + "embodiment_tag": "new_embodiment", + "embodiment_mapping": { + "oxe_droid_relative_eef_relative_joint": 24, + "xdof_relative_eef_relative_joint": 27, + "xdof_relative_eef_relative_joint_subtask": 27, + "real_g1_relative_eef_relative_joints": 25, + "real_r1_pro_sharpa_relative_eef": 26, + "real_r1_pro_sharpa_relative_eef_human": 26, + "real_r1_pro_sharpa_relative_eef_maxinsights": 26, + "real_r1_pro_sharpa_relative_eef_mecka": 26, + "unitree_g1_full_body_with_waist_height_nav_cmd": 25, + "simpler_env_google": 0, + "simpler_env_widowx": 1, + "libero_sim": 2, + "new_embodiment": 10 + }, + "normalize_min_max": true, + "state_dropout_prob": 0.0, + "clip_outliers": true, + "use_percentiles": false, + "video_modality_keys": null, + "raw_stats": null, + "modality_config": null + }, + "state_file": "policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors" + }, + { + "registry_name": "groot_n1_7_vlm_encode_v1", + "config": { + "model_name": "nvidia/Cosmos-Reason2-2B", + "image_crop_size": [ + 230, + 230 + ], + "image_target_size": [ + 256, + 256 + ], + "shortest_image_edge": null, + "crop_fraction": null, + "use_albumentations": false, + "letter_box_transform": false, + "device": "cuda" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/prompt_manifest.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/prompt_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..21e3a43c712e5842c4616c53e544703bf52fb7ab --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/prompt_manifest.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "prompt_set": "detailed", + "source_file": "task_prompts.json", + "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc", + "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018", + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "task_count": 13, + "task_codes": [ + "5f", + "5g", + "1ext", + "3b", + "5b", + "3a", + "3c", + "5h", + "5d", + "3d", + "5e", + "5a", + "5c" + ], + "prompts": { + "5f": "Close the bottle in clockwise direction.", + "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.", + "1ext": "Find the arrow guide and open the indicated drawer.", + "3b": "Open the door, rotate counter-clockwise and push.", + "5b": "Open the bottle in counter-clockwise direction.", + "3a": "Open the door, rotate clockwise and push.", + "3c": "Open the door, rotate clockwise and pull.", + "5h": "Close the bottle in counter-clockwise direction.", + "5d": "Close the bottle in clockwise direction.", + "3d": "Open the door, rotate counter-clockwise and pull.", + "5e": "Close the bottle in counter-clockwise direction.", + "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.", + "5c": "Open the bottle in clockwise direction." + } +} diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/train_config.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1439aa49384771d8f2d5e7de29e06a6a9784ca90 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/pretrained_model/train_config.json @@ -0,0 +1,269 @@ +{ + "dataset": { + "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.0 + }, + "env": null, + "policy": { + "type": "groot_rkd", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 16 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.right_shoulder": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.guide": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 10 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model", + "pretrained_revision": null, + "chunk_size": 16, + "n_action_steps": 16, + "max_state_dim": 132, + "max_action_dim": 132, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "IDENTITY", + "ACTION": "IDENTITY" + }, + "base_model_path": "nvidia/GR00T-N1.7-3B", + "action_decode_transform": null, + "embodiment_tag": "new_embodiment", + "tune_llm": false, + "tune_visual": false, + "tune_projector": true, + "tune_diffusion_model": true, + "tune_vlln": true, + "tune_top_llm_layers": 0, + "num_inference_timesteps": null, + "rtc_ramp_rate": null, + "use_flash_attention": false, + "use_relative_actions": false, + "relative_exclude_joints": [], + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.9, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-05, + "warmup_steps": null, + "warmup_ratio": 0.05, + "use_bf16": true, + "model_params_fp32": true, + "image_size": [ + 256, + 256 + ], + "tokenizer_assets_repo": null, + "lora_rank": 0, + "lora_alpha": 16, + "lora_dropout": 0.1, + "lora_full_model": false, + "video_backend": "decord", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "dataset_paths": null, + "output_dir": "./tmp/gr00t", + "save_steps": 1000, + "max_steps": 10000, + "batch_size": 32, + "dataloader_num_workers": 8, + "report_to": "wandb", + "resume": false, + "action_expert_num_layers": 16, + "rkd_enabled": false, + "rkd_vae_checkpoint": null, + "rkd_student_pooling": "attention", + "rkd_student_attention_hidden_dim": 512, + "rkd_student_attention_output_dim": 512, + "rkd_fm_loss_weight": 1.0, + "rkd_loss_weight": 0.1, + "rkd_distance_weight": 1.0, + "rkd_angle_weight": 1.0 + }, + "reward_model": null, + "output_dir": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline", + "job_name": "INSIGHT_6D_DiT16_Baseline", + "resume": true, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 64, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 60000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 0, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 0.0001, + "weight_decay": 1e-05, + "grad_clip_norm": 1.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "jeqgwsec", + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01" +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/prompt_manifest.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/prompt_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..21e3a43c712e5842c4616c53e544703bf52fb7ab --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/prompt_manifest.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "prompt_set": "detailed", + "source_file": "task_prompts.json", + "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc", + "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018", + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "task_count": 13, + "task_codes": [ + "5f", + "5g", + "1ext", + "3b", + "5b", + "3a", + "3c", + "5h", + "5d", + "3d", + "5e", + "5a", + "5c" + ], + "prompts": { + "5f": "Close the bottle in clockwise direction.", + "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.", + "1ext": "Find the arrow guide and open the indicated drawer.", + "3b": "Open the door, rotate counter-clockwise and push.", + "5b": "Open the bottle in counter-clockwise direction.", + "3a": "Open the door, rotate clockwise and push.", + "3c": "Open the door, rotate clockwise and pull.", + "5h": "Close the bottle in counter-clockwise direction.", + "5d": "Close the bottle in clockwise direction.", + "3d": "Open the door, rotate counter-clockwise and pull.", + "5e": "Close the bottle in counter-clockwise direction.", + "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.", + "5c": "Open the bottle in clockwise direction." + } +} diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/resolved_config.yaml b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/resolved_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9f5d3920058f949f9d3cbbd490383d71a04fe57 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/resolved_config.yaml @@ -0,0 +1,26 @@ +dataset: + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 +policy: + type: groot_rkd + device: cuda + chunk_size: 16 + n_action_steps: 16 + push_to_hub: false + tune_llm: false + tune_visual: false + tune_top_llm_layers: 0 + action_expert_num_layers: 16 + tune_projector: true + tune_diffusion_model: true + tune_vlln: true + rkd_enabled: false +seed: 42 +batch_size: 64 +steps: 60000 +log_freq: 200 +output_dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/16/Baseline +job_name: INSIGHT_6D_DiT16_Baseline +wandb: + enable: true + disable_artifact: true diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/source_config.yaml b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/source_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7403e5629320ae470a158921d45712d85e7aa940 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/source_config.yaml @@ -0,0 +1,69 @@ +# INSIGHT GR00T N1.7 training config +# INSIGHT wrapper-only fields are stripped before LeRobot config parsing. +insight: + checkpoint_interval: 5K + checkpoint_slots: 2 + # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta + action_space: ee_abs_rot6d + # Options: coarse | detailed + prompt_set: detailed + save_steps: ["10K", "30K", "60K"] + +dataset: + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2 + +policy: + type: groot_rkd + device: cuda + chunk_size: 16 + n_action_steps: 16 + push_to_hub: false + # Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen. + # VLM + tune_llm: false + tune_visual: false + tune_top_llm_layers: 0 + # Action Expert + action_expert_num_layers: 16 + tune_projector: true + tune_diffusion_model: true + tune_vlln: true + # RKD policy is used only to expose the shared DiT-depth setting. + # This remains the action-loss-only control. + rkd_enabled: false + + +seed: 42 +batch_size: 64 +steps: 60000 +log_freq: 200 +output_dir: ${HOME}/groot_insight/Abs_6D/DiT_Layer/16/Baseline +job_name: INSIGHT_6D_DiT16_Baseline + +wandb: + enable: true + disable_artifact: true + +# Omitted because these match LeRobot/GR00T defaults or are unused in this setup: +# +# dataset: +# eval_split: 0.0 +# image_transforms: +# enable: false # default; set true only for train-image augmentation. +# +# policy: +# base_model_path: nvidia/GR00T-N1.7-3B +# embodiment_tag: new_embodiment +# use_relative_actions: false +# use_bf16: true +# repo_id: Whalswp/INSIGHT # unused while push_to_hub is false +# +# save_checkpoint: true +# use_policy_training_preset: true +# eval_steps: 0 +# env_eval_freq: 0 +# +# insight.prompt_set options: +# coarse: High-level task text that requires following the visual guide. +# detailed: Direction- and interaction-specific text, including rotation and push/pull details. diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/optimizer_param_groups.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..ffea2e098b2b4419f5d3748b3fbd7cb3cbe3f643 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/optimizer_param_groups.json @@ -0,0 +1,353 @@ +[ + { + "lr": 1.7322955673980678e-06, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154 + ] + }, + { + "weight_decay": 0.0, + "lr": 1.7322955673980678e-06, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.0001, + "params": [ + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312 + ] + } +] \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/scheduler_state.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4d53c25f5cf2238d530247118fcc7f883c3edd59 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/scheduler_state.json @@ -0,0 +1,18 @@ +{ + "base_lrs": [ + 0.0001, + 0.0001 + ], + "last_epoch": 55000, + "_step_count": 55001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.7322955673980678e-06, + 1.7322955673980678e-06 + ], + "lr_lambdas": [ + null, + null + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/training_step.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..93d0e80124dd5e20fd5079d8367b704a54658e74 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_00/training_state/training_step.json @@ -0,0 +1,5 @@ +{ + "step": 55000, + "num_processes": 1, + "batch_size": 64 +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/action_space_manifest.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/action_space_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d3561af110f523f790a58730abbd57a87db28b05 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/action_space_manifest.json @@ -0,0 +1,144 @@ +{ + "schema_version": 2, + "action_space": "ee_abs_rot6d", + "action_views": [ + "ee_abs_rot6d" + ], + "source_feature": "action.ee_abs_rot6d", + "canonical_feature": "action", + "dtype": "float32", + "shape": [ + 10 + ], + "names": [ + "x", + "y", + "z", + "r_col0_x", + "r_col0_y", + "r_col0_z", + "r_col1_x", + "r_col1_y", + "r_col1_z", + "gripper" + ], + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "dataset_total_frames": 218367, + "dataset_total_episodes": 4930, + "dataset_fps": 10, + "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4", + "stats": { + "min": [ + -0.21784618496894836, + -0.4474811851978302, + 0.1702737659215927, + -0.9999986290931702, + -0.9999991655349731, + -0.9999999403953552, + -0.9999990463256836, + -0.9999998211860657, + -0.9999988079071045, + 0.0 + ], + "max": [ + 0.6283698678016663, + 0.4420371949672699, + 1.051027774810791, + 0.9999999403953552, + 0.9999995231628418, + 1.0, + 0.9999995231628418, + 1.0, + 0.9999996423721313, + 0.03999999910593033 + ], + "mean": [ + 0.4040228037735655, + -0.05247618696989652, + 0.6993459771218912, + 0.07011111991957117, + -0.03520558904460197, + 0.14470242846953205, + 0.05491166606739672, + -0.19073058893378805, + -0.06988117661871228, + 0.021167300266524632 + ], + "std": [ + 0.11861927913357073, + 0.11318943557281885, + 0.17849160646987516, + 0.4812528791886511, + 0.7487196258736938, + 0.4251126804030219, + 0.47797229807769137, + 0.6267695346407992, + 0.5782954306815554, + 0.01996590431050817 + ], + "count": [ + 218367 + ], + "q01": [ + 0.23988881827972064, + -0.10387461883034811, + 0.6297243923476871, + -0.43622841955353464, + -0.6885486527617446, + -0.14103555388039019, + -0.4327210063987595, + -0.9304790915716145, + -0.3465462258490417, + -1.000000013351432e-10 + ], + "q10": [ + 0.3042306436755216, + -0.09609991620622624, + 0.6394063072781145, + -0.3351306352998669, + -0.5751963015002924, + -0.08490621283307866, + -0.33849701676224275, + -0.8340557712529717, + -0.29760290790884514, + -1.000000013351432e-10 + ], + "q50": [ + 0.4160225263528291, + -0.06267459152989589, + 0.6811777491060801, + 0.0543536902230377, + -0.14012251058819217, + 0.04772394280004727, + 0.03838119138716219, + -0.14865169094195635, + -0.2073069401172425, + 0.020772594934870346 + ], + "q90": [ + 0.46641008755679864, + 0.020233143101890064, + 0.8095135405259374, + 0.48172401592474906, + 0.7398737043070319, + 0.5970841215636613, + 0.4674752875999555, + 0.36281754777672676, + 0.5033361883834496, + 0.039998362213352394 + ], + "q99": [ + 0.47320723171849244, + 0.04114262476135705, + 0.8467967259270389, + 0.5867781409980988, + 0.8529584859257228, + 0.7953936054341789, + 0.5768269858732034, + 0.49353349323389695, + 0.7168914440776504, + 0.03999983541667277 + ] + } +} diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/action_space_manifest.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/action_space_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d3561af110f523f790a58730abbd57a87db28b05 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/action_space_manifest.json @@ -0,0 +1,144 @@ +{ + "schema_version": 2, + "action_space": "ee_abs_rot6d", + "action_views": [ + "ee_abs_rot6d" + ], + "source_feature": "action.ee_abs_rot6d", + "canonical_feature": "action", + "dtype": "float32", + "shape": [ + 10 + ], + "names": [ + "x", + "y", + "z", + "r_col0_x", + "r_col0_y", + "r_col0_z", + "r_col1_x", + "r_col1_y", + "r_col1_z", + "gripper" + ], + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "dataset_total_frames": 218367, + "dataset_total_episodes": 4930, + "dataset_fps": 10, + "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4", + "stats": { + "min": [ + -0.21784618496894836, + -0.4474811851978302, + 0.1702737659215927, + -0.9999986290931702, + -0.9999991655349731, + -0.9999999403953552, + -0.9999990463256836, + -0.9999998211860657, + -0.9999988079071045, + 0.0 + ], + "max": [ + 0.6283698678016663, + 0.4420371949672699, + 1.051027774810791, + 0.9999999403953552, + 0.9999995231628418, + 1.0, + 0.9999995231628418, + 1.0, + 0.9999996423721313, + 0.03999999910593033 + ], + "mean": [ + 0.4040228037735655, + -0.05247618696989652, + 0.6993459771218912, + 0.07011111991957117, + -0.03520558904460197, + 0.14470242846953205, + 0.05491166606739672, + -0.19073058893378805, + -0.06988117661871228, + 0.021167300266524632 + ], + "std": [ + 0.11861927913357073, + 0.11318943557281885, + 0.17849160646987516, + 0.4812528791886511, + 0.7487196258736938, + 0.4251126804030219, + 0.47797229807769137, + 0.6267695346407992, + 0.5782954306815554, + 0.01996590431050817 + ], + "count": [ + 218367 + ], + "q01": [ + 0.23988881827972064, + -0.10387461883034811, + 0.6297243923476871, + -0.43622841955353464, + -0.6885486527617446, + -0.14103555388039019, + -0.4327210063987595, + -0.9304790915716145, + -0.3465462258490417, + -1.000000013351432e-10 + ], + "q10": [ + 0.3042306436755216, + -0.09609991620622624, + 0.6394063072781145, + -0.3351306352998669, + -0.5751963015002924, + -0.08490621283307866, + -0.33849701676224275, + -0.8340557712529717, + -0.29760290790884514, + -1.000000013351432e-10 + ], + "q50": [ + 0.4160225263528291, + -0.06267459152989589, + 0.6811777491060801, + 0.0543536902230377, + -0.14012251058819217, + 0.04772394280004727, + 0.03838119138716219, + -0.14865169094195635, + -0.2073069401172425, + 0.020772594934870346 + ], + "q90": [ + 0.46641008755679864, + 0.020233143101890064, + 0.8095135405259374, + 0.48172401592474906, + 0.7398737043070319, + 0.5970841215636613, + 0.4674752875999555, + 0.36281754777672676, + 0.5033361883834496, + 0.039998362213352394 + ], + "q99": [ + 0.47320723171849244, + 0.04114262476135705, + 0.8467967259270389, + 0.5867781409980988, + 0.8529584859257228, + 0.7953936054341789, + 0.5768269858732034, + 0.49353349323389695, + 0.7168914440776504, + 0.03999983541667277 + ] + } +} diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/config.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e285a793b49463c535b7a35727257ade26d07b77 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/config.json @@ -0,0 +1,118 @@ +{ + "type": "groot_rkd", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 16 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.right_shoulder": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.guide": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 10 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model", + "pretrained_revision": null, + "chunk_size": 16, + "n_action_steps": 16, + "max_state_dim": 132, + "max_action_dim": 132, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "IDENTITY", + "ACTION": "IDENTITY" + }, + "base_model_path": "nvidia/GR00T-N1.7-3B", + "action_decode_transform": null, + "embodiment_tag": "new_embodiment", + "tune_llm": false, + "tune_visual": false, + "tune_projector": true, + "tune_diffusion_model": true, + "tune_vlln": true, + "tune_top_llm_layers": 0, + "num_inference_timesteps": null, + "rtc_ramp_rate": null, + "use_flash_attention": false, + "use_relative_actions": false, + "relative_exclude_joints": [], + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.9, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-05, + "warmup_steps": null, + "warmup_ratio": 0.05, + "use_bf16": true, + "model_params_fp32": true, + "image_size": [ + 256, + 256 + ], + "tokenizer_assets_repo": null, + "lora_rank": 0, + "lora_alpha": 16, + "lora_dropout": 0.1, + "lora_full_model": false, + "video_backend": "decord", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "dataset_paths": null, + "output_dir": "./tmp/gr00t", + "save_steps": 1000, + "max_steps": 10000, + "batch_size": 32, + "dataloader_num_workers": 8, + "report_to": "wandb", + "resume": false, + "action_expert_num_layers": 16, + "rkd_enabled": false, + "rkd_vae_checkpoint": null, + "rkd_student_pooling": "attention", + "rkd_student_attention_hidden_dim": 512, + "rkd_student_attention_output_dim": 512, + "rkd_fm_loss_weight": 1.0, + "rkd_loss_weight": 0.1, + "rkd_distance_weight": 1.0, + "rkd_angle_weight": 1.0 +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/policy_postprocessor.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..4bb07c3b3b6c5e094251664271eb9f04152fe46f --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/policy_postprocessor.json @@ -0,0 +1,23 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "groot_action_unpack_unnormalize_v2", + "config": { + "env_action_dim": 10, + "normalize_min_max": true, + "clip_normalized_action": true, + "libero_gripper_action": false, + "libero_gripper_binarize": true + }, + "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/policy_preprocessor.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..d750c500708922d95b6fd0d98467a7bad5196d5d --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/policy_preprocessor.json @@ -0,0 +1,78 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "groot_n1_7_pack_inputs_v1", + "config": { + "state_horizon": 1, + "action_horizon": 16, + "valid_action_horizon": 16, + "video_horizon": null, + "max_state_dim": 132, + "max_action_dim": 132, + "language_key": "task", + "formalize_language": true, + "embodiment_tag": "new_embodiment", + "embodiment_mapping": { + "oxe_droid_relative_eef_relative_joint": 24, + "xdof_relative_eef_relative_joint": 27, + "xdof_relative_eef_relative_joint_subtask": 27, + "real_g1_relative_eef_relative_joints": 25, + "real_r1_pro_sharpa_relative_eef": 26, + "real_r1_pro_sharpa_relative_eef_human": 26, + "real_r1_pro_sharpa_relative_eef_maxinsights": 26, + "real_r1_pro_sharpa_relative_eef_mecka": 26, + "unitree_g1_full_body_with_waist_height_nav_cmd": 25, + "simpler_env_google": 0, + "simpler_env_widowx": 1, + "libero_sim": 2, + "new_embodiment": 10 + }, + "normalize_min_max": true, + "state_dropout_prob": 0.0, + "clip_outliers": true, + "use_percentiles": false, + "video_modality_keys": null, + "raw_stats": null, + "modality_config": null + }, + "state_file": "policy_preprocessor_step_2_groot_n1_7_pack_inputs_v1.safetensors" + }, + { + "registry_name": "groot_n1_7_vlm_encode_v1", + "config": { + "model_name": "nvidia/Cosmos-Reason2-2B", + "image_crop_size": [ + 230, + 230 + ], + "image_target_size": [ + 256, + 256 + ], + "shortest_image_edge": null, + "crop_fraction": null, + "use_albumentations": false, + "letter_box_transform": false, + "device": "cuda" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/prompt_manifest.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/prompt_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..21e3a43c712e5842c4616c53e544703bf52fb7ab --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/prompt_manifest.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "prompt_set": "detailed", + "source_file": "task_prompts.json", + "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc", + "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018", + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "task_count": 13, + "task_codes": [ + "5f", + "5g", + "1ext", + "3b", + "5b", + "3a", + "3c", + "5h", + "5d", + "3d", + "5e", + "5a", + "5c" + ], + "prompts": { + "5f": "Close the bottle in clockwise direction.", + "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.", + "1ext": "Find the arrow guide and open the indicated drawer.", + "3b": "Open the door, rotate counter-clockwise and push.", + "5b": "Open the bottle in counter-clockwise direction.", + "3a": "Open the door, rotate clockwise and push.", + "3c": "Open the door, rotate clockwise and pull.", + "5h": "Close the bottle in counter-clockwise direction.", + "5d": "Close the bottle in clockwise direction.", + "3d": "Open the door, rotate counter-clockwise and pull.", + "5e": "Close the bottle in counter-clockwise direction.", + "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.", + "5c": "Open the bottle in clockwise direction." + } +} diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/train_config.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1439aa49384771d8f2d5e7de29e06a6a9784ca90 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/train_config.json @@ -0,0 +1,269 @@ +{ + "dataset": { + "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.0 + }, + "env": null, + "policy": { + "type": "groot_rkd", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 16 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.right_shoulder": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.guide": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 10 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model", + "pretrained_revision": null, + "chunk_size": 16, + "n_action_steps": 16, + "max_state_dim": 132, + "max_action_dim": 132, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "IDENTITY", + "ACTION": "IDENTITY" + }, + "base_model_path": "nvidia/GR00T-N1.7-3B", + "action_decode_transform": null, + "embodiment_tag": "new_embodiment", + "tune_llm": false, + "tune_visual": false, + "tune_projector": true, + "tune_diffusion_model": true, + "tune_vlln": true, + "tune_top_llm_layers": 0, + "num_inference_timesteps": null, + "rtc_ramp_rate": null, + "use_flash_attention": false, + "use_relative_actions": false, + "relative_exclude_joints": [], + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.9, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-05, + "warmup_steps": null, + "warmup_ratio": 0.05, + "use_bf16": true, + "model_params_fp32": true, + "image_size": [ + 256, + 256 + ], + "tokenizer_assets_repo": null, + "lora_rank": 0, + "lora_alpha": 16, + "lora_dropout": 0.1, + "lora_full_model": false, + "video_backend": "decord", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "dataset_paths": null, + "output_dir": "./tmp/gr00t", + "save_steps": 1000, + "max_steps": 10000, + "batch_size": 32, + "dataloader_num_workers": 8, + "report_to": "wandb", + "resume": false, + "action_expert_num_layers": 16, + "rkd_enabled": false, + "rkd_vae_checkpoint": null, + "rkd_student_pooling": "attention", + "rkd_student_attention_hidden_dim": 512, + "rkd_student_attention_output_dim": 512, + "rkd_fm_loss_weight": 1.0, + "rkd_loss_weight": 0.1, + "rkd_distance_weight": 1.0, + "rkd_angle_weight": 1.0 + }, + "reward_model": null, + "output_dir": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline", + "job_name": "INSIGHT_6D_DiT16_Baseline", + "resume": true, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 64, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 60000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 0, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 0.0001, + "weight_decay": 1e-05, + "grad_clip_norm": 1.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "jeqgwsec", + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01" +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/prompt_manifest.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/prompt_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..21e3a43c712e5842c4616c53e544703bf52fb7ab --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/prompt_manifest.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "prompt_set": "detailed", + "source_file": "task_prompts.json", + "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc", + "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018", + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "task_count": 13, + "task_codes": [ + "5f", + "5g", + "1ext", + "3b", + "5b", + "3a", + "3c", + "5h", + "5d", + "3d", + "5e", + "5a", + "5c" + ], + "prompts": { + "5f": "Close the bottle in clockwise direction.", + "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.", + "1ext": "Find the arrow guide and open the indicated drawer.", + "3b": "Open the door, rotate counter-clockwise and push.", + "5b": "Open the bottle in counter-clockwise direction.", + "3a": "Open the door, rotate clockwise and push.", + "3c": "Open the door, rotate clockwise and pull.", + "5h": "Close the bottle in counter-clockwise direction.", + "5d": "Close the bottle in clockwise direction.", + "3d": "Open the door, rotate counter-clockwise and pull.", + "5e": "Close the bottle in counter-clockwise direction.", + "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.", + "5c": "Open the bottle in clockwise direction." + } +} diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/resolved_config.yaml b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/resolved_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9f5d3920058f949f9d3cbbd490383d71a04fe57 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/resolved_config.yaml @@ -0,0 +1,26 @@ +dataset: + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 +policy: + type: groot_rkd + device: cuda + chunk_size: 16 + n_action_steps: 16 + push_to_hub: false + tune_llm: false + tune_visual: false + tune_top_llm_layers: 0 + action_expert_num_layers: 16 + tune_projector: true + tune_diffusion_model: true + tune_vlln: true + rkd_enabled: false +seed: 42 +batch_size: 64 +steps: 60000 +log_freq: 200 +output_dir: /home/ext_minje/groot_insight/Abs_6D/DiT_Layer/16/Baseline +job_name: INSIGHT_6D_DiT16_Baseline +wandb: + enable: true + disable_artifact: true diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/source_config.yaml b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/source_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7403e5629320ae470a158921d45712d85e7aa940 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/source_config.yaml @@ -0,0 +1,69 @@ +# INSIGHT GR00T N1.7 training config +# INSIGHT wrapper-only fields are stripped before LeRobot config parsing. +insight: + checkpoint_interval: 5K + checkpoint_slots: 2 + # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta + action_space: ee_abs_rot6d + # Options: coarse | detailed + prompt_set: detailed + save_steps: ["10K", "30K", "60K"] + +dataset: + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2 + +policy: + type: groot_rkd + device: cuda + chunk_size: 16 + n_action_steps: 16 + push_to_hub: false + # Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen. + # VLM + tune_llm: false + tune_visual: false + tune_top_llm_layers: 0 + # Action Expert + action_expert_num_layers: 16 + tune_projector: true + tune_diffusion_model: true + tune_vlln: true + # RKD policy is used only to expose the shared DiT-depth setting. + # This remains the action-loss-only control. + rkd_enabled: false + + +seed: 42 +batch_size: 64 +steps: 60000 +log_freq: 200 +output_dir: ${HOME}/groot_insight/Abs_6D/DiT_Layer/16/Baseline +job_name: INSIGHT_6D_DiT16_Baseline + +wandb: + enable: true + disable_artifact: true + +# Omitted because these match LeRobot/GR00T defaults or are unused in this setup: +# +# dataset: +# eval_split: 0.0 +# image_transforms: +# enable: false # default; set true only for train-image augmentation. +# +# policy: +# base_model_path: nvidia/GR00T-N1.7-3B +# embodiment_tag: new_embodiment +# use_relative_actions: false +# use_bf16: true +# repo_id: Whalswp/INSIGHT # unused while push_to_hub is false +# +# save_checkpoint: true +# use_policy_training_preset: true +# eval_steps: 0 +# env_eval_freq: 0 +# +# insight.prompt_set options: +# coarse: High-level task text that requires following the visual guide. +# detailed: Direction- and interaction-specific text, including rotation and push/pull details. diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/optimizer_param_groups.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..94476a027f3bc7c82de3bc5fc81c295eb985aaf0 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/optimizer_param_groups.json @@ -0,0 +1,353 @@ +[ + { + "lr": 0.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154 + ] + }, + { + "weight_decay": 0.0, + "lr": 0.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.0001, + "params": [ + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312 + ] + } +] \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/scheduler_state.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a8301608cef57d586c44a4fcc79905580b0d63bf --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/scheduler_state.json @@ -0,0 +1,18 @@ +{ + "base_lrs": [ + 0.0001, + 0.0001 + ], + "last_epoch": 60000, + "_step_count": 60001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.0, + 0.0 + ], + "lr_lambdas": [ + null, + null + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/training_step.json b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..0c795acfc7d3498b7e790af15ae93492dc3be1b6 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/training_state/training_step.json @@ -0,0 +1,5 @@ +{ + "step": 60000, + "num_processes": 1, + "batch_size": 64 +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/debug.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/debug.log index 26b2cff6a251f4726b8b71cb7ca17acf07def143..bca46d7000270c3195be556d84f3b7cea74ac678 100644 --- a/Abs_6D/DiT_Layer/16/Baseline/wandb/debug.log +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/debug.log @@ -1,20 +1,65 @@ -2026-07-23 01:33:42,279 INFO MainThread:814713 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2 -2026-07-23 01:33:42,279 INFO MainThread:814713 [wandb_setup.py:_flush():81] Configure stats pid to 814713 -2026-07-23 01:33:42,280 INFO MainThread:814713 [wandb_setup.py:_flush():81] Loading settings from environment variables -2026-07-23 01:33:42,280 INFO MainThread:814713 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug.log -2026-07-23 01:33:42,280 INFO MainThread:814713 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-internal.log -2026-07-23 01:33:42,280 INFO MainThread:814713 [wandb_init.py:init():766] calling init triggers -2026-07-23 01:33:42,280 INFO MainThread:814713 [wandb_init.py:init():771] wandb.init called with sweep_config: {} -config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': , 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline', 'job_name': 'INSIGHT_6D_DiT16_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'svym8k0w', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000', '_wandb': {}} -2026-07-23 01:33:42,280 INFO MainThread:814713 [wandb_init.py:init():814] starting backend -2026-07-23 01:33:42,497 INFO MainThread:814713 [wandb_init.py:init():829] sending inform_init request -2026-07-23 01:33:42,962 INFO MainThread:814713 [wandb_init.py:init():834] backend started and connected -2026-07-23 01:33:42,964 INFO MainThread:814713 [wandb_init.py:init():904] updated telemetry -2026-07-23 01:33:42,969 INFO MainThread:814713 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout -2026-07-23 01:33:43,656 INFO MainThread:814713 [wandb_init.py:init():967] run resumed -2026-07-23 01:33:43,659 INFO MainThread:814713 [wandb_init.py:init():972] starting run threads in backend -2026-07-23 01:33:43,720 INFO MainThread:814713 [wandb_run.py:_console_start():2523] atexit reg -2026-07-23 01:33:43,720 INFO MainThread:814713 [wandb_run.py:_redirect():2373] redirect: wrap_raw -2026-07-23 01:33:43,720 INFO MainThread:814713 [wandb_run.py:_redirect():2442] Wrapping output streams. -2026-07-23 01:33:43,720 INFO MainThread:814713 [wandb_run.py:_redirect():2465] Redirects installed. -2026-07-23 01:33:43,722 INFO MainThread:814713 [wandb_init.py:init():1010] run started, returning control to user process +2026-07-23 16:35:01,612 INFO MainThread:1458 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2 +2026-07-23 16:35:01,612 INFO MainThread:1458 [wandb_setup.py:_flush():81] Configure stats pid to 1458 +2026-07-23 16:35:01,612 INFO MainThread:1458 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-07-23 16:35:01,612 INFO MainThread:1458 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/logs/debug.log +2026-07-23 16:35:01,612 INFO MainThread:1458 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/logs/debug-internal.log +2026-07-23 16:35:01,612 INFO MainThread:1458 [wandb_init.py:init():766] calling init triggers +2026-07-23 16:35:01,612 INFO MainThread:1458 [wandb_init.py:init():771] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': , 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline', 'job_name': 'INSIGHT_6D_DiT16_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'jeqgwsec', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01', '_wandb': {}} +2026-07-23 16:35:01,612 INFO MainThread:1458 [wandb_init.py:init():814] starting backend +2026-07-23 16:35:01,833 INFO MainThread:1458 [wandb_init.py:init():829] sending inform_init request +2026-07-23 16:35:02,308 INFO MainThread:1458 [wandb_init.py:init():834] backend started and connected +2026-07-23 16:35:02,310 INFO MainThread:1458 [wandb_init.py:init():904] updated telemetry +2026-07-23 16:35:02,310 INFO MainThread:1458 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-23 16:35:03,987 INFO MainThread:1458 [wandb_init.py:init():967] run resumed +2026-07-23 16:35:03,989 INFO MainThread:1458 [wandb_init.py:init():972] starting run threads in backend +2026-07-23 16:35:04,041 INFO MainThread:1458 [wandb_run.py:_console_start():2523] atexit reg +2026-07-23 16:35:04,041 INFO MainThread:1458 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-23 16:35:04,041 INFO MainThread:1458 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-23 16:35:04,041 INFO MainThread:1458 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-23 16:35:04,043 INFO MainThread:1458 [wandb_init.py:init():1010] run started, returning control to user process +2026-07-24 04:08:45,417 INFO wandb-AsyncioManager-main:1458 [service_client.py:_forward_responses():122] Reached EOF. +2026-07-24 04:08:45,417 INFO wandb-AsyncioManager-main:1458 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles. +2026-07-24 04:08:45,418 ERROR wandb-AsyncioManager-main:1458 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop + await asyncio_compat.race( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race + async with open_task_group(race=True) as tg: + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__ + await anext(self.gen) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group + await task_group._wait_all(race=race, timeout=exit_timeout) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all + raise exc + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all + result = await handle.wait_async(timeout=None) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async + response = await self._handle.wait_async(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async + raise HandleAbandonedError() +wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError +2026-07-24 04:08:45,582 ERROR wandb-AsyncioManager-main:1458 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop + should_stop = await self._check_should_stop() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop + handle = await self._interface.deliver_async(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async + handle = await self._client.deliver(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver + handle = self._mailbox.require_response(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response + raise MailboxClosedError() +wandb.sdk.mailbox.mailbox.MailboxClosedError diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/config.yaml b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6377905a3e0617753a3d6a9edf118b9f31575b34 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/config.yaml @@ -0,0 +1,334 @@ +_wandb: + value: + cli_version: 0.27.2 + e: + xf8nsz01flzs1n2ps78z3eqf7rnh8bxr: + args: + - --config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model/train_config.json + - --resume=true + - --output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline + - --dataset.root=/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 + codePath: benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + codePathLocal: my_scripts/lerobot_train_yaml.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.8" + disk: + /: + total: "1888425144320" + used: "1137071325184" + executable: /home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python + git: + commit: a329728c2c9524d79b966c4a7a82ee56a4c2abf7 + remote: git@github.com:paragon7060/clvla.git + gpu: NVIDIA H200 + gpu_count: 2 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc + host: ext_minje-box + memory: + total: "2164176568320" + os: Linux-6.8.0-55-generic-x86_64-with-glibc2.39 + program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + python: CPython 3.12.13 + root: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline + startedAt: "2026-07-23T01:33:42.278848Z" + writerId: xf8nsz01flzs1n2ps78z3eqf7rnh8bxr + m: [] + python_version: 3.12.13 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 15 + - 16 + - 61 + - 62 + "4": 3.12.13 + "5": 0.27.2 + "6": 5.5.4 + "10": + - 21 + "12": 0.27.2 + "13": linux-x86_64 +batch_size: + value: 64 +checkpoint_path: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000 +cudnn_deterministic: + value: false +dataset: + value: + depth_output_unit: mm + episodes: null + eval_split: 0 + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + return_uint8: false + revision: null + root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 + streaming: false + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +env_eval_freq: + value: 20000 +eval: + value: + batch_size: 50 + n_episodes: 50 + recording: false + recording_private: false + recording_repo_id: null + use_async_envs: true +eval_steps: + value: 0 +job: + value: + detach: false + image: huggingface/lerobot-gpu:latest + tags: [] + target: null + timeout: 2d +job_name: + value: INSIGHT_6D_DiT16_Baseline +log_freq: + value: 200 +max_eval_samples: + value: 0 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 1 + lr: 0.0001 + type: adamw + weight_decay: 1e-05 +output_dir: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline +peft: + value: null +persistent_workers: + value: true +policy: + value: + action_decode_transform: null + action_expert_num_layers: 16 + balance_dataset_weights: true + balance_trajectory_weights: true + base_model_path: nvidia/GR00T-N1.7-3B + batch_size: 32 + chunk_size: 16 + dataloader_num_workers: 8 + dataset_paths: null + device: cuda + embodiment_tag: new_embodiment + image_size: + - 256 + - 256 + input_features: + observation.images.guide: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.right_shoulder: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.wrist: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.state: + shape: + - 16 + type: STATE + license: null + lora_alpha: 16 + lora_dropout: 0.1 + lora_full_model: false + lora_rank: 0 + max_action_dim: 132 + max_state_dim: 132 + max_steps: 10000 + model_params_fp32: true + n_action_steps: 16 + n_obs_steps: 1 + normalization_mapping: + ACTION: IDENTITY + STATE: IDENTITY + VISUAL: IDENTITY + num_inference_timesteps: null + optimizer_betas: + - 0.9 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-05 + output_dir: ./tmp/gr00t + output_features: + action: + shape: + - 10 + type: ACTION + pretrained_path: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model + pretrained_revision: null + private: null + push_to_hub: false + relative_exclude_joints: [] + repo_id: null + report_to: wandb + resume: false + rkd_angle_weight: 1 + rkd_distance_weight: 1 + rkd_enabled: false + rkd_fm_loss_weight: 1 + rkd_loss_weight: 0.1 + rkd_student_attention_hidden_dim: 512 + rkd_student_attention_output_dim: 512 + rkd_student_pooling: attention + rkd_vae_checkpoint: null + rtc_ramp_rate: null + save_steps: 1000 + tags: null + tokenizer_assets_repo: null + tune_diffusion_model: true + tune_llm: false + tune_projector: true + tune_top_llm_layers: 0 + tune_visual: false + tune_vlln: true + type: groot_rkd + use_amp: false + use_bf16: true + use_flash_attention: false + use_peft: false + use_relative_actions: false + video_backend: decord + warmup_ratio: 0.05 + warmup_steps: null +prefetch_factor: + value: 4 +resume: + value: true +reward_model: + value: null +sample_weighting: + value: null +save_checkpoint: + value: true +save_checkpoint_to_hub: + value: false +save_freq: + value: 20000 +scheduler: + value: + name: cosine + num_warmup_steps: 500 + type: diffuser +seed: + value: 42 +steps: + value: 60000 +tolerance_s: + value: 0.0001 +use_policy_training_preset: + value: true +wandb: + value: + add_tags: true + disable_artifact: true + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: svym8k0w diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/output.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/output.log index c09d7552432600047f81b1e4f54d8fc1831717bb..da7a83569ddcab1e9af877d794ba7a33d6790207 100644 --- a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/output.log +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/output.log @@ -45,4 +45,46 @@ Training: 3%|▎ | 800/30000 [20:32<12:38:00, 1.56s/step]INFO 2026-07 Training: 3%|▎ | 1000/30000 [26:16<21:46:15, 2.70s/step]INFO 2026-07-23 02:00:27 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.09 loss:0.042 grdn:0.279 lr:4.8e-05 updt_s:0.840 data_s:0.878 smp/s:37 mem_gb:29.76 Training: 4%|▍ | 1200/30000 [31:32<10:00:36, 1.25s/step]INFO 2026-07-23 02:05:43 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.14 loss:0.044 grdn:0.297 lr:4.8e-05 updt_s:0.810 data_s:0.768 smp/s:41 mem_gb:29.76 Training: 5%|▍ | 1400/30000 [36:05<10:58:06, 1.38s/step]INFO 2026-07-23 02:10:17 ot_train.py:646 step:31K smpl:2M ep:45K epch:9.20 loss:0.042 grdn:0.283 lr:4.7e-05 updt_s:0.744 data_s:0.623 smp/s:47 mem_gb:29.76 -Training: 5%|▍ | 1485/30000 [38:08<11:09:17, 1.41s/step] +Training: 5%|▌ | 1600/30000 [40:59<18:07:32, 2.30s/step]INFO 2026-07-23 02:15:10 ot_train.py:646 step:32K smpl:2M ep:46K epch:9.26 loss:0.042 grdn:0.294 lr:4.7e-05 updt_s:0.790 data_s:0.675 smp/s:44 mem_gb:29.75 +Training: 6%|▌ | 1800/30000 [45:49<10:55:53, 1.40s/step]INFO 2026-07-23 02:20:00 ot_train.py:646 step:32K smpl:2M ep:46K epch:9.32 loss:0.040 grdn:0.283 lr:4.6e-05 updt_s:0.779 data_s:0.671 smp/s:44 mem_gb:29.75 +Training: 7%|▋ | 2000/30000 [50:37<10:47:26, 1.39s/step]INFO 2026-07-23 02:24:48 ot_train.py:646 step:32K smpl:2M ep:46K epch:9.38 loss:0.039 grdn:0.285 lr:4.6e-05 updt_s:0.779 data_s:0.658 smp/s:45 mem_gb:29.76 +Training: 7%|▋ | 2169/30000 [54:38<10:42:52, 1.39s/step]Traceback (most recent call last): + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1453, in + config_path = Path(config_value).expanduser() + ^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1449, in main + config_value = _find_cli_value(args, "--config_path") + ^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 813, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 615, in train + wandb_logger.log_dict(wandb_log_dict, step) + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 196, in update_policy + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/accelerate/optimizer.py", line 179, in step + self.optimizer.step(closure) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/lr_scheduler.py", line 166, in wrapper + return func.__get__(opt, opt.__class__)(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/optimizer.py", line 533, in wrapper + out = func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/optimizer.py", line 81, in _use_grad + ret = func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/adam.py", line 248, in step + adam( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/optimizer.py", line 151, in maybe_fallback + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/adam.py", line 971, in adam + func( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/adam.py", line 773, in _multi_tensor_adam + 1 - beta1 ** _get_value(step) for step in device_state_steps + ^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/torch/optim/optimizer.py", line 96, in _get_value + return x.item() if isinstance(x, torch.Tensor) else x + ^^^^^^^^ +KeyboardInterrupt diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/wandb-summary.json b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..bb1ab49a6d1772b154c5712eb50809ef33803bd5 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/episodes":53461.55783611993,"_timestamp":1.784740277496143e+09,"train/steps":37000,"_wandb":{"runtime":58892},"train/samples":2368000,"train/loss":0.030536407604813576,"train/lr":3.280139452204795e-05,"train/epochs":10.844129378523313,"train/gpu_mem_gb":29.777451770305632,"train/samples_per_s":50.02997034996543,"train/dataloading_s":0.5564608199794747,"_step":37000,"train/update_s":0.7227723986755155,"train/grad_norm":0.2982970765978098,"_runtime":58892} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-core.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-core.log index faf95803a0afb0e85aff2c724e0b044d3d6270a3..4076840ca668c5be48ad329b3b7674c3db4af3e5 100644 --- a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-core.log +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-core.log @@ -26,3 +26,39 @@ {"time":"2026-07-23T01:33:48.72926561Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"0uuooyt4of4j"} {"time":"2026-07-23T01:33:48.824026381Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"v9b9occvkv0z"} {"time":"2026-07-23T01:33:48.928044163Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3mgxscqyzkyn"} +{"time":"2026-07-23T02:28:50.768484688Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768650257Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768660338Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768702111Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768725726Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768737691Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:28:50.768728092Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:28:50.768751955Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768745569Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768805358Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768815828Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814707-815594-198914184/socket","Net":"unix"}} +{"time":"2026-07-23T02:28:50.76883043Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814713-815525-1635480438/socket","Net":"unix"}} +{"time":"2026-07-23T02:28:50.770084447Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:28:50.770158191Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:28:50.770173329Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:28:50.770146502Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:28:50.770246375Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814719-815551-3138903381/socket","Net":"unix"}} +{"time":"2026-07-23T02:28:50.770382899Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773003008Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773054953Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773111864Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773106808Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773073579Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:28:50.773200905Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814701-815561-915654594/socket","Net":"unix"}} +{"time":"2026-07-23T02:28:52.626908382Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:28:52.626929775Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:28:52.626940482Z","level":"INFO","msg":"server is closed"} +{"time":"2026-07-23T02:28:52.662753611Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:28:52.662779647Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:28:52.662788745Z","level":"INFO","msg":"server is closed"} +{"time":"2026-07-23T02:28:52.68387658Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:28:52.683943364Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:28:52.683953617Z","level":"INFO","msg":"server is closed"} +{"time":"2026-07-23T02:29:02.470789399Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:29:02.470826347Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:29:02.470833873Z","level":"INFO","msg":"server is closed"} diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-internal.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-internal.log index 18f8978101154a7f6bfd422373e40feb74fa7373..6f806034e15ddc2115b60de1bbdbd6645ac0caa9 100644 --- a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-internal.log +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug-internal.log @@ -322,3 +322,145 @@ {"time":"2026-07-23T02:12:00.967408237Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} {"time":"2026-07-23T02:12:15.571990351Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":306,"events_lines":2,"console_offset":47,"console_lines":1} {"time":"2026-07-23T02:12:15.99962508Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:12:30.572077763Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":308,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:12:30.919043368Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:12:45.571767312Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":310,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:12:46.019504323Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:13:00.57253239Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":312,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:13:01.087265906Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:13:15.571983877Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":314,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:13:15.974384985Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:13:30.572251144Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":316,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:13:30.977173841Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:13:45.57221059Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":318,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:13:46.241316548Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:14:00.572670577Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":320,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:14:01.023306593Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:14:15.57176854Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":322,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:14:16.132358342Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:14:30.57242089Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":324,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:14:31.099023189Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:14:45.57165267Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":326,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:14:46.083851981Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:15:00.571775386Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":328,"events_lines":2,"console_offset":47,"console_lines":1} +{"time":"2026-07-23T02:15:01.180994763Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:15:10.34397278Z","level":"WARN","msg":"handler: ignoring partial history record","step":31600,"current":37001} +{"time":"2026-07-23T02:15:15.572476294Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":330,"events_lines":2,"console_offset":47,"console_lines":2} +{"time":"2026-07-23T02:15:16.071355954Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:15:30.574107589Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":332,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:15:32.072341973Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:15:45.572433513Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":334,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:15:46.035308642Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:16:00.572615706Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":336,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:16:01.045603345Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:16:15.572099645Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":338,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:16:16.033118322Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:16:30.571789981Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":340,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:16:30.973875432Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:16:45.572039256Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":342,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:16:50.678188665Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:17:00.572648582Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":344,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:17:01.118492107Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:17:15.571969771Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":346,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:17:15.987739434Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:17:30.571939605Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":348,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:17:31.006737517Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:17:45.571815157Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":350,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:17:46.065856811Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:18:00.572429836Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":352,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:18:05.316116771Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:18:15.571696271Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":354,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:18:16.048072889Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:18:30.572417546Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":356,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:18:31.020240289Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:18:45.572300055Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":358,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:18:46.018867761Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:19:00.572596087Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":360,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:19:01.133577158Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:19:15.572022604Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":362,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:19:16.123757993Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:19:30.57193837Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":364,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:19:31.144756012Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:19:45.572191385Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":366,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:19:46.029360586Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:20:00.572190728Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":368,"events_lines":2,"console_offset":48,"console_lines":1} +{"time":"2026-07-23T02:20:00.624607888Z","level":"WARN","msg":"handler: ignoring partial history record","step":31800,"current":37001} +{"time":"2026-07-23T02:20:01.06336733Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:20:15.571825472Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":370,"events_lines":2,"console_offset":48,"console_lines":2} +{"time":"2026-07-23T02:20:16.050034803Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:20:30.571866542Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":372,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:20:31.077058719Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:20:45.572297048Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":374,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:20:46.099056398Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:21:00.572442729Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":376,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:21:01.039290608Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:21:15.572106859Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":378,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:21:16.119198366Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:21:30.574065633Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":380,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:21:31.057563939Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:21:45.571889035Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":382,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:21:45.991763188Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:22:00.572347747Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":384,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:22:01.11877397Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:22:15.571676227Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":386,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:22:16.195466356Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:22:30.572561938Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":388,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:22:31.156203527Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:22:45.571953289Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":390,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:22:46.096453182Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:23:00.572406554Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":392,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:23:01.15280111Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:23:15.572476772Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":394,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:23:15.997661649Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:23:30.571675416Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":396,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:23:31.042153827Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:23:45.571842581Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":398,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:23:46.013948114Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:24:00.572618553Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":400,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:24:01.112804396Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:24:15.571653775Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":402,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:24:19.116676651Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:24:30.571793404Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":404,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:24:31.111814247Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:24:45.572614744Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":406,"events_lines":2,"console_offset":49,"console_lines":1} +{"time":"2026-07-23T02:24:46.108966733Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:24:48.481542966Z","level":"WARN","msg":"handler: ignoring partial history record","step":32000,"current":37001} +{"time":"2026-07-23T02:25:00.572449255Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":408,"events_lines":2,"console_offset":49,"console_lines":2} +{"time":"2026-07-23T02:25:01.074339199Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:25:15.572461247Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":410,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:25:19.46720351Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:25:30.572713746Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":412,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:25:31.036352395Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:25:45.572202664Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":414,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:25:46.095330102Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:26:00.572057444Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":416,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:26:00.997437674Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:26:15.572572229Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":418,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:26:15.964506584Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:26:30.572311377Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":420,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:26:34.223228815Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:26:45.571673127Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":422,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:26:45.981568849Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:27:00.572075585Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":424,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:27:05.612198431Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:27:15.571939538Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":426,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:27:16.068326991Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:27:30.572025149Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":428,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:27:36.185985361Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:27:45.572361632Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":430,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:27:45.965826186Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:28:00.572268613Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":432,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:28:01.061533852Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:28:15.572253679Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":434,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:28:19.240182832Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:28:30.57631386Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":436,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:28:34.507927646Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:28:45.571968022Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":438,"events_lines":2,"console_offset":50,"console_lines":1} +{"time":"2026-07-23T02:28:45.957846418Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:28:52.099263267Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-07-23T02:28:52.099519179Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":50,"console_lines":40,"uploaded_len":3,"complete":true,"exit_code":255} +{"time":"2026-07-23T02:28:52.662183615Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-23T02:28:52.662368771Z","level":"INFO","msg":"stream: finishing up"} +{"time":"2026-07-23T02:28:52.662380976Z","level":"INFO","msg":"handler: closed"} +{"time":"2026-07-23T02:28:52.662460976Z","level":"INFO","msg":"sender: closed"} +{"time":"2026-07-23T02:28:52.662465356Z","level":"INFO","msg":"stream: all finished"} diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug.log index 26b2cff6a251f4726b8b71cb7ca17acf07def143..063a4472179ffd3148f678a8c5052fc0e0350e4e 100644 --- a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug.log +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_013342-svym8k0w/logs/debug.log @@ -18,3 +18,29 @@ config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2' 2026-07-23 01:33:43,720 INFO MainThread:814713 [wandb_run.py:_redirect():2442] Wrapping output streams. 2026-07-23 01:33:43,720 INFO MainThread:814713 [wandb_run.py:_redirect():2465] Redirects installed. 2026-07-23 01:33:43,722 INFO MainThread:814713 [wandb_init.py:init():1010] run started, returning control to user process +2026-07-23 02:28:50,768 INFO wandb-AsyncioManager-main:814713 [service_client.py:_forward_responses():122] Reached EOF. +2026-07-23 02:28:50,768 INFO wandb-AsyncioManager-main:814713 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles. +2026-07-23 02:28:50,769 ERROR wandb-AsyncioManager-main:814713 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop + await asyncio_compat.race( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race + async with open_task_group(race=True) as tg: + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__ + await anext(self.gen) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group + await task_group._wait_all(race=race, timeout=exit_timeout) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all + raise exc + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all + result = await handle.wait_async(timeout=None) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async + response = await self._handle.wait_async(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async + raise HandleAbandonedError() +wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/files/wandb-summary.json b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1d476fc88692f959c7a899096787abbc21a55dbc --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":0,"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-core.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..84a4a3b43024c0846e82666c78b3142412c71005 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-core.log @@ -0,0 +1,30 @@ +{"time":"2026-07-23T02:29:37.218272849Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpbi3491l3/port-2499418.txt","pid":2499418,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-07-23T02:29:37.218902899Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2499418} +{"time":"2026-07-23T02:29:37.218864854Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2499418-2500321-1544228546/socket","Net":"unix"}} +{"time":"2026-07-23T02:29:37.402297125Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-07-23T02:29:37.409066123Z","level":"INFO","msg":"handleInformInit: received","streamId":"eb0sb786","id":"1(@)"} +{"time":"2026-07-23T02:29:37.868916316Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"eb0sb786","id":"1(@)"} +{"time":"2026-07-23T02:29:37.870710243Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpewnffvrr/port-2499551.txt","pid":2499551,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-07-23T02:29:37.871145347Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2499551-2500408-2757815951/socket","Net":"unix"}} +{"time":"2026-07-23T02:29:37.871202129Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2499551} +{"time":"2026-07-23T02:29:38.054483267Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-07-23T02:29:38.064116376Z","level":"INFO","msg":"handleInformInit: received","streamId":"cemf0rge","id":"1(@)"} +{"time":"2026-07-23T02:29:38.350897746Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:29:38.351103797Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:29:38.351133399Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:29:38.351150646Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:29:38.351193796Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:29:38.351238698Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2499418-2500321-1544228546/socket","Net":"unix"}} +{"time":"2026-07-23T02:29:38.352704212Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:29:38.352724918Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:29:38.352730597Z","level":"INFO","msg":"server is closed"} +{"time":"2026-07-23T02:29:38.445119298Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"cemf0rge","id":"1(@)"} +{"time":"2026-07-23T02:29:38.796444376Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:29:38.796598651Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:29:38.796618722Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:29:38.796624128Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:29:38.796668926Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:29:38.796751175Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2499551-2500408-2757815951/socket","Net":"unix"}} +{"time":"2026-07-23T02:29:38.797992882Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:29:38.798016256Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:29:38.798022072Z","level":"INFO","msg":"server is closed"} diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-internal.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f6ac2d09efe43e78faa124d3ad60bd7e09a33641 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2026-07-23T02:29:37.409217201Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-07-23T02:29:37.40945435Z","level":"INFO","msg":"stream: starting","core version":"0.27.2"} +{"time":"2026-07-23T02:29:37.868642986Z","level":"INFO","msg":"stream: created new stream","id":"eb0sb786"} +{"time":"2026-07-23T02:29:37.868729478Z","level":"INFO","msg":"handler: started"} +{"time":"2026-07-23T02:29:37.868907916Z","level":"INFO","msg":"stream: started"} +{"time":"2026-07-23T02:29:37.868962463Z","level":"INFO","msg":"writer: started","stream_id":"eb0sb786"} +{"time":"2026-07-23T02:29:37.868991239Z","level":"INFO","msg":"sender: started"} +{"time":"2026-07-23T02:29:38.34120831Z","level":"ERROR","msg":"runupserter: failed to init run","error":"no data but must resume"} +{"time":"2026-07-23T02:29:38.351341243Z","level":"ERROR","msg":"sender: uploadConfigFile: runhandle: run not yet initialized"} +{"time":"2026-07-23T02:29:38.351613477Z","level":"ERROR","msg":"runfiles: runhandle: run not yet initialized"} +{"time":"2026-07-23T02:29:38.351689734Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-07-23T02:29:38.351697183Z","level":"INFO","msg":"stream: finishing up"} +{"time":"2026-07-23T02:29:38.351703469Z","level":"INFO","msg":"handler: closed"} +{"time":"2026-07-23T02:29:38.351730985Z","level":"INFO","msg":"sender: closed"} +{"time":"2026-07-23T02:29:38.351733862Z","level":"INFO","msg":"stream: all finished"} diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4c9c5cc1c0214d6535395cb9ac2784c651dba600 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug.log @@ -0,0 +1,13 @@ +2026-07-23 02:29:37,186 INFO MainThread:2499418 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2 +2026-07-23 02:29:37,186 INFO MainThread:2499418 [wandb_setup.py:_flush():81] Configure stats pid to 2499418 +2026-07-23 02:29:37,186 INFO MainThread:2499418 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-07-23 02:29:37,186 INFO MainThread:2499418 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug.log +2026-07-23 02:29:37,186 INFO MainThread:2499418 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/logs/debug-internal.log +2026-07-23 02:29:37,186 INFO MainThread:2499418 [wandb_init.py:init():766] calling init triggers +2026-07-23 02:29:37,186 INFO MainThread:2499418 [wandb_init.py:init():771] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': , 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline', 'job_name': 'INSIGHT_6D_DiT16_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'eb0sb786', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000', '_wandb': {}} +2026-07-23 02:29:37,186 INFO MainThread:2499418 [wandb_init.py:init():814] starting backend +2026-07-23 02:29:37,402 INFO MainThread:2499418 [wandb_init.py:init():829] sending inform_init request +2026-07-23 02:29:37,869 INFO MainThread:2499418 [wandb_init.py:init():834] backend started and connected +2026-07-23 02:29:37,871 INFO MainThread:2499418 [wandb_init.py:init():904] updated telemetry +2026-07-23 02:29:37,875 INFO MainThread:2499418 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/run-eb0sb786.wandb b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/run-eb0sb786.wandb new file mode 100644 index 0000000000000000000000000000000000000000..fcb4bc6b6110d6bee21b9ab66a4d5d1af46ceafd Binary files /dev/null and b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_022937-eb0sb786/run-eb0sb786.wandb differ diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/files/output.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0f5631f8a11a7c9d0af86e9b611c937e17212232 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_023124-jeqgwsec/files/output.log @@ -0,0 +1,149 @@ +INFO 2026-07-23 02:31:25 db_utils.py:126 Logs will be synced with wandb. +INFO 2026-07-23 02:31:25 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/jeqgwsec +INFO 2026-07-23 02:31:25 ot_train.py:307 Creating dataset +INFO 2026-07-23 02:31:29 ot_train.py:341 Creating policy +INFO 2026-07-23 02:31:29 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model +INFO 2026-07-23 02:31:29 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 4284.76it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 807/807 [00:00<00:00, 1691.21it/s] +GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495 +Key | Status | | +---------------------------------------------------------------------+------------+--+- +action_head.model.transformer_blocks.{16...31}.ff.net.2.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.norm1.linear.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.ff.net.2.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_k.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.norm1.linear.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_q.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_v.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_q.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_k.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_v.weight | UNEXPECTED | | + +Notes: +- UNEXPECTED: can be ignored when loading from different task/architecture; not ok if you expect identical arch. +Loading weights from local directory +INFO 2026-07-23 02:31:44 ot_train.py:414 Creating optimizer and scheduler +INFO 2026-07-23 02:31:46 ot_train.py:446 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline +INFO 2026-07-23 02:31:46 ot_train.py:453 cfg.steps=60000 (60K) +INFO 2026-07-23 02:31:46 ot_train.py:454 dataset.num_frames=218367 (218K) +INFO 2026-07-23 02:31:46 ot_train.py:455 dataset.num_episodes=4930 +INFO 2026-07-23 02:31:46 ot_train.py:458 Effective batch size: 64 x 1 = 64 +INFO 2026-07-23 02:31:46 ot_train.py:459 num_learnable_params=1079180416 (1B) +INFO 2026-07-23 02:31:46 ot_train.py:460 num_total_params=2604255361 (3B) +INFO 2026-07-23 02:31:46 ot_train.py:501 Resuming data order at epoch 13, sample 42176 +Training: 0%| | 0/30000 [00:00, 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline', 'job_name': 'INSIGHT_6D_DiT16_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'jeqgwsec', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/030000', '_wandb': {}} +2026-07-23 02:31:24,520 INFO MainThread:2500948 [wandb_init.py:init():814] starting backend +2026-07-23 02:31:24,737 INFO MainThread:2500948 [wandb_init.py:init():829] sending inform_init request +2026-07-23 02:31:25,127 INFO MainThread:2500948 [wandb_init.py:init():834] backend started and connected +2026-07-23 02:31:25,130 INFO MainThread:2500948 [wandb_init.py:init():904] updated telemetry +2026-07-23 02:31:25,134 INFO MainThread:2500948 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-23 02:31:25,832 INFO MainThread:2500948 [wandb_init.py:init():972] starting run threads in backend +2026-07-23 02:31:25,900 INFO MainThread:2500948 [wandb_run.py:_console_start():2523] atexit reg +2026-07-23 02:31:25,900 INFO MainThread:2500948 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-23 02:31:25,900 INFO MainThread:2500948 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-23 02:31:25,900 INFO MainThread:2500948 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-23 02:31:25,902 INFO MainThread:2500948 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/config.yaml b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..470708ed6cc4d61445e076a1a4a8d7eba4996d95 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/config.yaml @@ -0,0 +1,329 @@ +_wandb: + value: + cli_version: 0.27.2 + e: + q52i38bd1zs4dxfa7y10j3r5h3dxa68o: + args: + - --config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model/train_config.json + - --resume=true + codePath: clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + codePathLocal: clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1888425144320" + used: "1327086579712" + executable: /home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python + gpu: NVIDIA H200 + gpu_count: 2 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc + host: ext_minje-box + memory: + total: "2164176568320" + os: Linux-6.8.0-55-generic-x86_64-with-glibc2.39 + program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + python: CPython 3.12.13 + root: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline + startedAt: "2026-07-23T16:35:01.611488Z" + writerId: q52i38bd1zs4dxfa7y10j3r5h3dxa68o + m: [] + python_version: 3.12.13 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 15 + - 16 + - 61 + - 62 + "4": 3.12.13 + "5": 0.27.2 + "6": 5.5.4 + "10": + - 21 + "12": 0.27.2 + "13": linux-x86_64 +batch_size: + value: 64 +checkpoint_path: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01 +cudnn_deterministic: + value: false +dataset: + value: + depth_output_unit: mm + episodes: null + eval_split: 0 + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + return_uint8: false + revision: null + root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 + streaming: false + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +env_eval_freq: + value: 20000 +eval: + value: + batch_size: 50 + n_episodes: 50 + recording: false + recording_private: false + recording_repo_id: null + use_async_envs: true +eval_steps: + value: 0 +job: + value: + detach: false + image: huggingface/lerobot-gpu:latest + tags: [] + target: null + timeout: 2d +job_name: + value: INSIGHT_6D_DiT16_Baseline +log_freq: + value: 200 +max_eval_samples: + value: 0 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 1 + lr: 0.0001 + type: adamw + weight_decay: 1e-05 +output_dir: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline +peft: + value: null +persistent_workers: + value: true +policy: + value: + action_decode_transform: null + action_expert_num_layers: 16 + balance_dataset_weights: true + balance_trajectory_weights: true + base_model_path: nvidia/GR00T-N1.7-3B + batch_size: 32 + chunk_size: 16 + dataloader_num_workers: 8 + dataset_paths: null + device: cuda + embodiment_tag: new_embodiment + image_size: + - 256 + - 256 + input_features: + observation.images.guide: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.right_shoulder: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.wrist: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.state: + shape: + - 16 + type: STATE + license: null + lora_alpha: 16 + lora_dropout: 0.1 + lora_full_model: false + lora_rank: 0 + max_action_dim: 132 + max_state_dim: 132 + max_steps: 10000 + model_params_fp32: true + n_action_steps: 16 + n_obs_steps: 1 + normalization_mapping: + ACTION: IDENTITY + STATE: IDENTITY + VISUAL: IDENTITY + num_inference_timesteps: null + optimizer_betas: + - 0.9 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-05 + output_dir: ./tmp/gr00t + output_features: + action: + shape: + - 10 + type: ACTION + pretrained_path: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model + pretrained_revision: null + private: null + push_to_hub: false + relative_exclude_joints: [] + repo_id: null + report_to: wandb + resume: false + rkd_angle_weight: 1 + rkd_distance_weight: 1 + rkd_enabled: false + rkd_fm_loss_weight: 1 + rkd_loss_weight: 0.1 + rkd_student_attention_hidden_dim: 512 + rkd_student_attention_output_dim: 512 + rkd_student_pooling: attention + rkd_vae_checkpoint: null + rtc_ramp_rate: null + save_steps: 1000 + tags: null + tokenizer_assets_repo: null + tune_diffusion_model: true + tune_llm: false + tune_projector: true + tune_top_llm_layers: 0 + tune_visual: false + tune_vlln: true + type: groot_rkd + use_amp: false + use_bf16: true + use_flash_attention: false + use_peft: false + use_relative_actions: false + video_backend: decord + warmup_ratio: 0.05 + warmup_steps: null +prefetch_factor: + value: 4 +resume: + value: true +reward_model: + value: null +sample_weighting: + value: null +save_checkpoint: + value: true +save_checkpoint_to_hub: + value: false +save_freq: + value: 20000 +scheduler: + value: + name: cosine + num_warmup_steps: 500 + type: diffuser +seed: + value: 42 +steps: + value: 60000 +tolerance_s: + value: 0.0001 +use_policy_training_preset: + value: true +wandb: + value: + add_tags: true + disable_artifact: true + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: jeqgwsec diff --git a/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/output.log b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f97c8e79620edb3d012e1110639c902f74ad5c48 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/Baseline/wandb/run-20260723_163501-jeqgwsec/files/output.log @@ -0,0 +1,96 @@ +INFO 2026-07-23 16:35:04 db_utils.py:126 Logs will be synced with wandb. +INFO 2026-07-23 16:35:04 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/jeqgwsec +INFO 2026-07-23 16:35:04 ot_train.py:310 Creating dataset +INFO 2026-07-23 16:35:06 ot_train.py:344 Creating policy +INFO 2026-07-23 16:35:06 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model +INFO 2026-07-23 16:35:06 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 332.64it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 807/807 [00:01<00:00, 424.83it/s] +GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495 +Key | Status | | +---------------------------------------------------------------------+------------+--+- +action_head.model.transformer_blocks.{16...31}.attn1.to_q.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.norm1.linear.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_v.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.ff.net.2.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_v.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_q.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.ff.net.2.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_k.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.ff.net.0.proj.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_out.0.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.attn1.to_k.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{16...31}.norm1.linear.bias | UNEXPECTED | | + +Notes: +- UNEXPECTED: can be ignored when loading from different task/architecture; not ok if you expect identical arch. +Loading weights from local directory +INFO 2026-07-23 16:35:28 ot_train.py:417 Creating optimizer and scheduler +INFO 2026-07-23 16:35:31 ot_train.py:449 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline +INFO 2026-07-23 16:35:31 ot_train.py:456 cfg.steps=60000 (60K) +INFO 2026-07-23 16:35:31 ot_train.py:457 dataset.num_frames=218367 (218K) +INFO 2026-07-23 16:35:31 ot_train.py:458 dataset.num_episodes=4930 +INFO 2026-07-23 16:35:31 ot_train.py:461 Effective batch size: 64 x 1 = 64 +INFO 2026-07-23 16:35:31 ot_train.py:462 num_learnable_params=1079180416 (1B) +INFO 2026-07-23 16:35:31 ot_train.py:463 num_total_params=2604255361 (3B) +INFO 2026-07-23 16:35:31 ot_train.py:504 Resuming data order at epoch 22, sample 22144 +Training: 0%| | 0/10000 [00:00, 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': None, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 16, 'rkd_enabled': False, 'rkd_vae_checkpoint': None, 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 0.1, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 1.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline', 'job_name': 'INSIGHT_6D_DiT16_Baseline', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'jeqgwsec', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/Baseline/checkpoints/rolling_01', '_wandb': {}} +2026-07-23 16:35:01,612 INFO MainThread:1458 [wandb_init.py:init():814] starting backend +2026-07-23 16:35:01,833 INFO MainThread:1458 [wandb_init.py:init():829] sending inform_init request +2026-07-23 16:35:02,308 INFO MainThread:1458 [wandb_init.py:init():834] backend started and connected +2026-07-23 16:35:02,310 INFO MainThread:1458 [wandb_init.py:init():904] updated telemetry +2026-07-23 16:35:02,310 INFO MainThread:1458 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-23 16:35:03,987 INFO MainThread:1458 [wandb_init.py:init():967] run resumed +2026-07-23 16:35:03,989 INFO MainThread:1458 [wandb_init.py:init():972] starting run threads in backend +2026-07-23 16:35:04,041 INFO MainThread:1458 [wandb_run.py:_console_start():2523] atexit reg +2026-07-23 16:35:04,041 INFO MainThread:1458 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-23 16:35:04,041 INFO MainThread:1458 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-23 16:35:04,041 INFO MainThread:1458 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-23 16:35:04,043 INFO MainThread:1458 [wandb_init.py:init():1010] run started, returning control to user process +2026-07-24 04:08:45,417 INFO wandb-AsyncioManager-main:1458 [service_client.py:_forward_responses():122] Reached EOF. +2026-07-24 04:08:45,417 INFO wandb-AsyncioManager-main:1458 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles. +2026-07-24 04:08:45,418 ERROR wandb-AsyncioManager-main:1458 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop + await asyncio_compat.race( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race + async with open_task_group(race=True) as tg: + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__ + await anext(self.gen) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group + await task_group._wait_all(race=race, timeout=exit_timeout) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all + raise exc + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all + result = await handle.wait_async(timeout=None) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async + response = await self._handle.wait_async(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async + raise HandleAbandonedError() +wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError +2026-07-24 04:08:45,582 ERROR wandb-AsyncioManager-main:1458 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop + should_stop = await self._check_should_stop() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop + handle = await self._interface.deliver_async(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async + handle = await self._client.deliver(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver + handle = self._mailbox.require_response(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response + raise MailboxClosedError() +wandb.sdk.mailbox.mailbox.MailboxClosedError diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/action_space_manifest.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/action_space_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..d3561af110f523f790a58730abbd57a87db28b05 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/action_space_manifest.json @@ -0,0 +1,144 @@ +{ + "schema_version": 2, + "action_space": "ee_abs_rot6d", + "action_views": [ + "ee_abs_rot6d" + ], + "source_feature": "action.ee_abs_rot6d", + "canonical_feature": "action", + "dtype": "float32", + "shape": [ + 10 + ], + "names": [ + "x", + "y", + "z", + "r_col0_x", + "r_col0_y", + "r_col0_z", + "r_col1_x", + "r_col1_y", + "r_col1_z", + "gripper" + ], + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "dataset_total_frames": 218367, + "dataset_total_episodes": 4930, + "dataset_fps": 10, + "stats_sha256": "71698c7337070aa8267df5aea7ec08bc00d3925510f2649bc20c9201eeb669e4", + "stats": { + "min": [ + -0.21784618496894836, + -0.4474811851978302, + 0.1702737659215927, + -0.9999986290931702, + -0.9999991655349731, + -0.9999999403953552, + -0.9999990463256836, + -0.9999998211860657, + -0.9999988079071045, + 0.0 + ], + "max": [ + 0.6283698678016663, + 0.4420371949672699, + 1.051027774810791, + 0.9999999403953552, + 0.9999995231628418, + 1.0, + 0.9999995231628418, + 1.0, + 0.9999996423721313, + 0.03999999910593033 + ], + "mean": [ + 0.4040228037735655, + -0.05247618696989652, + 0.6993459771218912, + 0.07011111991957117, + -0.03520558904460197, + 0.14470242846953205, + 0.05491166606739672, + -0.19073058893378805, + -0.06988117661871228, + 0.021167300266524632 + ], + "std": [ + 0.11861927913357073, + 0.11318943557281885, + 0.17849160646987516, + 0.4812528791886511, + 0.7487196258736938, + 0.4251126804030219, + 0.47797229807769137, + 0.6267695346407992, + 0.5782954306815554, + 0.01996590431050817 + ], + "count": [ + 218367 + ], + "q01": [ + 0.23988881827972064, + -0.10387461883034811, + 0.6297243923476871, + -0.43622841955353464, + -0.6885486527617446, + -0.14103555388039019, + -0.4327210063987595, + -0.9304790915716145, + -0.3465462258490417, + -1.000000013351432e-10 + ], + "q10": [ + 0.3042306436755216, + -0.09609991620622624, + 0.6394063072781145, + -0.3351306352998669, + -0.5751963015002924, + -0.08490621283307866, + -0.33849701676224275, + -0.8340557712529717, + -0.29760290790884514, + -1.000000013351432e-10 + ], + "q50": [ + 0.4160225263528291, + -0.06267459152989589, + 0.6811777491060801, + 0.0543536902230377, + -0.14012251058819217, + 0.04772394280004727, + 0.03838119138716219, + -0.14865169094195635, + -0.2073069401172425, + 0.020772594934870346 + ], + "q90": [ + 0.46641008755679864, + 0.020233143101890064, + 0.8095135405259374, + 0.48172401592474906, + 0.7398737043070319, + 0.5970841215636613, + 0.4674752875999555, + 0.36281754777672676, + 0.5033361883834496, + 0.039998362213352394 + ], + "q99": [ + 0.47320723171849244, + 0.04114262476135705, + 0.8467967259270389, + 0.5867781409980988, + 0.8529584859257228, + 0.7953936054341789, + 0.5768269858732034, + 0.49353349323389695, + 0.7168914440776504, + 0.03999983541667277 + ] + } +} diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/config.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..af23ddb29679168ec1128aa11ce385b740f4d641 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/config.json @@ -0,0 +1,118 @@ +{ + "type": "groot_rkd", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 16 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.right_shoulder": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.guide": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 10 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model", + "pretrained_revision": null, + "chunk_size": 16, + "n_action_steps": 16, + "max_state_dim": 132, + "max_action_dim": 132, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "IDENTITY", + "ACTION": "IDENTITY" + }, + "base_model_path": "nvidia/GR00T-N1.7-3B", + "action_decode_transform": null, + "embodiment_tag": "new_embodiment", + "tune_llm": false, + "tune_visual": false, + "tune_projector": true, + "tune_diffusion_model": true, + "tune_vlln": true, + "tune_top_llm_layers": 0, + "num_inference_timesteps": null, + "rtc_ramp_rate": null, + "use_flash_attention": false, + "use_relative_actions": false, + "relative_exclude_joints": [], + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.9, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-05, + "warmup_steps": 3000, + "warmup_ratio": 0.05, + "use_bf16": true, + "model_params_fp32": true, + "image_size": [ + 256, + 256 + ], + "tokenizer_assets_repo": null, + "lora_rank": 0, + "lora_alpha": 16, + "lora_dropout": 0.1, + "lora_full_model": false, + "video_backend": "decord", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "dataset_paths": null, + "output_dir": "./tmp/gr00t", + "save_steps": 1000, + "max_steps": 10000, + "batch_size": 32, + "dataloader_num_workers": 8, + "report_to": "wandb", + "resume": false, + "action_expert_num_layers": 16, + "rkd_enabled": true, + "rkd_vae_checkpoint": "/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11", + "rkd_student_pooling": "attention", + "rkd_student_attention_hidden_dim": 512, + "rkd_student_attention_output_dim": 512, + "rkd_fm_loss_weight": 1.0, + "rkd_loss_weight": 1.0, + "rkd_distance_weight": 1.0, + "rkd_angle_weight": 2.0 +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/phase_schedule.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/phase_schedule.json new file mode 100644 index 0000000000000000000000000000000000000000..f7e4d4be997c76539eafba9602aecab4d33e137e --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/phase_schedule.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "step_interval": "[start_step, until_step)", + "total_steps": 60000, + "optimizer_lifecycle": "single_continuous_optimizer", + "scheduler_lifecycle": "single_continuous_scheduler", + "phases": [ + { + "index": 0, + "name": "rscl_rkd_cosine_decay_60K", + "start_step": 0, + "until_step": 60000, + "policy": { + "rkd_enabled": true, + "rkd_fm_loss_weight": 1.0, + "rkd_loss_weight": 1.0, + "rkd_loss_weight_end": 0.0, + "rkd_loss_weight_schedule": "cosine", + "rkd_distance_weight": 1.0, + "rkd_angle_weight": 2.0 + }, + "trainable": { + "tune_llm": false, + "tune_visual": false, + "tune_top_llm_layers": 0, + "tune_projector": true, + "tune_diffusion_model": true, + "tune_vlln": true + } + } + ] +} diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/policy_postprocessor.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..4bb07c3b3b6c5e094251664271eb9f04152fe46f --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/policy_postprocessor.json @@ -0,0 +1,23 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "groot_action_unpack_unnormalize_v2", + "config": { + "env_action_dim": 10, + "normalize_min_max": true, + "clip_normalized_action": true, + "libero_gripper_action": false, + "libero_gripper_binarize": true + }, + "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/policy_preprocessor.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..26ee04595ddf08b7d0f7a7e0d9fb0b08d15f503a --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/policy_preprocessor.json @@ -0,0 +1,82 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "rkd_preserve_raw_action", + "config": {} + }, + { + "registry_name": "groot_n1_7_pack_inputs_v1", + "config": { + "state_horizon": 1, + "action_horizon": 16, + "valid_action_horizon": 16, + "video_horizon": null, + "max_state_dim": 132, + "max_action_dim": 132, + "language_key": "task", + "formalize_language": true, + "embodiment_tag": "new_embodiment", + "embodiment_mapping": { + "oxe_droid_relative_eef_relative_joint": 24, + "xdof_relative_eef_relative_joint": 27, + "xdof_relative_eef_relative_joint_subtask": 27, + "real_g1_relative_eef_relative_joints": 25, + "real_r1_pro_sharpa_relative_eef": 26, + "real_r1_pro_sharpa_relative_eef_human": 26, + "real_r1_pro_sharpa_relative_eef_maxinsights": 26, + "real_r1_pro_sharpa_relative_eef_mecka": 26, + "unitree_g1_full_body_with_waist_height_nav_cmd": 25, + "simpler_env_google": 0, + "simpler_env_widowx": 1, + "libero_sim": 2, + "new_embodiment": 10 + }, + "normalize_min_max": true, + "state_dropout_prob": 0.0, + "clip_outliers": true, + "use_percentiles": false, + "video_modality_keys": null, + "raw_stats": null, + "modality_config": null + }, + "state_file": "policy_preprocessor_step_3_groot_n1_7_pack_inputs_v1.safetensors" + }, + { + "registry_name": "groot_n1_7_vlm_encode_v1", + "config": { + "model_name": "nvidia/Cosmos-Reason2-2B", + "image_crop_size": [ + 230, + 230 + ], + "image_target_size": [ + 256, + 256 + ], + "shortest_image_edge": null, + "crop_fraction": null, + "use_albumentations": false, + "letter_box_transform": false, + "device": "cuda" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/prompt_manifest.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/prompt_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..21e3a43c712e5842c4616c53e544703bf52fb7ab --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/prompt_manifest.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "prompt_set": "detailed", + "source_file": "task_prompts.json", + "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc", + "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018", + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "task_count": 13, + "task_codes": [ + "5f", + "5g", + "1ext", + "3b", + "5b", + "3a", + "3c", + "5h", + "5d", + "3d", + "5e", + "5a", + "5c" + ], + "prompts": { + "5f": "Close the bottle in clockwise direction.", + "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.", + "1ext": "Find the arrow guide and open the indicated drawer.", + "3b": "Open the door, rotate counter-clockwise and push.", + "5b": "Open the bottle in counter-clockwise direction.", + "3a": "Open the door, rotate clockwise and push.", + "3c": "Open the door, rotate clockwise and pull.", + "5h": "Close the bottle in counter-clockwise direction.", + "5d": "Close the bottle in clockwise direction.", + "3d": "Open the door, rotate counter-clockwise and pull.", + "5e": "Close the bottle in counter-clockwise direction.", + "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.", + "5c": "Open the bottle in clockwise direction." + } +} diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/train_config.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d1a3d98551d44a96165152003b49b27f07082894 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/train_config.json @@ -0,0 +1,269 @@ +{ + "dataset": { + "repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "root": "/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec", + "return_uint8": false, + "depth_output_unit": "mm", + "streaming": false, + "eval_split": 0.0 + }, + "env": null, + "policy": { + "type": "groot_rkd", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 16 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.right_shoulder": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.guide": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 10 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": false, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model", + "pretrained_revision": null, + "chunk_size": 16, + "n_action_steps": 16, + "max_state_dim": 132, + "max_action_dim": 132, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "IDENTITY", + "ACTION": "IDENTITY" + }, + "base_model_path": "nvidia/GR00T-N1.7-3B", + "action_decode_transform": null, + "embodiment_tag": "new_embodiment", + "tune_llm": false, + "tune_visual": false, + "tune_projector": true, + "tune_diffusion_model": true, + "tune_vlln": true, + "tune_top_llm_layers": 0, + "num_inference_timesteps": null, + "rtc_ramp_rate": null, + "use_flash_attention": false, + "use_relative_actions": false, + "relative_exclude_joints": [], + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.9, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-05, + "warmup_steps": 3000, + "warmup_ratio": 0.05, + "use_bf16": true, + "model_params_fp32": true, + "image_size": [ + 256, + 256 + ], + "tokenizer_assets_repo": null, + "lora_rank": 0, + "lora_alpha": 16, + "lora_dropout": 0.1, + "lora_full_model": false, + "video_backend": "decord", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "dataset_paths": null, + "output_dir": "./tmp/gr00t", + "save_steps": 1000, + "max_steps": 10000, + "batch_size": 32, + "dataloader_num_workers": 8, + "report_to": "wandb", + "resume": false, + "action_expert_num_layers": 16, + "rkd_enabled": true, + "rkd_vae_checkpoint": "/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11", + "rkd_student_pooling": "attention", + "rkd_student_attention_hidden_dim": 512, + "rkd_student_attention_output_dim": 512, + "rkd_fm_loss_weight": 1.0, + "rkd_loss_weight": 1.0, + "rkd_distance_weight": 1.0, + "rkd_angle_weight": 2.0 + }, + "reward_model": null, + "output_dir": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K", + "job_name": "INSIGHT_6D_DiT16_RKD_TimewarpVAE_RSCLstyle_C60", + "resume": true, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 4, + "batch_size": 64, + "prefetch_factor": 4, + "persistent_workers": true, + "steps": 60000, + "env_eval_freq": 20000, + "log_freq": 200, + "eval_steps": 0, + "max_eval_samples": 0, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 0.0001, + "weight_decay": 1e-05, + "grad_clip_norm": 1.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 3000, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": true, + "recording": false, + "recording_repo_id": null, + "recording_private": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "rs1egwv8", + "mode": null, + "add_tags": true + }, + "peft": null, + "job": { + "target": null, + "image": "huggingface/lerobot-gpu:latest", + "timeout": "2d", + "detach": false, + "tags": [] + }, + "save_checkpoint_to_hub": false, + "sample_weighting": null, + "rename_map": {}, + "checkpoint_path": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01" +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/prompt_manifest.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/prompt_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..21e3a43c712e5842c4616c53e544703bf52fb7ab --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/prompt_manifest.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "prompt_set": "detailed", + "source_file": "task_prompts.json", + "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc", + "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018", + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "task_count": 13, + "task_codes": [ + "5f", + "5g", + "1ext", + "3b", + "5b", + "3a", + "3c", + "5h", + "5d", + "3d", + "5e", + "5a", + "5c" + ], + "prompts": { + "5f": "Close the bottle in clockwise direction.", + "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.", + "1ext": "Find the arrow guide and open the indicated drawer.", + "3b": "Open the door, rotate counter-clockwise and push.", + "5b": "Open the bottle in counter-clockwise direction.", + "3a": "Open the door, rotate clockwise and push.", + "3c": "Open the door, rotate clockwise and pull.", + "5h": "Close the bottle in counter-clockwise direction.", + "5d": "Close the bottle in clockwise direction.", + "3d": "Open the door, rotate counter-clockwise and pull.", + "5e": "Close the bottle in counter-clockwise direction.", + "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.", + "5c": "Open the bottle in clockwise direction." + } +} diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/training_state/scheduler_state.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3c3581af9b9c38660399de2fe9df9ef91e937c52 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_00/training_state/scheduler_state.json @@ -0,0 +1,18 @@ +{ + "base_lrs": [ + 0.0001, + 0.0001 + ], + "last_epoch": 55000, + "_step_count": 55001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.8865999845374793e-06, + 1.8865999845374793e-06 + ], + "lr_lambdas": [ + null, + null + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/policy_postprocessor.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..4bb07c3b3b6c5e094251664271eb9f04152fe46f --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/policy_postprocessor.json @@ -0,0 +1,23 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "groot_action_unpack_unnormalize_v2", + "config": { + "env_action_dim": 10, + "normalize_min_max": true, + "clip_normalized_action": true, + "libero_gripper_action": false, + "libero_gripper_binarize": true + }, + "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v2.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/policy_preprocessor.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..26ee04595ddf08b7d0f7a7e0d9fb0b08d15f503a --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/pretrained_model/policy_preprocessor.json @@ -0,0 +1,82 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "rkd_preserve_raw_action", + "config": {} + }, + { + "registry_name": "groot_n1_7_pack_inputs_v1", + "config": { + "state_horizon": 1, + "action_horizon": 16, + "valid_action_horizon": 16, + "video_horizon": null, + "max_state_dim": 132, + "max_action_dim": 132, + "language_key": "task", + "formalize_language": true, + "embodiment_tag": "new_embodiment", + "embodiment_mapping": { + "oxe_droid_relative_eef_relative_joint": 24, + "xdof_relative_eef_relative_joint": 27, + "xdof_relative_eef_relative_joint_subtask": 27, + "real_g1_relative_eef_relative_joints": 25, + "real_r1_pro_sharpa_relative_eef": 26, + "real_r1_pro_sharpa_relative_eef_human": 26, + "real_r1_pro_sharpa_relative_eef_maxinsights": 26, + "real_r1_pro_sharpa_relative_eef_mecka": 26, + "unitree_g1_full_body_with_waist_height_nav_cmd": 25, + "simpler_env_google": 0, + "simpler_env_widowx": 1, + "libero_sim": 2, + "new_embodiment": 10 + }, + "normalize_min_max": true, + "state_dropout_prob": 0.0, + "clip_outliers": true, + "use_percentiles": false, + "video_modality_keys": null, + "raw_stats": null, + "modality_config": null + }, + "state_file": "policy_preprocessor_step_3_groot_n1_7_pack_inputs_v1.safetensors" + }, + { + "registry_name": "groot_n1_7_vlm_encode_v1", + "config": { + "model_name": "nvidia/Cosmos-Reason2-2B", + "image_crop_size": [ + 230, + 230 + ], + "image_target_size": [ + 256, + 256 + ], + "shortest_image_edge": null, + "crop_fraction": null, + "use_albumentations": false, + "letter_box_transform": false, + "device": "cuda" + } + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/training_state/scheduler_state.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a8301608cef57d586c44a4fcc79905580b0d63bf --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/training_state/scheduler_state.json @@ -0,0 +1,18 @@ +{ + "base_lrs": [ + 0.0001, + 0.0001 + ], + "last_epoch": 60000, + "_step_count": 60001, + "_is_initial": false, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.0, + 0.0 + ], + "lr_lambdas": [ + null, + null + ] +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/training_state/training_step.json b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..0c795acfc7d3498b7e790af15ae93492dc3be1b6 --- /dev/null +++ b/Abs_6D/DiT_Layer/16/RSCLstyle_cosine_60K/checkpoints/rolling_01/training_state/training_step.json @@ -0,0 +1,5 @@ +{ + "step": 60000, + "num_processes": 1, + "batch_size": 64 +} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/8/Baseline/baseline_gpu0_rolling5k_20260723_022922.log b/Abs_6D/DiT_Layer/8/Baseline/baseline_gpu0_rolling5k_20260723_022922.log new file mode 100644 index 0000000000000000000000000000000000000000..ceadd921be95e18a12b076b6dd4aadff1fcaf10c --- /dev/null +++ b/Abs_6D/DiT_Layer/8/Baseline/baseline_gpu0_rolling5k_20260723_022922.log @@ -0,0 +1,198 @@ +INFO 2026-07-23 02:29:35 ot_train.py:281 {'batch_size': 64, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_6D_DiT8_Baseline', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'action_expert_num_layers': 8, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [10], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 1.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': False, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 0.1, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': None, + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': None}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 500, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'bmlpbci8'}} +Traceback (most recent call last): + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1613, in + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1609, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 822, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 285, in train + if not policy.is_trainable: + ^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/common/wandb_utils.py", line 99, in __init__ + wandb.init( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1528, in init + get_sentry().reraise(e) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/analytics/sentry.py", line 191, in reraise + raise exc.with_traceback(tb) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1511, in init + run = wi.init(run_settings, run_config, run_printer) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 961, in init + raise error +wandb.errors.errors.UsageError: You provided an invalid value for the `resume` argument. The value 'must' is not a valid option for resuming the run (bmlpbci8) that has not been initialized. Please check your inputs and try again with a valid run ID. If you are trying to start a new run, please omit the `resume` argument or use `resume='allow'`. diff --git a/Abs_6D/DiT_Layer/8/Baseline/baseline_gpu0_rolling5k_20260723_023110_newwandb.log b/Abs_6D/DiT_Layer/8/Baseline/baseline_gpu0_rolling5k_20260723_023110_newwandb.log new file mode 100644 index 0000000000000000000000000000000000000000..e1b5228e533f6ee203d7378b4d3bb2282a909719 --- /dev/null +++ b/Abs_6D/DiT_Layer/8/Baseline/baseline_gpu0_rolling5k_20260723_023110_newwandb.log @@ -0,0 +1,354 @@ +INFO 2026-07-23 02:31:22 ot_train.py:281 {'batch_size': 64, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_6D_DiT8_Baseline', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'action_expert_num_layers': 8, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [10], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 1.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': False, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 0.1, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': None, + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': None}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 500, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': '2vl3enlf'}} +INFO 2026-07-23 02:31:24 db_utils.py:126 Logs will be synced with wandb. +INFO 2026-07-23 02:31:24 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/2vl3enlf +INFO 2026-07-23 02:31:24 ot_train.py:307 Creating dataset +INFO 2026-07-23 02:31:26 ot_train.py:341 Creating policy +INFO 2026-07-23 02:31:26 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/Baseline/checkpoints/030000/pretrained_model +INFO 2026-07-23 02:31:26 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [10], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'x6da8gup'}} +INFO 2026-07-23 02:31:26 db_utils.py:126 Logs will be synced with wandb. +INFO 2026-07-23 02:31:26 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/x6da8gup +INFO 2026-07-23 02:31:26 ot_train.py:307 Creating dataset +INFO 2026-07-23 02:31:29 ot_train.py:341 Creating policy +INFO 2026-07-23 02:31:29 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-23 02:31:29 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00, 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 8, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_6D_DiT8_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'rd8tz18f', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000', '_wandb': {}} -2026-07-23 01:33:42,330 INFO MainThread:814707 [wandb_init.py:init():814] starting backend -2026-07-23 01:33:42,549 INFO MainThread:814707 [wandb_init.py:init():829] sending inform_init request -2026-07-23 01:33:42,941 INFO MainThread:814707 [wandb_init.py:init():834] backend started and connected -2026-07-23 01:33:42,943 INFO MainThread:814707 [wandb_init.py:init():904] updated telemetry -2026-07-23 01:33:42,947 INFO MainThread:814707 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout -2026-07-23 01:33:43,847 INFO MainThread:814707 [wandb_init.py:init():967] run resumed -2026-07-23 01:33:43,849 INFO MainThread:814707 [wandb_init.py:init():972] starting run threads in backend -2026-07-23 01:33:43,908 INFO MainThread:814707 [wandb_run.py:_console_start():2523] atexit reg -2026-07-23 01:33:43,908 INFO MainThread:814707 [wandb_run.py:_redirect():2373] redirect: wrap_raw -2026-07-23 01:33:43,908 INFO MainThread:814707 [wandb_run.py:_redirect():2442] Wrapping output streams. -2026-07-23 01:33:43,908 INFO MainThread:814707 [wandb_run.py:_redirect():2465] Redirects installed. -2026-07-23 01:33:43,910 INFO MainThread:814707 [wandb_init.py:init():1010] run started, returning control to user process +2026-07-23 16:35:01,612 INFO MainThread:1457 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2 +2026-07-23 16:35:01,612 INFO MainThread:1457 [wandb_setup.py:_flush():81] Configure stats pid to 1457 +2026-07-23 16:35:01,612 INFO MainThread:1457 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-07-23 16:35:01,612 INFO MainThread:1457 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_163501-x6da8gup/logs/debug.log +2026-07-23 16:35:01,612 INFO MainThread:1457 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_163501-x6da8gup/logs/debug-internal.log +2026-07-23 16:35:01,612 INFO MainThread:1457 [wandb_init.py:init():766] calling init triggers +2026-07-23 16:35:01,612 INFO MainThread:1457 [wandb_init.py:init():771] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': , 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 8, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_6D_DiT8_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'x6da8gup', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/rolling_00', '_wandb': {}} +2026-07-23 16:35:01,612 INFO MainThread:1457 [wandb_init.py:init():814] starting backend +2026-07-23 16:35:01,833 INFO MainThread:1457 [wandb_init.py:init():829] sending inform_init request +2026-07-23 16:35:02,307 INFO MainThread:1457 [wandb_init.py:init():834] backend started and connected +2026-07-23 16:35:02,308 INFO MainThread:1457 [wandb_init.py:init():904] updated telemetry +2026-07-23 16:35:02,309 INFO MainThread:1457 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-23 16:35:03,204 INFO MainThread:1457 [wandb_init.py:init():967] run resumed +2026-07-23 16:35:03,206 INFO MainThread:1457 [wandb_init.py:init():972] starting run threads in backend +2026-07-23 16:35:03,262 INFO MainThread:1457 [wandb_run.py:_console_start():2523] atexit reg +2026-07-23 16:35:03,262 INFO MainThread:1457 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-23 16:35:03,262 INFO MainThread:1457 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-23 16:35:03,262 INFO MainThread:1457 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-23 16:35:03,264 INFO MainThread:1457 [wandb_init.py:init():1010] run started, returning control to user process +2026-07-23 18:11:27,126 INFO wandb-AsyncioManager-main:1457 [service_client.py:_forward_responses():122] Reached EOF. +2026-07-23 18:11:27,126 INFO wandb-AsyncioManager-main:1457 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles. +2026-07-23 18:11:27,128 ERROR wandb-AsyncioManager-main:1457 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop + await asyncio_compat.race( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race + async with open_task_group(race=True) as tg: + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__ + await anext(self.gen) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group + await task_group._wait_all(race=race, timeout=exit_timeout) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all + raise exc + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all + result = await handle.wait_async(timeout=None) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async + response = await self._handle.wait_async(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async + raise HandleAbandonedError() +wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError +2026-07-23 18:11:34,104 ERROR wandb-AsyncioManager-main:1457 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop + should_stop = await self._check_should_stop() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop + handle = await self._interface.deliver_async(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async + handle = await self._client.deliver(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver + handle = self._mailbox.require_response(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response + raise MailboxClosedError() +wandb.sdk.mailbox.mailbox.MailboxClosedError diff --git a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/files/config.yaml b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5758ca5db99c0b8c5e0914ea55d0ed780426f95 --- /dev/null +++ b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/files/config.yaml @@ -0,0 +1,334 @@ +_wandb: + value: + cli_version: 0.27.2 + e: + fslf4w7acsnj7qn4s2ce6ar5k1rugjh7: + args: + - --config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model/train_config.json + - --resume=true + - --output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K + - --dataset.root=/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 + codePath: benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + codePathLocal: my_scripts/lerobot_train_yaml.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.8" + disk: + /: + total: "1888425144320" + used: "1137071325184" + executable: /home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python + git: + commit: a329728c2c9524d79b966c4a7a82ee56a4c2abf7 + remote: git@github.com:paragon7060/clvla.git + gpu: NVIDIA H200 + gpu_count: 2 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc + host: ext_minje-box + memory: + total: "2164176568320" + os: Linux-6.8.0-55-generic-x86_64-with-glibc2.39 + program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + python: CPython 3.12.13 + root: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K + startedAt: "2026-07-23T01:33:42.328417Z" + writerId: fslf4w7acsnj7qn4s2ce6ar5k1rugjh7 + m: [] + python_version: 3.12.13 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 15 + - 16 + - 61 + - 62 + "4": 3.12.13 + "5": 0.27.2 + "6": 5.5.4 + "10": + - 21 + "12": 0.27.2 + "13": linux-x86_64 +batch_size: + value: 64 +checkpoint_path: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000 +cudnn_deterministic: + value: false +dataset: + value: + depth_output_unit: mm + episodes: null + eval_split: 0 + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + return_uint8: false + revision: null + root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 + streaming: false + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +env_eval_freq: + value: 20000 +eval: + value: + batch_size: 50 + n_episodes: 50 + recording: false + recording_private: false + recording_repo_id: null + use_async_envs: true +eval_steps: + value: 0 +job: + value: + detach: false + image: huggingface/lerobot-gpu:latest + tags: [] + target: null + timeout: 2d +job_name: + value: INSIGHT_6D_DiT8_RKD_TimewarpVAE_RSCLstyle_C60 +log_freq: + value: 200 +max_eval_samples: + value: 0 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 1 + lr: 0.0001 + type: adamw + weight_decay: 1e-05 +output_dir: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K +peft: + value: null +persistent_workers: + value: true +policy: + value: + action_decode_transform: null + action_expert_num_layers: 8 + balance_dataset_weights: true + balance_trajectory_weights: true + base_model_path: nvidia/GR00T-N1.7-3B + batch_size: 32 + chunk_size: 16 + dataloader_num_workers: 8 + dataset_paths: null + device: cuda + embodiment_tag: new_embodiment + image_size: + - 256 + - 256 + input_features: + observation.images.guide: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.right_shoulder: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.wrist: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.state: + shape: + - 16 + type: STATE + license: null + lora_alpha: 16 + lora_dropout: 0.1 + lora_full_model: false + lora_rank: 0 + max_action_dim: 132 + max_state_dim: 132 + max_steps: 10000 + model_params_fp32: true + n_action_steps: 16 + n_obs_steps: 1 + normalization_mapping: + ACTION: IDENTITY + STATE: IDENTITY + VISUAL: IDENTITY + num_inference_timesteps: null + optimizer_betas: + - 0.9 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-05 + output_dir: ./tmp/gr00t + output_features: + action: + shape: + - 10 + type: ACTION + pretrained_path: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model + pretrained_revision: null + private: null + push_to_hub: false + relative_exclude_joints: [] + repo_id: null + report_to: wandb + resume: false + rkd_angle_weight: 2 + rkd_distance_weight: 1 + rkd_enabled: true + rkd_fm_loss_weight: 1 + rkd_loss_weight: 1 + rkd_student_attention_hidden_dim: 512 + rkd_student_attention_output_dim: 512 + rkd_student_pooling: attention + rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11 + rtc_ramp_rate: null + save_steps: 1000 + tags: null + tokenizer_assets_repo: null + tune_diffusion_model: true + tune_llm: false + tune_projector: true + tune_top_llm_layers: 0 + tune_visual: false + tune_vlln: true + type: groot_rkd + use_amp: false + use_bf16: true + use_flash_attention: false + use_peft: false + use_relative_actions: false + video_backend: decord + warmup_ratio: 0.05 + warmup_steps: 3000 +prefetch_factor: + value: 4 +resume: + value: true +reward_model: + value: null +sample_weighting: + value: null +save_checkpoint: + value: true +save_checkpoint_to_hub: + value: false +save_freq: + value: 20000 +scheduler: + value: + name: cosine + num_warmup_steps: 3000 + type: diffuser +seed: + value: 42 +steps: + value: 60000 +tolerance_s: + value: 0.0001 +use_policy_training_preset: + value: true +wandb: + value: + add_tags: true + disable_artifact: true + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: rd8tz18f diff --git a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/logs/debug-core.log b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/logs/debug-core.log index faf95803a0afb0e85aff2c724e0b044d3d6270a3..4076840ca668c5be48ad329b3b7674c3db4af3e5 100644 --- a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/logs/debug-core.log +++ b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/logs/debug-core.log @@ -26,3 +26,39 @@ {"time":"2026-07-23T01:33:48.72926561Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"0uuooyt4of4j"} {"time":"2026-07-23T01:33:48.824026381Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"v9b9occvkv0z"} {"time":"2026-07-23T01:33:48.928044163Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"3mgxscqyzkyn"} +{"time":"2026-07-23T02:28:50.768484688Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768650257Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768660338Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768702111Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768725726Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768737691Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:28:50.768728092Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:28:50.768751955Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768745569Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768805358Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:28:50.768815828Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814707-815594-198914184/socket","Net":"unix"}} +{"time":"2026-07-23T02:28:50.76883043Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814713-815525-1635480438/socket","Net":"unix"}} +{"time":"2026-07-23T02:28:50.770084447Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:28:50.770158191Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:28:50.770173329Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:28:50.770146502Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:28:50.770246375Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814719-815551-3138903381/socket","Net":"unix"}} +{"time":"2026-07-23T02:28:50.770382899Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773003008Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773054953Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773111864Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773106808Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-23T02:28:50.773073579Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-23T02:28:50.773200905Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-814701-815561-915654594/socket","Net":"unix"}} +{"time":"2026-07-23T02:28:52.626908382Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:28:52.626929775Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:28:52.626940482Z","level":"INFO","msg":"server is closed"} +{"time":"2026-07-23T02:28:52.662753611Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:28:52.662779647Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:28:52.662788745Z","level":"INFO","msg":"server is closed"} +{"time":"2026-07-23T02:28:52.68387658Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:28:52.683943364Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:28:52.683953617Z","level":"INFO","msg":"server is closed"} +{"time":"2026-07-23T02:29:02.470789399Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-23T02:29:02.470826347Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-23T02:29:02.470833873Z","level":"INFO","msg":"server is closed"} diff --git a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/logs/debug.log b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/logs/debug.log index 7a3e46d1782ad59ba3b65e8bcf331a7452c97511..e96e9c77955186c40ee26954dbba2b54d0f0d7f2 100644 --- a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/logs/debug.log +++ b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_013342-rd8tz18f/logs/debug.log @@ -18,3 +18,29 @@ config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2' 2026-07-23 01:33:43,908 INFO MainThread:814707 [wandb_run.py:_redirect():2442] Wrapping output streams. 2026-07-23 01:33:43,908 INFO MainThread:814707 [wandb_run.py:_redirect():2465] Redirects installed. 2026-07-23 01:33:43,910 INFO MainThread:814707 [wandb_init.py:init():1010] run started, returning control to user process +2026-07-23 02:28:50,768 INFO wandb-AsyncioManager-main:814707 [service_client.py:_forward_responses():122] Reached EOF. +2026-07-23 02:28:50,768 INFO wandb-AsyncioManager-main:814707 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles. +2026-07-23 02:28:50,769 ERROR wandb-AsyncioManager-main:814707 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop + await asyncio_compat.race( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race + async with open_task_group(race=True) as tg: + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__ + await anext(self.gen) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group + await task_group._wait_all(race=race, timeout=exit_timeout) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all + raise exc + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all + result = await handle.wait_async(timeout=None) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async + response = await self._handle.wait_async(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async + raise HandleAbandonedError() +wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError diff --git a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_022936-n29mechz/files/wandb-summary.json b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_022936-n29mechz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1d476fc88692f959c7a899096787abbc21a55dbc --- /dev/null +++ b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_022936-n29mechz/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":0,"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_022936-n29mechz/logs/debug-internal.log b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_022936-n29mechz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0b0a62b4c44840c46cb7acf8a2b9b9e5855cf9ea --- /dev/null +++ b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_022936-n29mechz/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2026-07-23T02:29:36.399183236Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-07-23T02:29:36.399416118Z","level":"INFO","msg":"stream: starting","core version":"0.27.2"} +{"time":"2026-07-23T02:29:36.88425252Z","level":"INFO","msg":"stream: created new stream","id":"n29mechz"} +{"time":"2026-07-23T02:29:36.88445717Z","level":"INFO","msg":"stream: started"} +{"time":"2026-07-23T02:29:36.884591987Z","level":"INFO","msg":"writer: started","stream_id":"n29mechz"} +{"time":"2026-07-23T02:29:36.884711168Z","level":"INFO","msg":"handler: started"} +{"time":"2026-07-23T02:29:36.884718084Z","level":"INFO","msg":"sender: started"} +{"time":"2026-07-23T02:29:37.175558891Z","level":"ERROR","msg":"runupserter: failed to init run","error":"no data but must resume"} +{"time":"2026-07-23T02:29:37.187206286Z","level":"ERROR","msg":"sender: uploadConfigFile: runhandle: run not yet initialized"} +{"time":"2026-07-23T02:29:37.187450238Z","level":"ERROR","msg":"runfiles: runhandle: run not yet initialized"} +{"time":"2026-07-23T02:29:37.18752772Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-07-23T02:29:37.187537638Z","level":"INFO","msg":"stream: finishing up"} +{"time":"2026-07-23T02:29:37.187543521Z","level":"INFO","msg":"handler: closed"} +{"time":"2026-07-23T02:29:37.187569184Z","level":"INFO","msg":"sender: closed"} +{"time":"2026-07-23T02:29:37.187571577Z","level":"INFO","msg":"stream: all finished"} diff --git a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_022936-n29mechz/run-n29mechz.wandb b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_022936-n29mechz/run-n29mechz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..7ee964ea2b84590b59b7e3fd00e471b22b2cbde8 Binary files /dev/null and b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_022936-n29mechz/run-n29mechz.wandb differ diff --git a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_023125-x6da8gup/files/output.log b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_023125-x6da8gup/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..65acb7fe82f12a1173df8d976807a311e2e448d9 --- /dev/null +++ b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_023125-x6da8gup/files/output.log @@ -0,0 +1,182 @@ +INFO 2026-07-23 02:31:26 db_utils.py:126 Logs will be synced with wandb. +INFO 2026-07-23 02:31:26 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/x6da8gup +INFO 2026-07-23 02:31:26 ot_train.py:307 Creating dataset +INFO 2026-07-23 02:31:29 ot_train.py:341 Creating policy +INFO 2026-07-23 02:31:29 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-23 02:31:29 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 5815.25it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 695/695 [00:00<00:00, 906.39it/s] +GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495 +Key | Status | | +--------------------------------------------------------------------+------------+--+- +action_head.model.transformer_blocks.{8...31}.attn1.to_v.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.ff.net.2.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_q.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.norm1.linear.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.ff.net.2.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_v.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_k.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_k.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.norm1.linear.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_q.bias | UNEXPECTED | | + +Notes: +- UNEXPECTED: can be ignored when loading from different task/architecture; not ok if you expect identical arch. +Loading weights from local directory +INFO 2026-07-23 02:31:46 ot_train.py:414 Creating optimizer and scheduler +INFO 2026-07-23 02:31:48 ot_train.py:448 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K +INFO 2026-07-23 02:31:48 ot_train.py:455 cfg.steps=60000 (60K) +INFO 2026-07-23 02:31:48 ot_train.py:456 dataset.num_frames=218367 (218K) +INFO 2026-07-23 02:31:48 ot_train.py:457 dataset.num_episodes=4930 +INFO 2026-07-23 02:31:48 ot_train.py:460 Effective batch size: 64 x 1 = 64 +INFO 2026-07-23 02:31:48 ot_train.py:461 num_learnable_params=810087553 (810M) +INFO 2026-07-23 02:31:48 ot_train.py:462 num_total_params=2333706401 (2B) +INFO 2026-07-23 02:31:48 ot_train.py:503 Resuming data order at epoch 13, sample 42176 +Training: 0%| | 0/30000 [00:00, 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 8, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_6D_DiT8_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'x6da8gup', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/030000', '_wandb': {}} +2026-07-23 02:31:25,118 INFO MainThread:2500816 [wandb_init.py:init():814] starting backend +2026-07-23 02:31:25,348 INFO MainThread:2500816 [wandb_init.py:init():829] sending inform_init request +2026-07-23 02:31:25,812 INFO MainThread:2500816 [wandb_init.py:init():834] backend started and connected +2026-07-23 02:31:25,816 INFO MainThread:2500816 [wandb_init.py:init():904] updated telemetry +2026-07-23 02:31:25,821 INFO MainThread:2500816 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-23 02:31:26,604 INFO MainThread:2500816 [wandb_init.py:init():972] starting run threads in backend +2026-07-23 02:31:26,704 INFO MainThread:2500816 [wandb_run.py:_console_start():2523] atexit reg +2026-07-23 02:31:26,705 INFO MainThread:2500816 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-23 02:31:26,705 INFO MainThread:2500816 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-23 02:31:26,705 INFO MainThread:2500816 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-23 02:31:26,707 INFO MainThread:2500816 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_163501-x6da8gup/files/config.yaml b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_163501-x6da8gup/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73454c68c0188d4efe0e304c9b2db85cc6110e43 --- /dev/null +++ b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_163501-x6da8gup/files/config.yaml @@ -0,0 +1,329 @@ +_wandb: + value: + cli_version: 0.27.2 + e: + yxy7kbzb7ty0n66653n01mpfixm5fpy5: + args: + - --config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model/train_config.json + - --resume=true + codePath: clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + codePathLocal: clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.9" + disk: + /: + total: "1888425144320" + used: "1327086579712" + executable: /home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python + gpu: NVIDIA H200 + gpu_count: 2 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc + host: ext_minje-box + memory: + total: "2164176568320" + os: Linux-6.8.0-55-generic-x86_64-with-glibc2.39 + program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + python: CPython 3.12.13 + root: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K + startedAt: "2026-07-23T16:35:01.611474Z" + writerId: yxy7kbzb7ty0n66653n01mpfixm5fpy5 + m: [] + python_version: 3.12.13 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 15 + - 16 + - 61 + - 62 + "4": 3.12.13 + "5": 0.27.2 + "6": 5.5.4 + "10": + - 21 + "12": 0.27.2 + "13": linux-x86_64 +batch_size: + value: 64 +checkpoint_path: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/rolling_00 +cudnn_deterministic: + value: false +dataset: + value: + depth_output_unit: mm + episodes: null + eval_split: 0 + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + return_uint8: false + revision: null + root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 + streaming: false + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +env_eval_freq: + value: 20000 +eval: + value: + batch_size: 50 + n_episodes: 50 + recording: false + recording_private: false + recording_repo_id: null + use_async_envs: true +eval_steps: + value: 0 +job: + value: + detach: false + image: huggingface/lerobot-gpu:latest + tags: [] + target: null + timeout: 2d +job_name: + value: INSIGHT_6D_DiT8_RKD_TimewarpVAE_RSCLstyle_C60 +log_freq: + value: 200 +max_eval_samples: + value: 0 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 1 + lr: 0.0001 + type: adamw + weight_decay: 1e-05 +output_dir: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K +peft: + value: null +persistent_workers: + value: true +policy: + value: + action_decode_transform: null + action_expert_num_layers: 8 + balance_dataset_weights: true + balance_trajectory_weights: true + base_model_path: nvidia/GR00T-N1.7-3B + batch_size: 32 + chunk_size: 16 + dataloader_num_workers: 8 + dataset_paths: null + device: cuda + embodiment_tag: new_embodiment + image_size: + - 256 + - 256 + input_features: + observation.images.guide: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.right_shoulder: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.wrist: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.state: + shape: + - 16 + type: STATE + license: null + lora_alpha: 16 + lora_dropout: 0.1 + lora_full_model: false + lora_rank: 0 + max_action_dim: 132 + max_state_dim: 132 + max_steps: 10000 + model_params_fp32: true + n_action_steps: 16 + n_obs_steps: 1 + normalization_mapping: + ACTION: IDENTITY + STATE: IDENTITY + VISUAL: IDENTITY + num_inference_timesteps: null + optimizer_betas: + - 0.9 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-05 + output_dir: ./tmp/gr00t + output_features: + action: + shape: + - 10 + type: ACTION + pretrained_path: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model + pretrained_revision: null + private: null + push_to_hub: false + relative_exclude_joints: [] + repo_id: null + report_to: wandb + resume: false + rkd_angle_weight: 2 + rkd_distance_weight: 1 + rkd_enabled: true + rkd_fm_loss_weight: 1 + rkd_loss_weight: 1 + rkd_student_attention_hidden_dim: 512 + rkd_student_attention_output_dim: 512 + rkd_student_pooling: attention + rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11 + rtc_ramp_rate: null + save_steps: 1000 + tags: null + tokenizer_assets_repo: null + tune_diffusion_model: true + tune_llm: false + tune_projector: true + tune_top_llm_layers: 0 + tune_visual: false + tune_vlln: true + type: groot_rkd + use_amp: false + use_bf16: true + use_flash_attention: false + use_peft: false + use_relative_actions: false + video_backend: decord + warmup_ratio: 0.05 + warmup_steps: 3000 +prefetch_factor: + value: 4 +resume: + value: true +reward_model: + value: null +sample_weighting: + value: null +save_checkpoint: + value: true +save_checkpoint_to_hub: + value: false +save_freq: + value: 20000 +scheduler: + value: + name: cosine + num_warmup_steps: 3000 + type: diffuser +seed: + value: 42 +steps: + value: 60000 +tolerance_s: + value: 0.0001 +use_policy_training_preset: + value: true +wandb: + value: + add_tags: true + disable_artifact: true + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: x6da8gup diff --git a/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_163501-x6da8gup/files/output.log b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_163501-x6da8gup/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b284e876748970da437603b8dc334961148cad3c --- /dev/null +++ b/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/wandb/run-20260723_163501-x6da8gup/files/output.log @@ -0,0 +1,69 @@ +INFO 2026-07-23 16:35:03 db_utils.py:126 Logs will be synced with wandb. +INFO 2026-07-23 16:35:03 db_utils.py:127 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/x6da8gup +INFO 2026-07-23 16:35:03 ot_train.py:310 Creating dataset +INFO 2026-07-23 16:35:05 ot_train.py:344 Creating policy +INFO 2026-07-23 16:35:05 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model +INFO 2026-07-23 16:35:05 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 7734.87it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 695/695 [00:04<00:00, 160.27it/s] +GR00TN17 LOAD REPORT from: /home/ext_minje/.cache/huggingface/hub/models--nvidia--GR00T-N1.7-3B/snapshots/2fc962b973bccdd5d8ce4f67cc63b264d6886495 +Key | Status | | +--------------------------------------------------------------------+------------+--+- +action_head.model.transformer_blocks.{8...31}.attn1.to_v.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.ff.net.2.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_q.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_q.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_k.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_v.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.norm1.linear.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_out.0.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.norm1.linear.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.attn1.to_k.weight | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.ff.net.0.proj.bias | UNEXPECTED | | +action_head.model.transformer_blocks.{8...31}.ff.net.2.bias | UNEXPECTED | | + +Notes: +- UNEXPECTED: can be ignored when loading from different task/architecture; not ok if you expect identical arch. +Loading weights from local directory +INFO 2026-07-23 16:35:27 ot_train.py:417 Creating optimizer and scheduler +INFO 2026-07-23 16:35:29 ot_train.py:451 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K +INFO 2026-07-23 16:35:29 ot_train.py:458 cfg.steps=60000 (60K) +INFO 2026-07-23 16:35:29 ot_train.py:459 dataset.num_frames=218367 (218K) +INFO 2026-07-23 16:35:29 ot_train.py:460 dataset.num_episodes=4930 +INFO 2026-07-23 16:35:29 ot_train.py:463 Effective batch size: 64 x 1 = 64 +INFO 2026-07-23 16:35:29 ot_train.py:464 num_learnable_params=810087553 (810M) +INFO 2026-07-23 16:35:29 ot_train.py:465 num_total_params=2333706401 (2B) +INFO 2026-07-23 16:35:29 ot_train.py:506 Resuming data order at epoch 24, sample 53248 +Training: 0%| | 0/5000 [00:00, 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [10]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/rolling_00/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': 8, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/TimewarpVAE_combo_top8_30k/C8_01_C3_008_beta_5em05_latent_dim_16_granularity_4_decoder_hidden_dim_512/checkpoint-step-30000-epoch-39.11', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_6D_DiT8_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 64, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'x6da8gup', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D/DiT_Layer/8/RSCLstyle_cosine_60K/checkpoints/rolling_00', '_wandb': {}} +2026-07-23 16:35:01,612 INFO MainThread:1457 [wandb_init.py:init():814] starting backend +2026-07-23 16:35:01,833 INFO MainThread:1457 [wandb_init.py:init():829] sending inform_init request +2026-07-23 16:35:02,307 INFO MainThread:1457 [wandb_init.py:init():834] backend started and connected +2026-07-23 16:35:02,308 INFO MainThread:1457 [wandb_init.py:init():904] updated telemetry +2026-07-23 16:35:02,309 INFO MainThread:1457 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-23 16:35:03,204 INFO MainThread:1457 [wandb_init.py:init():967] run resumed +2026-07-23 16:35:03,206 INFO MainThread:1457 [wandb_init.py:init():972] starting run threads in backend +2026-07-23 16:35:03,262 INFO MainThread:1457 [wandb_run.py:_console_start():2523] atexit reg +2026-07-23 16:35:03,262 INFO MainThread:1457 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-23 16:35:03,262 INFO MainThread:1457 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-23 16:35:03,262 INFO MainThread:1457 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-23 16:35:03,264 INFO MainThread:1457 [wandb_init.py:init():1010] run started, returning control to user process +2026-07-23 18:11:27,126 INFO wandb-AsyncioManager-main:1457 [service_client.py:_forward_responses():122] Reached EOF. +2026-07-23 18:11:27,126 INFO wandb-AsyncioManager-main:1457 [mailbox.py:close():154] Closing mailbox, abandoning 2 handles. +2026-07-23 18:11:27,128 ERROR wandb-AsyncioManager-main:1457 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 90, in loop + await asyncio_compat.race( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 278, in race + async with open_task_group(race=True) as tg: + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/contextlib.py", line 217, in __aexit__ + await anext(self.gen) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 240, in open_task_group + await task_group._wait_all(race=race, timeout=exit_timeout) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_compat.py", line 180, in _wait_all + raise exc + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_messages.py", line 113, in _print_all + result = await handle.wait_async(timeout=None) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox_handle.py", line 127, in wait_async + response = await self._handle.wait_async(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/response_handle.py", line 123, in wait_async + raise HandleAbandonedError() +wandb.sdk.mailbox.mailbox_handle.HandleAbandonedError +2026-07-23 18:11:34,104 ERROR wandb-AsyncioManager-main:1457 [asyncio_manager.py:fn_wrap_exceptions():184] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 182, in fn_wrap_exceptions + await fn() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 87, in loop + should_stop = await self._check_should_stop() + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/run_stopping.py", line 115, in _check_should_stop + handle = await self._interface.deliver_async(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 58, in deliver_async + handle = await self._client.deliver(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 57, in deliver + handle = self._mailbox.require_response(request) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/mailbox/mailbox.py", line 89, in require_response + raise MailboxClosedError() +wandb.sdk.mailbox.mailbox.MailboxClosedError