diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..5b59eb7d0cf3c517d87882fcffcc2f7751922df8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +wandb/run-20260518_153916-gnre0khg/run-gnre0khg.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20260519_221333-qz452p60/run-qz452p60.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/checkpoints/050000/pretrained_model/config.json b/checkpoints/050000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..34eef622558d21abd3a4bbac5ebe75ca670c9782 --- /dev/null +++ b/checkpoints/050000/pretrained_model/config.json @@ -0,0 +1,103 @@ +{ + "type": "groot", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 16 + ] + }, + "observation.images.guide": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.right_shoulder": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 8 + ] + }, + "action.skill_id": { + "type": "ACTION", + "shape": [ + 1 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 16, + "n_action_steps": 16, + "max_state_dim": 64, + "max_action_dim": 32, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "image_size": [ + 224, + 224 + ], + "base_model_path": "nvidia/GR00T-N1.5-3B", + "tokenizer_assets_repo": "lerobot/eagle2hg-processor-groot-n1p5", + "embodiment_tag": "new_embodiment", + "tune_llm": false, + "tune_visual": false, + "tune_projector": true, + "tune_diffusion_model": true, + "lora_rank": 0, + "lora_alpha": 16, + "lora_dropout": 0.1, + "lora_full_model": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-05, + "warmup_ratio": 0.05, + "use_bf16": true, + "video_backend": "pyav", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "dataset_paths": null, + "output_dir": "./tmp/gr00t", + "save_steps": 1000, + "max_steps": 10000, + "batch_size": 32, + "dataloader_num_workers": 8, + "report_to": "wandb", + "resume": false +} \ No newline at end of file diff --git a/checkpoints/050000/pretrained_model/model.safetensors b/checkpoints/050000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8b4761e2a693f0ef10e9bad6f2423fc523283cbd --- /dev/null +++ b/checkpoints/050000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:224dc1aa459dcd4f0077260e508b34fef13f3555b3c9912ea9fbee3a787ca76c +size 5448472032 diff --git a/checkpoints/050000/pretrained_model/policy_postprocessor.json b/checkpoints/050000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..27b83736b731eff24d5124ea9239a61afc4504e8 --- /dev/null +++ b/checkpoints/050000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,20 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "groot_action_unpack_unnormalize_v1", + "config": { + "env_action_dim": 8, + "normalize_min_max": true + }, + "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v1.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/050000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v1.safetensors b/checkpoints/050000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v1.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e24f6ab7a8fa734f2d997c07ac47be1612fe2a1d --- /dev/null +++ b/checkpoints/050000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v1.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cacc0ce3063ffe52d5e066d64ce32242b357c4ba7694ecc40f9937b7ca6a95a4 +size 15424 diff --git a/checkpoints/050000/pretrained_model/policy_preprocessor.json b/checkpoints/050000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..120161f63cbb3798cbaaad383abc365002f5fe0d --- /dev/null +++ b/checkpoints/050000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,52 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "groot_pack_inputs_v3", + "config": { + "state_horizon": 1, + "action_horizon": 16, + "max_state_dim": 64, + "max_action_dim": 32, + "language_key": "task", + "formalize_language": false, + "embodiment_tag": "new_embodiment", + "embodiment_mapping": { + "new_embodiment": 31, + "oxe_droid": 17, + "agibot_genie1": 26, + "gr1": 24, + "so100": 2, + "unitree_g1": 3 + }, + "normalize_min_max": true + }, + "state_file": "policy_preprocessor_step_2_groot_pack_inputs_v3.safetensors" + }, + { + "registry_name": "groot_eagle_encode_v3", + "config": {} + }, + { + "registry_name": "groot_eagle_collate_v3", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/050000/pretrained_model/policy_preprocessor_step_2_groot_pack_inputs_v3.safetensors b/checkpoints/050000/pretrained_model/policy_preprocessor_step_2_groot_pack_inputs_v3.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e24f6ab7a8fa734f2d997c07ac47be1612fe2a1d --- /dev/null +++ b/checkpoints/050000/pretrained_model/policy_preprocessor_step_2_groot_pack_inputs_v3.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cacc0ce3063ffe52d5e066d64ce32242b357c4ba7694ecc40f9937b7ca6a95a4 +size 15424 diff --git a/checkpoints/050000/pretrained_model/train_config.json b/checkpoints/050000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..86aba5dd9d67e8bea8e36655f6e84bdea228e9d6 --- /dev/null +++ b/checkpoints/050000/pretrained_model/train_config.json @@ -0,0 +1,258 @@ +{ + "dataset": { + "repo_id": "/home/seonho/insight/insight_dataset", + "root": "/home/seonho/insight/insight_dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav", + "streaming": false + }, + "env": null, + "policy": { + "type": "groot", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 16 + ] + }, + "observation.images.guide": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.right_shoulder": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 8 + ] + }, + "action.skill_id": { + "type": "ACTION", + "shape": [ + 1 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 16, + "n_action_steps": 16, + "max_state_dim": 64, + "max_action_dim": 32, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "image_size": [ + 224, + 224 + ], + "base_model_path": "nvidia/GR00T-N1.5-3B", + "tokenizer_assets_repo": "lerobot/eagle2hg-processor-groot-n1p5", + "embodiment_tag": "new_embodiment", + "tune_llm": false, + "tune_visual": false, + "tune_projector": true, + "tune_diffusion_model": true, + "lora_rank": 0, + "lora_alpha": 16, + "lora_dropout": 0.1, + "lora_full_model": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-05, + "warmup_ratio": 0.05, + "use_bf16": true, + "video_backend": "pyav", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "dataset_paths": null, + "output_dir": "./tmp/gr00t", + "save_steps": 1000, + "max_steps": 10000, + "batch_size": 32, + "dataloader_num_workers": 8, + "report_to": "wandb", + "resume": false + }, + "output_dir": "/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "job_name": "groot_insight", + "resume": false, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 8, + "batch_size": 64, + "steps": 100000, + "eval_freq": 20000, + "log_freq": 100, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 50000, + "use_policy_training_preset": true, + "optimizer": null, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false, + "max_episodes_rendered": 10 + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "groot_insight_baseline_base", + "entity": "RwHlabs", + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true, + "name": "insight_phase1_dit_only_groot", + "group": "insight_phase1_dit_only", + "tags": "base,phase2,groot,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + }, + "peft": null, + "use_rabc": false, + "rabc_progress_path": null, + "rabc_kappa": 0.01, + "rabc_epsilon": 1e-06, + "rabc_head_mode": "sparse", + "rename_map": {}, + "checkpoint_path": null, + "method": "base", + "phase": "phase2", + "pretrained_path": "", + "gradient_checkpointing": false, + "smoke_test": false, + "dry_run": false, + "grad_clip_norm": 10.0, + "write_reports": true, + "fail_on_language_fallback": true, + "action_dim": 8, + "state_dim": 16, + "camera_set": "rgb", + "video_feature_keys": "", + "task_prompt_mode": "non_guide", + "task_descriptions_path": "", + "excluded_indices_path": "", + "exclude_index_scope": "none", + "recipe_yaml": "", + "pipeline_yaml": "", + "pipeline_mode": "both", + "dry_print_commands": false, + "launcher": "python", + "phase2_output_dir": "", + "phase3_output_dir": "", + "wandb_auto_name": true, + "run_name": "", + "wandb_project_prefix": "groot_insight", + "wandb_group": "", + "wandb_tags": "" +} \ No newline at end of file diff --git a/checkpoints/050000/training_state/optimizer_param_groups.json b/checkpoints/050000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..3638115ca753d4578ee45a9391e18fbc87eee415 --- /dev/null +++ b/checkpoints/050000/training_state/optimizer_param_groups.json @@ -0,0 +1,335 @@ +[ + { + "lr": 5.4128967273616625e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313 + ] + } +] \ No newline at end of file diff --git a/checkpoints/050000/training_state/optimizer_state.safetensors b/checkpoints/050000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..58e5a0546cf0a692c79a6e18d23c966f3606f72b --- /dev/null +++ b/checkpoints/050000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b7d4f28f0f9036678bfc35f6175e2eec7348050ca406eeb2e8d41a6b4593a56 +size 4275306416 diff --git a/checkpoints/050000/training_state/rng_state.safetensors b/checkpoints/050000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c3d2082195a87ae3530d981c8c6b2c1339b79d73 --- /dev/null +++ b/checkpoints/050000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddf10679b738bfb52bcb08b700d0fdb5fa15c0aaff47068661cfe7bb427cf9ba +size 15708 diff --git a/checkpoints/050000/training_state/scheduler_state.json b/checkpoints/050000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..10d0aaef2b466b1e9af07952ff880b366993fe95 --- /dev/null +++ b/checkpoints/050000/training_state/scheduler_state.json @@ -0,0 +1,14 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 50000, + "_step_count": 50001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.4128967273616625e-05 + ], + "lr_lambdas": [ + {} + ] +} \ No newline at end of file diff --git a/checkpoints/050000/training_state/training_step.json b/checkpoints/050000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..b3c03563f17d6d98552117ccf00705b693aa3fbd --- /dev/null +++ b/checkpoints/050000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 50000 +} \ No newline at end of file diff --git a/checkpoints/100000/pretrained_model/config.json b/checkpoints/100000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..34eef622558d21abd3a4bbac5ebe75ca670c9782 --- /dev/null +++ b/checkpoints/100000/pretrained_model/config.json @@ -0,0 +1,103 @@ +{ + "type": "groot", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 16 + ] + }, + "observation.images.guide": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.right_shoulder": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 8 + ] + }, + "action.skill_id": { + "type": "ACTION", + "shape": [ + 1 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 16, + "n_action_steps": 16, + "max_state_dim": 64, + "max_action_dim": 32, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "image_size": [ + 224, + 224 + ], + "base_model_path": "nvidia/GR00T-N1.5-3B", + "tokenizer_assets_repo": "lerobot/eagle2hg-processor-groot-n1p5", + "embodiment_tag": "new_embodiment", + "tune_llm": false, + "tune_visual": false, + "tune_projector": true, + "tune_diffusion_model": true, + "lora_rank": 0, + "lora_alpha": 16, + "lora_dropout": 0.1, + "lora_full_model": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-05, + "warmup_ratio": 0.05, + "use_bf16": true, + "video_backend": "pyav", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "dataset_paths": null, + "output_dir": "./tmp/gr00t", + "save_steps": 1000, + "max_steps": 10000, + "batch_size": 32, + "dataloader_num_workers": 8, + "report_to": "wandb", + "resume": false +} \ No newline at end of file diff --git a/checkpoints/100000/pretrained_model/model.safetensors b/checkpoints/100000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6025f13275651901fe1200084b47cdf0f30bf8b1 --- /dev/null +++ b/checkpoints/100000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c06fb1f18b720c7d5870fd740a4bd777d06f04d826c9bc483254e643c296318 +size 5448472032 diff --git a/checkpoints/100000/pretrained_model/policy_postprocessor.json b/checkpoints/100000/pretrained_model/policy_postprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..27b83736b731eff24d5124ea9239a61afc4504e8 --- /dev/null +++ b/checkpoints/100000/pretrained_model/policy_postprocessor.json @@ -0,0 +1,20 @@ +{ + "name": "policy_postprocessor", + "steps": [ + { + "registry_name": "groot_action_unpack_unnormalize_v1", + "config": { + "env_action_dim": 8, + "normalize_min_max": true + }, + "state_file": "policy_postprocessor_step_0_groot_action_unpack_unnormalize_v1.safetensors" + }, + { + "registry_name": "device_processor", + "config": { + "device": "cpu", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/100000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v1.safetensors b/checkpoints/100000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v1.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e24f6ab7a8fa734f2d997c07ac47be1612fe2a1d --- /dev/null +++ b/checkpoints/100000/pretrained_model/policy_postprocessor_step_0_groot_action_unpack_unnormalize_v1.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cacc0ce3063ffe52d5e066d64ce32242b357c4ba7694ecc40f9937b7ca6a95a4 +size 15424 diff --git a/checkpoints/100000/pretrained_model/policy_preprocessor.json b/checkpoints/100000/pretrained_model/policy_preprocessor.json new file mode 100644 index 0000000000000000000000000000000000000000..120161f63cbb3798cbaaad383abc365002f5fe0d --- /dev/null +++ b/checkpoints/100000/pretrained_model/policy_preprocessor.json @@ -0,0 +1,52 @@ +{ + "name": "policy_preprocessor", + "steps": [ + { + "registry_name": "rename_observations_processor", + "config": { + "rename_map": {} + } + }, + { + "registry_name": "to_batch_processor", + "config": {} + }, + { + "registry_name": "groot_pack_inputs_v3", + "config": { + "state_horizon": 1, + "action_horizon": 16, + "max_state_dim": 64, + "max_action_dim": 32, + "language_key": "task", + "formalize_language": false, + "embodiment_tag": "new_embodiment", + "embodiment_mapping": { + "new_embodiment": 31, + "oxe_droid": 17, + "agibot_genie1": 26, + "gr1": 24, + "so100": 2, + "unitree_g1": 3 + }, + "normalize_min_max": true + }, + "state_file": "policy_preprocessor_step_2_groot_pack_inputs_v3.safetensors" + }, + { + "registry_name": "groot_eagle_encode_v3", + "config": {} + }, + { + "registry_name": "groot_eagle_collate_v3", + "config": {} + }, + { + "registry_name": "device_processor", + "config": { + "device": "cuda", + "float_dtype": null + } + } + ] +} \ No newline at end of file diff --git a/checkpoints/100000/pretrained_model/policy_preprocessor_step_2_groot_pack_inputs_v3.safetensors b/checkpoints/100000/pretrained_model/policy_preprocessor_step_2_groot_pack_inputs_v3.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e24f6ab7a8fa734f2d997c07ac47be1612fe2a1d --- /dev/null +++ b/checkpoints/100000/pretrained_model/policy_preprocessor_step_2_groot_pack_inputs_v3.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cacc0ce3063ffe52d5e066d64ce32242b357c4ba7694ecc40f9937b7ca6a95a4 +size 15424 diff --git a/checkpoints/100000/pretrained_model/train_config.json b/checkpoints/100000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..86aba5dd9d67e8bea8e36655f6e84bdea228e9d6 --- /dev/null +++ b/checkpoints/100000/pretrained_model/train_config.json @@ -0,0 +1,258 @@ +{ + "dataset": { + "repo_id": "/home/seonho/insight/insight_dataset", + "root": "/home/seonho/insight/insight_dataset", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "affine": { + "weight": 1.0, + "type": "RandomAffine", + "kwargs": { + "degrees": [ + -5.0, + 5.0 + ], + "translate": [ + 0.05, + 0.05 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "pyav", + "streaming": false + }, + "env": null, + "policy": { + "type": "groot", + "n_obs_steps": 1, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 16 + ] + }, + "observation.images.guide": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.right_shoulder": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 224, + 224 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 8 + ] + }, + "action.skill_id": { + "type": "ACTION", + "shape": [ + 1 + ] + } + }, + "device": "cuda", + "use_amp": false, + "use_peft": false, + "push_to_hub": true, + "repo_id": null, + "private": null, + "tags": null, + "license": null, + "pretrained_path": null, + "chunk_size": 16, + "n_action_steps": 16, + "max_state_dim": 64, + "max_action_dim": 32, + "normalization_mapping": { + "VISUAL": "IDENTITY", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "image_size": [ + 224, + 224 + ], + "base_model_path": "nvidia/GR00T-N1.5-3B", + "tokenizer_assets_repo": "lerobot/eagle2hg-processor-groot-n1p5", + "embodiment_tag": "new_embodiment", + "tune_llm": false, + "tune_visual": false, + "tune_projector": true, + "tune_diffusion_model": true, + "lora_rank": 0, + "lora_alpha": 16, + "lora_dropout": 0.1, + "lora_full_model": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-05, + "warmup_ratio": 0.05, + "use_bf16": true, + "video_backend": "pyav", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "dataset_paths": null, + "output_dir": "./tmp/gr00t", + "save_steps": 1000, + "max_steps": 10000, + "batch_size": 32, + "dataloader_num_workers": 8, + "report_to": "wandb", + "resume": false + }, + "output_dir": "/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "job_name": "groot_insight", + "resume": false, + "seed": 42, + "cudnn_deterministic": false, + "num_workers": 8, + "batch_size": 64, + "steps": 100000, + "eval_freq": 20000, + "log_freq": 100, + "tolerance_s": 0.0001, + "save_checkpoint": true, + "save_freq": 50000, + "use_policy_training_preset": true, + "optimizer": null, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false, + "max_episodes_rendered": 10 + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "groot_insight_baseline_base", + "entity": "RwHlabs", + "notes": null, + "run_id": null, + "mode": null, + "add_tags": true, + "name": "insight_phase1_dit_only_groot", + "group": "insight_phase1_dit_only", + "tags": "base,phase2,groot,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + }, + "peft": null, + "use_rabc": false, + "rabc_progress_path": null, + "rabc_kappa": 0.01, + "rabc_epsilon": 1e-06, + "rabc_head_mode": "sparse", + "rename_map": {}, + "checkpoint_path": null, + "method": "base", + "phase": "phase2", + "pretrained_path": "", + "gradient_checkpointing": false, + "smoke_test": false, + "dry_run": false, + "grad_clip_norm": 10.0, + "write_reports": true, + "fail_on_language_fallback": true, + "action_dim": 8, + "state_dim": 16, + "camera_set": "rgb", + "video_feature_keys": "", + "task_prompt_mode": "non_guide", + "task_descriptions_path": "", + "excluded_indices_path": "", + "exclude_index_scope": "none", + "recipe_yaml": "", + "pipeline_yaml": "", + "pipeline_mode": "both", + "dry_print_commands": false, + "launcher": "python", + "phase2_output_dir": "", + "phase3_output_dir": "", + "wandb_auto_name": true, + "run_name": "", + "wandb_project_prefix": "groot_insight", + "wandb_group": "", + "wandb_tags": "" +} \ No newline at end of file diff --git a/checkpoints/100000/training_state/optimizer_param_groups.json b/checkpoints/100000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..e6279ef9a801a19a10ae68b12d75e2abaded8797 --- /dev/null +++ b/checkpoints/100000/training_state/optimizer_param_groups.json @@ -0,0 +1,335 @@ +[ + { + "lr": 0.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-05, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313 + ] + } +] \ No newline at end of file diff --git a/checkpoints/100000/training_state/optimizer_state.safetensors b/checkpoints/100000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f1d24d0192664ddbaf374f58b2c180294c33dba3 --- /dev/null +++ b/checkpoints/100000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:538fd88805669011d3587f7dbce7727d1ba801e4649a970275978e5b963057ff +size 4275306416 diff --git a/checkpoints/100000/training_state/rng_state.safetensors b/checkpoints/100000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..74fc7b068f595854b31ea8418e1d91e831f0daf7 --- /dev/null +++ b/checkpoints/100000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34b815630d4ce3f560a5b0a3d07e08fd8cd8a03c15fdd6df6ef15a81e44ceb7f +size 15708 diff --git a/checkpoints/100000/training_state/scheduler_state.json b/checkpoints/100000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..785c85ce6f2b6daeed130e4ccaccf8c24441dd4c --- /dev/null +++ b/checkpoints/100000/training_state/scheduler_state.json @@ -0,0 +1,14 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 100000, + "_step_count": 100001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.0 + ], + "lr_lambdas": [ + {} + ] +} \ No newline at end of file diff --git a/checkpoints/100000/training_state/training_step.json b/checkpoints/100000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8eb73c13d28bca88058c08796abbf931c3f9b012 --- /dev/null +++ b/checkpoints/100000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 100000 +} \ No newline at end of file diff --git a/dataset_sample_report.json b/dataset_sample_report.json new file mode 100644 index 0000000000000000000000000000000000000000..7fd6ffa4135a00021b80c0c141fc7201a662c0f7 --- /dev/null +++ b/dataset_sample_report.json @@ -0,0 +1,43 @@ +{ + "action_shape": [ + 16, + 8 + ], + "camera_keys_present": { + "observation.images.guide": true, + "observation.images.right_shoulder": true, + "observation.images.wrist": true + }, + "camera_order_effective": [ + "observation.images.guide", + "observation.images.right_shoulder", + "observation.images.wrist" + ], + "keys_after_filter_slice": [ + "action", + "action.skill_id", + "action_is_pad", + "episode_index", + "frame_index", + "index", + "next.reward", + "observation.images.guide", + "observation.images.right_shoulder", + "observation.images.wrist", + "observation.state", + "task", + "task_index", + "timestamp" + ], + "observation_state_shape": [ + 16 + ], + "sample_index": 0, + "task": "1ext", + "task_index": { + "preview": [ + 0 + ], + "shape": [] + } +} \ No newline at end of file diff --git a/first_batch_report.json b/first_batch_report.json new file mode 100644 index 0000000000000000000000000000000000000000..f47d82190319a5652f125f022c3cd224f6456f4e --- /dev/null +++ b/first_batch_report.json @@ -0,0 +1,207 @@ +{ + "action_dim_effective": 8, + "batch_spec": { + "action_effective_dim": 8, + "action_horizon": 16, + "action_order": [ + "action" + ], + "action_storage_dim": 32, + "camera_order_effective": [ + "observation.images.guide", + "observation.images.right_shoulder", + "observation.images.wrist" + ], + "chunk_size": 16, + "padded_action_dim": 32, + "padded_state_dim": 64, + "state_effective_dim": 16, + "state_order": [ + "observation.state" + ], + "state_storage_dim": 32 + }, + "camera_order_effective": [ + "observation.images.guide", + "observation.images.right_shoulder", + "observation.images.wrist" + ], + "camera_set": "rgb", + "dataset_len": 246793, + "language_fallback_detected": false, + "language_fallback_scan": { + "all_paths": [], + "detected": false, + "processed_paths": [], + "raw_paths": [] + }, + "method": "base", + "phase": "phase2", + "policy_type": "groot", + "processed_action_shape": [ + 64, + 16, + 32 + ], + "processed_state_shape": [ + 64, + 1, + 64 + ], + "raw_action_shape": [ + 64, + 16, + 8 + ], + "raw_keys": [ + "action", + "action.skill_id", + "action_is_pad", + "episode_index", + "frame_index", + "index", + "next.reward", + "observation.images.guide", + "observation.images.right_shoulder", + "observation.images.wrist", + "observation.state", + "task", + "task_index", + "timestamp" + ], + "raw_observation_state_shape": [ + 64, + 16 + ], + "raw_task_preview": [ + "Open the bottle following the guide.", + "Open the bottle following the guide.", + "Open the door following the guide." + ], + "script_name": "train_groot_insight_yaml.py", + "state_dim_effective": 16, + "task_prompt_mode": "non_guide", + "train_batch_tensor_report": { + "action": { + "device": "cuda:0", + "dtype": "torch.float32", + "shape": [ + 64, + 16, + 32 + ] + }, + "action_is_pad": { + "device": "cuda:0", + "dtype": "torch.bool", + "shape": [ + 64, + 16 + ] + }, + "action_mask": { + "device": "cuda:0", + "dtype": "torch.bool", + "shape": [ + 64, + 16, + 32 + ] + }, + "eagle_attention_mask": { + "device": "cuda:0", + "dtype": "torch.int64", + "shape": [ + 64, + 816 + ] + }, + "eagle_image_sizes": { + "device": "cuda:0", + "dtype": "torch.int64", + "shape": [ + 192, + 2 + ] + }, + "eagle_input_ids": { + "device": "cuda:0", + "dtype": "torch.int64", + "shape": [ + 64, + 816 + ] + }, + "eagle_pixel_values": { + "device": "cuda:0", + "dtype": "torch.float32", + "shape": [ + 192, + 3, + 224, + 224 + ] + }, + "embodiment_id": { + "device": "cuda:0", + "dtype": "torch.int64", + "shape": [ + 64 + ] + }, + "episode_index": { + "device": "cuda:0", + "dtype": "torch.int64", + "shape": [ + 64 + ] + }, + "index": { + "device": "cuda:0", + "dtype": "torch.int64", + "shape": [ + 64 + ] + }, + "next.reward": { + "device": "cuda:0", + "dtype": "torch.float32", + "shape": [ + 64 + ] + }, + "observation.state": { + "device": "cuda:0", + "dtype": "torch.float32", + "shape": [ + 64, + 16 + ] + }, + "state": { + "device": "cuda:0", + "dtype": "torch.float32", + "shape": [ + 64, + 1, + 64 + ] + }, + "state_mask": { + "device": "cuda:0", + "dtype": "torch.bool", + "shape": [ + 64, + 1, + 64 + ] + }, + "task_index": { + "device": "cuda:0", + "dtype": "torch.int64", + "shape": [ + 64 + ] + } + } +} \ No newline at end of file diff --git a/pipeline_commands.json b/pipeline_commands.json new file mode 100644 index 0000000000000000000000000000000000000000..67f501c74c3f6b4d12191f1d4d6cf77c737dc6d5 --- /dev/null +++ b/pipeline_commands.json @@ -0,0 +1,65 @@ +{ + "phase2_args": { + "batch_size": 64, + "camera_set": "rgb", + "dataset.repo_id": "/home/seonho/insight/insight_dataset", + "dataset.root": "/home/seonho/insight/insight_dataset", + "dataset.video_backend": "pyav", + "log_freq": 100, + "num_workers": 8, + "output_dir": "/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "phase": "phase2", + "policy.base_model_path": "nvidia/GR00T-N1.5-3B", + "policy.lora_rank": 0, + "policy.tune_diffusion_model": true, + "policy.tune_llm": false, + "policy.tune_projector": true, + "policy.tune_visual": false, + "policy.type": "groot", + "save_freq": 50000, + "steps": 100000, + "task_prompt_mode": "non_guide", + "wandb.enable": true, + "wandb.entity": "RwHlabs", + "wandb.group": "insight_phase1_dit_only", + "wandb.name": "insight_phase1_dit_only_groot", + "wandb.project": "groot_insight_baseline_base", + "wandb.tags": "base,phase2,groot,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + }, + "phase2_command": [ + "accelerate", + "launch", + "--num_processes", + "1", + "--mixed_precision", + "no", + "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "--batch_size=64", + "--camera_set=rgb", + "--dataset.repo_id=/home/seonho/insight/insight_dataset", + "--dataset.root=/home/seonho/insight/insight_dataset", + "--dataset.video_backend=pyav", + "--log_freq=100", + "--num_workers=8", + "--output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "--phase=phase2", + "--policy.base_model_path=nvidia/GR00T-N1.5-3B", + "--policy.lora_rank=0", + "--policy.tune_diffusion_model=true", + "--policy.tune_llm=false", + "--policy.tune_projector=true", + "--policy.tune_visual=false", + "--policy.type=groot", + "--save_freq=50000", + "--steps=100000", + "--task_prompt_mode=non_guide", + "--wandb.enable=true", + "--wandb.entity=RwHlabs", + "--wandb.group=insight_phase1_dit_only", + "--wandb.name=insight_phase1_dit_only_groot", + "--wandb.project=groot_insight_baseline_base", + "--wandb.tags=base,phase2,groot,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + ], + "phase3_args": null, + "phase3_command": null +} \ No newline at end of file diff --git a/pipeline_manifest.json b/pipeline_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..1c515b92e27107d7cb94f3e5008bd41e7c571d03 --- /dev/null +++ b/pipeline_manifest.json @@ -0,0 +1,91 @@ +{ + "end_timestamp": "2026-05-23T05:32:47.031197+00:00", + "git_commit": "f8482718aaa75ce2ad60b9b8dd7070a607da4249", + "launcher": "accelerate launch --num_processes 1 --mixed_precision no", + "phase2_command": [ + "accelerate", + "launch", + "--num_processes", + "1", + "--mixed_precision", + "no", + "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "--batch_size=64", + "--camera_set=rgb", + "--dataset.repo_id=/home/seonho/insight/insight_dataset", + "--dataset.root=/home/seonho/insight/insight_dataset", + "--dataset.video_backend=pyav", + "--log_freq=100", + "--num_workers=8", + "--output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "--phase=phase2", + "--policy.base_model_path=nvidia/GR00T-N1.5-3B", + "--policy.lora_rank=0", + "--policy.tune_diffusion_model=true", + "--policy.tune_llm=false", + "--policy.tune_projector=true", + "--policy.tune_visual=false", + "--policy.type=groot", + "--save_freq=50000", + "--steps=100000", + "--task_prompt_mode=non_guide", + "--wandb.enable=true", + "--wandb.entity=RwHlabs", + "--wandb.group=insight_phase1_dit_only", + "--wandb.name=insight_phase1_dit_only_groot", + "--wandb.project=groot_insight_baseline_base", + "--wandb.tags=base,phase2,groot,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + ], + "phase2_output_dir": "/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "phase3_command": null, + "phase3_output_dir": null, + "pipeline_mode": "phase2_only", + "recipe_yaml": "/home/seonho/clvla/Experiments_cfg/insight/phase1_dit_only.yaml", + "resolved_phase2_checkpoint": null, + "return_status": "success", + "start_timestamp": "2026-05-19T13:13:01.780734+00:00", + "wandb_resolved": { + "group": "insight_phase1_dit_only", + "phase2": { + "enable": true, + "entity": "RwHlabs", + "group": "insight_phase1_dit_only", + "method": "base", + "name": "insight_phase1_dit_only_groot", + "phase": "phase2", + "project": "groot_insight_baseline_base", + "tags": [ + "base", + "phase2", + "groot", + "rgb", + "non_guide", + "insight", + "insight_native", + "cartesian_8d", + "phase1", + "baseline", + "dit_only", + "nvidia_base" + ] + }, + "phase2_name": "insight_phase1_dit_only_groot", + "phase3": null, + "phase3_name": null, + "project": "groot_insight_baseline_base", + "tags": [ + "base", + "phase2", + "groot", + "rgb", + "non_guide", + "insight", + "insight_native", + "cartesian_8d", + "phase1", + "baseline", + "dit_only", + "nvidia_base" + ] + } +} \ No newline at end of file diff --git a/run_manifest.json b/run_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..82322643df232ab4410396fc774405f1dc34a072 --- /dev/null +++ b/run_manifest.json @@ -0,0 +1,114 @@ +{ + "action_dim_effective": 8, + "action_dim_storage": 32, + "batch_spec": { + "action_effective_dim": 8, + "action_horizon": 16, + "action_order": [ + "action" + ], + "action_storage_dim": 32, + "camera_order_effective": [ + "observation.images.guide", + "observation.images.right_shoulder", + "observation.images.wrist" + ], + "chunk_size": 16, + "padded_action_dim": 32, + "padded_state_dim": 64, + "state_effective_dim": 16, + "state_order": [ + "observation.state" + ], + "state_storage_dim": 32 + }, + "camera_order_effective": [ + "observation.images.guide", + "observation.images.right_shoulder", + "observation.images.wrist" + ], + "camera_set": "rgb", + "checkpoint_processor_state_loaded": false, + "checkpoint_processor_state_note": "phase3 loads model.safetensors weights only; pre/post processors are rebuilt from the current INSIGHT dataset stats for normalization consistency.", + "cl_v2_fm_loss_weight": null, + "cl_v2_loss_weight": null, + "cl_v2_phase": null, + "cl_v2_trainable_mode": null, + "created_at_utc": "2026-05-19T13:13:32.874286+00:00", + "dataset_len": 246793, + "dataset_root": "/home/seonho/insight/insight_dataset", + "exclude_index_scope": "none", + "excluded_indices_path": "", + "fps": 10, + "loaded_pretrained_model_path": null, + "method": "base", + "mgd_enabled": null, + "mgd_loss_weight": null, + "mgd_trainable_mode": null, + "phase": "phase2", + "policy_type": "groot", + "preprocessor_stats_source": "INSIGHT dataset stats sliced to effective action/state dims", + "pretrained_path": "", + "repo_id": "/home/seonho/insight/insight_dataset", + "rkd_loss_weight": null, + "script_name": "train_groot_insight_yaml.py", + "state_dim_effective": 16, + "state_dim_storage": 32, + "task_prompt_mode": "non_guide", + "total_episodes_metadata": 5647, + "total_frames_metadata": 246793, + "total_tasks_metadata": 13, + "trainable_report": { + "dit_core_only_unexpected_trainable_params": 0, + "groups": { + "_groot_model.action_head.model": { + "total": 550386688, + "trainable": 550386688 + }, + "_groot_model.action_head.vl_self_attention": { + "total": 201433088, + "trainable": 201433088 + }, + "_groot_model.action_head.vlln": { + "total": 4096, + "trainable": 4096 + }, + "sequence_mgd_head": { + "total": 0, + "trainable": 0 + } + }, + "num_learnable_params": 1068806144, + "num_total_params": 2724163520, + "other_trainable_params": 1068806144, + "policy_type": "groot", + "selected_prefixes": [], + "trainable_mode": null, + "trainable_ratio": 0.3923428737493702 + }, + "wandb_resolved": { + "camera_set": "rgb", + "enable": true, + "entity": "RwHlabs", + "group": "insight_phase1_dit_only", + "method": "base", + "name": "insight_phase1_dit_only_groot", + "phase": "phase2", + "project": "groot_insight_baseline_base", + "tags": [ + "base", + "phase2", + "groot", + "rgb", + "non_guide", + "insight", + "insight_native", + "cartesian_8d", + "phase1", + "baseline", + "dit_only", + "nvidia_base" + ], + "task_prompt_mode": "non_guide" + } +} \ No newline at end of file diff --git a/train_summary_report.json b/train_summary_report.json new file mode 100644 index 0000000000000000000000000000000000000000..72bf48cdd993dfa4dee29fd6aeac8556b1581aef --- /dev/null +++ b/train_summary_report.json @@ -0,0 +1,17 @@ +{ + "final_checkpoint": "/home/seonho/insight/outputs/dit_only_phase1_ckpt/checkpoints/100000", + "last_grad_norm": 1.2890625, + "last_loss_finite": true, + "last_lr": 0.0, + "last_policy_update_called": false, + "last_scalars": { + "loss": 0.06392817199230194 + }, + "loaded_pretrained_model_path": null, + "method": "base", + "phase": "phase2", + "policy_type": "groot", + "script_name": "train_groot_insight_yaml.py", + "steps_completed": 100000, + "steps_requested": 100000 +} \ No newline at end of file diff --git a/trainable_param_report.json b/trainable_param_report.json new file mode 100644 index 0000000000000000000000000000000000000000..3a661cc8f3739830b6adb920ee02166fa5bea3c1 --- /dev/null +++ b/trainable_param_report.json @@ -0,0 +1,28 @@ +{ + "dit_core_only_unexpected_trainable_params": 0, + "groups": { + "_groot_model.action_head.model": { + "total": 550386688, + "trainable": 550386688 + }, + "_groot_model.action_head.vl_self_attention": { + "total": 201433088, + "trainable": 201433088 + }, + "_groot_model.action_head.vlln": { + "total": 4096, + "trainable": 4096 + }, + "sequence_mgd_head": { + "total": 0, + "trainable": 0 + } + }, + "num_learnable_params": 1068806144, + "num_total_params": 2724163520, + "other_trainable_params": 1068806144, + "policy_type": "groot", + "selected_prefixes": [], + "trainable_mode": null, + "trainable_ratio": 0.3923428737493702 +} \ No newline at end of file diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..270ee03a66ab99851e9429a03aac4d728af8cc98 --- /dev/null +++ b/wandb/debug-internal.log @@ -0,0 +1,41 @@ +{"time":"2026-05-19T22:13:33.757508612+09:00","level":"INFO","msg":"stream: starting","core version":"0.24.2"} +{"time":"2026-05-19T22:13:34.249681253+09:00","level":"INFO","msg":"stream: created new stream","id":"qz452p60"} +{"time":"2026-05-19T22:13:34.249816267+09:00","level":"INFO","msg":"handler: started","stream_id":"qz452p60"} +{"time":"2026-05-19T22:13:34.250122028+09:00","level":"INFO","msg":"stream: started","id":"qz452p60"} +{"time":"2026-05-19T22:13:34.250159759+09:00","level":"INFO","msg":"sender: started","stream_id":"qz452p60"} +{"time":"2026-05-19T22:13:34.250156101+09:00","level":"INFO","msg":"writer: started","stream_id":"qz452p60"} +{"time":"2026-05-20T15:55:10.818156537+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T15:56:44.994984532+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T15:58:23.494572793+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T17:32:56.207023413+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T17:33:53.844718799+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T17:34:53.908902815+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T17:35:57.323694482+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:30:49.563054911+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:48:36.484690792+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:49:38.236616601+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:51:49.732015936+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:59:43.774277496+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:03:49.908408418+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:04:48.543201774+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:05:45.963581069+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:07:50.284610418+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:12:30.299479148+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:14:13.564300748+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:15:14.101427763+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:16:15.40790845+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:29:32.060104195+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:31:30.297052013+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:49:26.161299534+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:50:23.803656458+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T20:04:29.31970088+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T20:49:42.028043576+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T20:50:44.339074594+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-23T13:56:03.488569868+09:00","level":"INFO","msg":"api: retrying error","error":"unexpected HTTP status 502 Bad Gateway"} +{"time":"2026-05-23T14:25:43.511326685+09:00","level":"INFO","msg":"api: retrying error","error":"unexpected HTTP status 502 Bad Gateway"} +{"time":"2026-05-23T14:32:39.967322696+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-23T14:32:40.260656129+09:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-23T14:32:40.266561008+09:00","level":"INFO","msg":"stream: closing","id":"qz452p60"} +{"time":"2026-05-23T14:32:40.266576703+09:00","level":"INFO","msg":"handler: closed","stream_id":"qz452p60"} +{"time":"2026-05-23T14:32:40.266689508+09:00","level":"INFO","msg":"sender: closed","stream_id":"qz452p60"} +{"time":"2026-05-23T14:32:40.266726133+09:00","level":"INFO","msg":"stream: closed","id":"qz452p60"} diff --git a/wandb/debug.log b/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..87f2bab93a1a119919fbccca18966bebfc447aaa --- /dev/null +++ b/wandb/debug.log @@ -0,0 +1,24 @@ +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_setup.py:_flush():81] Current SDK version is 0.24.2 +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_setup.py:_flush():81] Configure stats pid to 3941155 +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260519_221333-qz452p60/logs/debug.log +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260519_221333-qz452p60/logs/debug-internal.log +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:init():844] calling init triggers +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'repo_id': '/home/seonho/insight/insight_dataset', 'dataset_root': '/home/seonho/insight/insight_dataset', 'method': 'base', 'phase': 'phase2', 'steps': 100000, 'lr': 0.0001, 'batch_size': 64, 'num_processes': 1, 'effective_batch_size': 64, 'action_dim': 8, 'state_dim': 16, 'camera_set': 'rgb', 'video_feature_keys': '', 'task_prompt_mode': 'non_guide', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_target': None, 'mgd_enabled': None, 'mgd_trainable_mode': None, 'mgd_loss_weight': None, 'mgd_backprop_backbone': None, 'cl_v2_phase': None, 'cl_v2_trainable_mode': None, 'cl_v2_loss_weight': None, 'cl_v2_fm_loss_weight': None, 'gradient_checkpointing': False, 'seed': 42, 'dataset_len': 246793, '_wandb': {}} +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:init():892] starting backend +2026-05-19 22:13:33,746 INFO MainThread:3941155 [wandb_init.py:init():895] sending inform_init request +2026-05-19 22:13:33,753 INFO MainThread:3941155 [wandb_init.py:init():903] backend started and connected +2026-05-19 22:13:33,755 INFO MainThread:3941155 [wandb_init.py:init():973] updated telemetry +2026-05-19 22:13:33,756 INFO MainThread:3941155 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-19 22:13:35,113 INFO MainThread:3941155 [wandb_init.py:init():1042] starting run threads in backend +2026-05-19 22:13:35,241 INFO MainThread:3941155 [wandb_run.py:_console_start():2529] atexit reg +2026-05-19 22:13:35,241 INFO MainThread:3941155 [wandb_run.py:_redirect():2377] redirect: wrap_raw +2026-05-19 22:13:35,241 INFO MainThread:3941155 [wandb_run.py:_redirect():2446] Wrapping output streams. +2026-05-19 22:13:35,241 INFO MainThread:3941155 [wandb_run.py:_redirect():2469] Redirects installed. +2026-05-19 22:13:35,246 INFO MainThread:3941155 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-23 13:53:31,185 INFO MainThread:3941155 [wandb_run.py:_finish():2295] finishing run RwHlabs/groot_insight_baseline_base/qz452p60 +2026-05-23 13:53:31,187 INFO MainThread:3941155 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0 +2026-05-23 13:53:31,187 INFO MainThread:3941155 [wandb_run.py:_restore():2476] restore +2026-05-23 13:53:31,187 INFO MainThread:3941155 [wandb_run.py:_restore():2482] restore done +2026-05-23 14:32:40,265 INFO MainThread:3941155 [wandb_run.py:_footer_sync_info():3871] logging synced files diff --git a/wandb/run-20260518_152847-smy9mkg4/files/config.yaml b/wandb/run-20260518_152847-smy9mkg4/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dbc4525ecebd76cf4e9e8bc96ad64923a98d85e --- /dev/null +++ b/wandb/run-20260518_152847-smy9mkg4/files/config.yaml @@ -0,0 +1,178 @@ +_wandb: + value: + cli_version: 0.24.2 + e: + ys7pyc71ohdki3edek8hpsuxl4fkwwj9: + args: + - --action_dim=8 + - --batch_size=64 + - --camera_set=rgb + - --dataset.repo_id=/home/seonho/insight/insight_dataset + - --dataset.root=/home/seonho/insight/insight_dataset + - --dataset.video_backend=torchcodec + - --log_freq=100 + - --num_workers=4 + - --output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt + - --phase=phase2 + - --policy.base_model_path=nvidia/GR00T-N1.5-3B + - --policy.lora_rank=0 + - --policy.mgd_enabled=false + - --policy.mgd_loss_weight=0.0 + - --policy.mgd_trainable_mode=dit_only + - --policy.tune_llm=false + - --policy.tune_visual=false + - --policy.type=groot_processed_mgd + - --save_freq=50000 + - --state_dim=16 + - --steps=100000 + - --task_prompt_mode=non_guide + - --wandb.enable=true + - --wandb.entity=RwHlabs + - --wandb.group=insight_phase1_dit_only + - --wandb.name=insight_phase1_dit_only_groot + - --wandb.project=groot_insight_baseline_mgd + - --wandb.tags=mgd,phase2,groot_processed_mgd,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base + codePath: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + codePathLocal: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + cpu_count: 32 + cpu_count_logical: 64 + cudaVersion: "12.2" + disk: + /: + total: "3778130485248" + used: "3526731341824" + email: dankk98@gmail.com + executable: /home/seonho/miniconda3/envs/minje/bin/python3.12 + gpu: NVIDIA A100 80GB PCIe + gpu_count: 4 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-ca1c0319-e178-a3c3-ac41-386c2723deac + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-47b6036e-4fde-e399-5515-f73eed752c6d + host: worker1 + memory: + total: "1081620840448" + os: Linux-6.8.0-101-generic-x86_64-with-glibc2.35 + program: /home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + python: CPython 3.12.12 + root: /home/seonho/insight/outputs/dit_only_phase1_ckpt + startedAt: "2026-05-18T06:28:47.096325Z" + writerId: ys7pyc71ohdki3edek8hpsuxl4fkwwj9 + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "3": + - 13 + - 15 + - 16 + - 62 + "4": 3.12.12 + "5": 0.24.2 + "6": 5.3.0 + "12": 0.24.2 + "13": linux-x86_64 +action_dim: + value: 8 +batch_size: + value: 64 +camera_set: + value: rgb +cl_v2_fm_loss_weight: + value: null +cl_v2_loss_weight: + value: null +cl_v2_phase: + value: null +cl_v2_trainable_mode: + value: null +dataset_len: + value: 246793 +dataset_root: + value: /home/seonho/insight/insight_dataset +effective_batch_size: + value: 64 +gradient_checkpointing: + value: false +lora_alpha: + value: 16 +lora_dropout: + value: 0.1 +lora_rank: + value: 0 +lora_target: + value: llm +lr: + value: 0.0001 +method: + value: mgd +mgd_backprop_backbone: + value: true +mgd_enabled: + value: false +mgd_loss_weight: + value: 0 +mgd_trainable_mode: + value: dit_only +num_processes: + value: 1 +phase: + value: phase2 +repo_id: + value: /home/seonho/insight/insight_dataset +seed: + value: 42 +state_dim: + value: 16 +steps: + value: 100000 +task_prompt_mode: + value: non_guide +tune_diffusion_model: + value: true +tune_llm: + value: false +tune_projector: + value: true +tune_visual: + value: false +video_feature_keys: + value: "" diff --git a/wandb/run-20260518_152847-smy9mkg4/files/output.log b/wandb/run-20260518_152847-smy9mkg4/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9e0e68e7b595fc5ed586872ff3a7cc34dfa7ee10 --- /dev/null +++ b/wandb/run-20260518_152847-smy9mkg4/files/output.log @@ -0,0 +1,103 @@ +Traceback (most recent call last): + File "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", line 1735, in + main() + File "/home/seonho/clvla/lerobot_cl/src/lerobot/configs/parser.py", line 233, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", line 1621, in main + raw_batch = prepare_raw_batch(next(data_stream), cfg, preset, task_desc_map) + ^^^^^^^^^^^^^^^^^ + File "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", line 1453, in infinite_dataloader + yield from dataloader + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/accelerate/data_loader.py", line 577, in __iter__ + current_batch = next(dataloader_iter) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/dataloader.py", line 733, in __next__ + data = self._next_data() + ^^^^^^^^^^^^^^^^^ + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/dataloader.py", line 1515, in _next_data + return self._process_data(data, worker_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/dataloader.py", line 1550, in _process_data + data.reraise() + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/_utils.py", line 750, in reraise + raise exception +RuntimeError: Caught RuntimeError in DataLoader worker process 0. +Original Traceback (most recent call last): + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/_utils/worker.py", line 349, in _worker_loop + data = fetcher.fetch(index) # type: ignore[possibly-undefined] + ^^^^^^^^^^^^^^^^^^^^ + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/_utils/fetch.py", line 52, in fetch + data = [self.dataset[idx] for idx in possibly_batched_index] + ~~~~~~~~~~~~^^^^^ + File "/home/seonho/clvla/lerobot_cl/src/lerobot/datasets/lerobot_dataset.py", line 627, in __getitem__ + video_frames = self._query_videos(query_timestamps, ep_idx) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/seonho/clvla/lerobot_cl/src/lerobot/datasets/lerobot_dataset.py", line 590, in _query_videos + frames = decode_video_frames(video_path, shifted_query_ts, self.tolerance_s, self.video_backend) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/seonho/clvla/lerobot_cl/src/lerobot/datasets/video_utils.py", line 152, in decode_video_frames + return decode_video_frames_torchcodec(video_path, timestamps, tolerance_s) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/seonho/clvla/lerobot_cl/src/lerobot/datasets/video_utils.py", line 420, in decode_video_frames_torchcodec + frames_batch = decoder.get_frames_at(indices=frame_indices) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torchcodec/decoders/_video_decoder.py", line 227, in get_frames_at + data, pts_seconds, duration_seconds = core.get_frames_at_indices( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/_ops.py", line 756, in __call__ + return self._op(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^ +RuntimeError: Could not push packet to decoder: Invalid data found when processing input + +[rank0]: Traceback (most recent call last): +[rank0]: File "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", line 1735, in +[rank0]: main() +[rank0]: File "/home/seonho/clvla/lerobot_cl/src/lerobot/configs/parser.py", line 233, in wrapper_inner +[rank0]: response = fn(cfg, *args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", line 1621, in main +[rank0]: raw_batch = prepare_raw_batch(next(data_stream), cfg, preset, task_desc_map) +[rank0]: ^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", line 1453, in infinite_dataloader +[rank0]: yield from dataloader +[rank0]: File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/accelerate/data_loader.py", line 577, in __iter__ +[rank0]: current_batch = next(dataloader_iter) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/dataloader.py", line 733, in __next__ +[rank0]: data = self._next_data() +[rank0]: ^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/dataloader.py", line 1515, in _next_data +[rank0]: return self._process_data(data, worker_id) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/dataloader.py", line 1550, in _process_data +[rank0]: data.reraise() +[rank0]: File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/_utils.py", line 750, in reraise +[rank0]: raise exception +[rank0]: RuntimeError: Caught RuntimeError in DataLoader worker process 0. +[rank0]: Original Traceback (most recent call last): +[rank0]: File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/_utils/worker.py", line 349, in _worker_loop +[rank0]: data = fetcher.fetch(index) # type: ignore[possibly-undefined] +[rank0]: ^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/utils/data/_utils/fetch.py", line 52, in fetch +[rank0]: data = [self.dataset[idx] for idx in possibly_batched_index] +[rank0]: ~~~~~~~~~~~~^^^^^ +[rank0]: File "/home/seonho/clvla/lerobot_cl/src/lerobot/datasets/lerobot_dataset.py", line 627, in __getitem__ +[rank0]: video_frames = self._query_videos(query_timestamps, ep_idx) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/clvla/lerobot_cl/src/lerobot/datasets/lerobot_dataset.py", line 590, in _query_videos +[rank0]: frames = decode_video_frames(video_path, shifted_query_ts, self.tolerance_s, self.video_backend) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/clvla/lerobot_cl/src/lerobot/datasets/video_utils.py", line 152, in decode_video_frames +[rank0]: return decode_video_frames_torchcodec(video_path, timestamps, tolerance_s) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/clvla/lerobot_cl/src/lerobot/datasets/video_utils.py", line 420, in decode_video_frames_torchcodec +[rank0]: frames_batch = decoder.get_frames_at(indices=frame_indices) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torchcodec/decoders/_video_decoder.py", line 227, in get_frames_at +[rank0]: data, pts_seconds, duration_seconds = core.get_frames_at_indices( +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/torch/_ops.py", line 756, in __call__ +[rank0]: return self._op(*args, **kwargs) +[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank0]: RuntimeError: Could not push packet to decoder: Invalid data found when processing input diff --git a/wandb/run-20260518_152847-smy9mkg4/files/requirements.txt b/wandb/run-20260518_152847-smy9mkg4/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8d89644db6c7aa4d72b923c55c62ae8132609a93 --- /dev/null +++ b/wandb/run-20260518_152847-smy9mkg4/files/requirements.txt @@ -0,0 +1,209 @@ +attrs==25.4.0 +kiwisolver==1.5.0 +setuptools==80.10.2 +transformers==5.3.0 +pynndescent==0.6.0 +filelock==3.25.2 +numba==0.61.2 +gymnasium==1.2.3 +shellingham==1.5.4 +dill==0.3.8 +ninja==1.13.0 +hidapi==0.15.0 +scikit-image==0.26.0 +orderly-set==5.5.0 +mypy_extensions==1.1.0 +stringzilla==4.6.1 +tianshou==0.4.10 +numpy==2.2.5 +xxhash==3.6.0 +protobuf==3.19.6 +numpydantic==1.6.7 +pyserial==3.5 +omegaconf==2.3.0 +Farama-Notifications==0.0.4 +nvidia-nccl-cu12==2.26.2 +plotly==6.6.0 +platformdirs==4.9.4 +nvidia-cufft-cu12==11.3.0.4 +eval_type_backport==0.3.1 +packaging==25.0 +typing-inspect==0.9.0 +opencv-python-headless==4.12.0.88 +cmeel-assimp==6.0.2 +lerobot==0.5.1 +pyyaml-include==1.4.1 +urllib3==2.6.3 +lxml==6.1.0 +peft==0.18.1 +virtualenv==21.2.4 +cmeel==0.59.0 +pre_commit==4.6.0 +importlib_metadata==8.7.1 +click==8.3.1 +accelerate==1.13.0 +multiprocess==0.70.16 +distlib==0.4.0 +websockets==16.0 +albumentations==1.4.18 +tokenizers==0.22.2 +nvidia-cusparselt-cu12==0.6.3 +certifi==2026.2.25 +annotated-doc==0.0.4 +triton==3.3.1 +libpinocchio==3.9.0 +mpmath==1.3.0 +decord==0.6.0 +matplotlib==3.10.8 +llvmlite==0.44.0 +python-xlib==0.33 +regex==2026.2.28 +colored==2.3.2 +charset-normalizer==3.4.5 +mdurl==0.1.2 +GitPython==3.1.46 +gym==0.26.2 +av==15.1.0 +markdown-it-py==4.0.0 +glfw==2.10.0 +nvidia-cufile-cu12==1.11.1.6 +PyYAML==6.0.3 +python-dateutil==2.9.0.post0 +cmeel-tinyxml2==10.0.0 +blinker==1.9.0 +opencv-python==4.13.0.92 +cmake==4.1.3 +gym-notices==0.1.0 +typing-inspection==0.4.2 +timm==1.0.25 +torchcodec==0.5 +cmeel-octomap==1.10.0 +umap-learn==0.5.11 +MarkupSafe==3.0.3 +pluggy==1.6.0 +smmap==5.0.3 +fsspec==2025.3.0 +evdev==1.9.3 +python-discovery==1.2.2 +hf-xet==1.5.0 +scikit-learn==1.8.0 +h5py==3.16.0 +cmeel-console-bridge==1.0.2.3 +narwhals==2.18.1 +msgpack==1.1.2 +flash_attn==2.8.3 +ansitable==0.11.5 +pyarrow==23.0.1 +h11==0.16.0 +mergedeep==1.3.4 +tqdm==4.67.3 +vla-eval==0.1.1.dev27+g195d46338 +nvidia-nvjitlink-cu12==12.6.85 +cmeel-qhull==8.0.2.1 +libcoal==3.0.2 +cmeel-boost==1.89.0 +rtb-data==1.0.1 +mujoco==3.3.1 +nvidia-cuda-nvrtc-cu12==12.6.77 +PyOpenGL==3.1.10 +draccus==0.10.0 +pip==26.0.1 +threadpoolctl==3.6.0 +annotated-types==0.7.0 +anyio==4.12.1 +imageio-ffmpeg==0.6.0 +qpsolvers==4.11.0 +wrapt==2.1.2 +networkx==3.6.1 +contourpy==1.3.3 +Werkzeug==3.1.8 +tensorboard-data-server==0.7.2 +gitdb==4.0.12 +nvidia-cusparse-cu12==12.5.4.2 +typer==0.24.1 +toml==0.10.2 +tensorboard==2.20.0 +rerun-sdk==0.26.2 +diffusers==0.35.2 +albucore==0.0.17 +wandb==0.24.2 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-runtime-cu12==12.6.77 +cycler==0.12.1 +coal==3.0.2 +pydantic_core==2.27.2 +torch==2.7.1 +nvidia-cuda-cupti-cu12==12.6.80 +Pygments==2.19.2 +pytest==9.0.3 +ImageIO==2.37.3 +sentry-sdk==2.54.0 +antlr4-python3-runtime==4.9.3 +sympy==1.14.0 +safetensors==0.7.0 +cmeel-zlib==1.3.1 +einops==0.8.2 +spatialgeometry==1.1.0 +pygame==2.6.1 +absl-py==2.4.0 +pandas==3.0.1 +httpx==0.28.1 +robosuite==1.5.2 +fonttools==4.62.1 +progress==1.6.1 +nvidia-cusolver-cu12==11.7.1.2 +aiohttp==3.13.3 +frozenlist==1.8.0 +deepdiff==8.6.1 +yarl==1.23.0 +nvidia-cudnn-cu12==9.5.1.17 +torchvision==0.22.1 +idna==3.11 +lazy-loader==0.5 +termcolor==3.3.0 +nvidia-curand-cu12==10.3.7.77 +multidict==6.7.1 +Markdown==3.10.2 +pynput==1.8.1 +propcache==0.4.1 +zipp==3.23.0 +nvidia-nvtx-cu12==12.6.77 +Flask==3.1.3 +httpcore==1.0.9 +etils==1.14.0 +psutil==7.2.2 +pillow==12.1.1 +nodeenv==1.10.0 +pin==3.9.0 +quadprog==0.1.13 +eigenpy==3.12.0 +wheel==0.46.3 +simsimd==6.5.16 +itsdangerous==2.2.0 +aiohappyeyeballs==2.6.1 +cfgv==3.5.0 +cloudpickle==3.1.2 +iniconfig==2.3.0 +requests==2.32.5 +tifffile==2026.5.2 +joblib==1.5.3 +identify==2.6.19 +huggingface_hub==1.14.0 +datasets==4.8.4 +typing_extensions==4.15.0 +pydantic==2.10.6 +jsonlines==4.0.0 +swift-sim==1.1.0 +lazyregistry==0.4.0 +robocasa==1.0.0 +pgraph-python==0.6.3 +Jinja2==3.1.6 +aiosignal==1.4.0 +dm-tree==0.1.9 +six==1.17.0 +grpcio==1.80.0 +rich==14.3.3 +scipy==1.15.3 +cmeel-urdfdom==4.0.1 +pyparsing==3.3.2 +lerobot==0.5.1 diff --git a/wandb/run-20260518_152847-smy9mkg4/files/wandb-metadata.json b/wandb/run-20260518_152847-smy9mkg4/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7255889be90d2b3e62d78e38251ba63f2136295a --- /dev/null +++ b/wandb/run-20260518_152847-smy9mkg4/files/wandb-metadata.json @@ -0,0 +1,87 @@ +{ + "os": "Linux-6.8.0-101-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.12", + "startedAt": "2026-05-18T06:28:47.096325Z", + "args": [ + "--action_dim=8", + "--batch_size=64", + "--camera_set=rgb", + "--dataset.repo_id=/home/seonho/insight/insight_dataset", + "--dataset.root=/home/seonho/insight/insight_dataset", + "--dataset.video_backend=torchcodec", + "--log_freq=100", + "--num_workers=4", + "--output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "--phase=phase2", + "--policy.base_model_path=nvidia/GR00T-N1.5-3B", + "--policy.lora_rank=0", + "--policy.mgd_enabled=false", + "--policy.mgd_loss_weight=0.0", + "--policy.mgd_trainable_mode=dit_only", + "--policy.tune_llm=false", + "--policy.tune_visual=false", + "--policy.type=groot_processed_mgd", + "--save_freq=50000", + "--state_dim=16", + "--steps=100000", + "--task_prompt_mode=non_guide", + "--wandb.enable=true", + "--wandb.entity=RwHlabs", + "--wandb.group=insight_phase1_dit_only", + "--wandb.name=insight_phase1_dit_only_groot", + "--wandb.project=groot_insight_baseline_mgd", + "--wandb.tags=mgd,phase2,groot_processed_mgd,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + ], + "program": "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePath": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePathLocal": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "email": "dankk98@gmail.com", + "root": "/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "host": "worker1", + "executable": "/home/seonho/miniconda3/envs/minje/bin/python3.12", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 4, + "disk": { + "/": { + "total": "3778130485248", + "used": "3526731341824" + } + }, + "memory": { + "total": "1081620840448" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-ca1c0319-e178-a3c3-ac41-386c2723deac" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-47b6036e-4fde-e399-5515-f73eed752c6d" + } + ], + "cudaVersion": "12.2", + "writerId": "ys7pyc71ohdki3edek8hpsuxl4fkwwj9" +} \ No newline at end of file diff --git a/wandb/run-20260518_152847-smy9mkg4/files/wandb-summary.json b/wandb/run-20260518_152847-smy9mkg4/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..beb2616282aececb70c6428258394a86bcdd3144 --- /dev/null +++ b/wandb/run-20260518_152847-smy9mkg4/files/wandb-summary.json @@ -0,0 +1 @@ +{"params/whole_total":2427681217,"params/whole_trainable":840593408,"_wandb":{"runtime":1},"_runtime":1} \ No newline at end of file diff --git a/wandb/run-20260518_152847-smy9mkg4/logs/debug-core.log b/wandb/run-20260518_152847-smy9mkg4/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..df1a1f8c722e61ff8eabbc0ff125d1a5d070376d --- /dev/null +++ b/wandb/run-20260518_152847-smy9mkg4/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2026-05-18T15:28:47.134214528+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp6v9hfjeh/port-3560821.txt","pid":3560821,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-18T15:28:47.134775705+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3560821} +{"time":"2026-05-18T15:28:47.134766733+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3560821-3566957-720346231/socket","Net":"unix"}} +{"time":"2026-05-18T15:28:47.322025048+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-18T15:28:47.336551071+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"smy9mkg4","id":"1(@)"} +{"time":"2026-05-18T15:28:47.715349144+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"smy9mkg4","id":"1(@)"} +{"time":"2026-05-18T15:28:50.380217573+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-18T15:28:50.380327768+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-18T15:28:50.380307348+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-18T15:28:50.380576767+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-18T15:28:50.38058255+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3560821-3566957-720346231/socket","Net":"unix"}} +{"time":"2026-05-18T15:28:52.389072306+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-18T15:28:52.389105339+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-18T15:28:52.389126226+09:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260518_152847-smy9mkg4/logs/debug-internal.log b/wandb/run-20260518_152847-smy9mkg4/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1f29250cc54c5079fa9fbb8ca52cd48a7c0e2162 --- /dev/null +++ b/wandb/run-20260518_152847-smy9mkg4/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-18T15:28:47.336692013+09:00","level":"INFO","msg":"stream: starting","core version":"0.24.2"} +{"time":"2026-05-18T15:28:47.71499011+09:00","level":"INFO","msg":"stream: created new stream","id":"smy9mkg4"} +{"time":"2026-05-18T15:28:47.715138854+09:00","level":"INFO","msg":"handler: started","stream_id":"smy9mkg4"} +{"time":"2026-05-18T15:28:47.715332098+09:00","level":"INFO","msg":"stream: started","id":"smy9mkg4"} +{"time":"2026-05-18T15:28:47.715342118+09:00","level":"INFO","msg":"writer: started","stream_id":"smy9mkg4"} +{"time":"2026-05-18T15:28:47.715343845+09:00","level":"INFO","msg":"sender: started","stream_id":"smy9mkg4"} +{"time":"2026-05-18T15:28:50.380317028+09:00","level":"INFO","msg":"stream: closing","id":"smy9mkg4"} +{"time":"2026-05-18T15:28:52.036130686+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-18T15:28:52.388015098+09:00","level":"INFO","msg":"handler: closed","stream_id":"smy9mkg4"} +{"time":"2026-05-18T15:28:52.388154428+09:00","level":"INFO","msg":"sender: closed","stream_id":"smy9mkg4"} +{"time":"2026-05-18T15:28:52.388166789+09:00","level":"INFO","msg":"stream: closed","id":"smy9mkg4"} diff --git a/wandb/run-20260518_152847-smy9mkg4/logs/debug.log b/wandb/run-20260518_152847-smy9mkg4/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f0ef909bb6d83994aba4f403f8f3ad5d66e251d7 --- /dev/null +++ b/wandb/run-20260518_152847-smy9mkg4/logs/debug.log @@ -0,0 +1,34 @@ +2026-05-18 15:28:47,098 INFO MainThread:3560821 [wandb_setup.py:_flush():81] Current SDK version is 0.24.2 +2026-05-18 15:28:47,098 INFO MainThread:3560821 [wandb_setup.py:_flush():81] Configure stats pid to 3560821 +2026-05-18 15:28:47,098 INFO MainThread:3560821 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-18 15:28:47,098 INFO MainThread:3560821 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260518_152847-smy9mkg4/logs/debug.log +2026-05-18 15:28:47,098 INFO MainThread:3560821 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260518_152847-smy9mkg4/logs/debug-internal.log +2026-05-18 15:28:47,098 INFO MainThread:3560821 [wandb_init.py:init():844] calling init triggers +2026-05-18 15:28:47,098 INFO MainThread:3560821 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'repo_id': '/home/seonho/insight/insight_dataset', 'dataset_root': '/home/seonho/insight/insight_dataset', 'method': 'mgd', 'phase': 'phase2', 'steps': 100000, 'lr': 0.0001, 'batch_size': 64, 'num_processes': 1, 'effective_batch_size': 64, 'action_dim': 8, 'state_dim': 16, 'camera_set': 'rgb', 'video_feature_keys': '', 'task_prompt_mode': 'non_guide', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_target': 'llm', 'mgd_enabled': False, 'mgd_trainable_mode': 'dit_only', 'mgd_loss_weight': 0.0, 'mgd_backprop_backbone': True, 'cl_v2_phase': None, 'cl_v2_trainable_mode': None, 'cl_v2_loss_weight': None, 'cl_v2_fm_loss_weight': None, 'gradient_checkpointing': False, 'seed': 42, 'dataset_len': 246793, '_wandb': {}} +2026-05-18 15:28:47,098 INFO MainThread:3560821 [wandb_init.py:init():892] starting backend +2026-05-18 15:28:47,322 INFO MainThread:3560821 [wandb_init.py:init():895] sending inform_init request +2026-05-18 15:28:47,332 INFO MainThread:3560821 [wandb_init.py:init():903] backend started and connected +2026-05-18 15:28:47,335 INFO MainThread:3560821 [wandb_init.py:init():973] updated telemetry +2026-05-18 15:28:47,336 INFO MainThread:3560821 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-18 15:28:48,978 INFO MainThread:3560821 [wandb_init.py:init():1042] starting run threads in backend +2026-05-18 15:28:49,171 INFO MainThread:3560821 [wandb_run.py:_console_start():2529] atexit reg +2026-05-18 15:28:49,171 INFO MainThread:3560821 [wandb_run.py:_redirect():2377] redirect: wrap_raw +2026-05-18 15:28:49,171 INFO MainThread:3560821 [wandb_run.py:_redirect():2446] Wrapping output streams. +2026-05-18 15:28:49,171 INFO MainThread:3560821 [wandb_run.py:_redirect():2469] Redirects installed. +2026-05-18 15:28:49,176 INFO MainThread:3560821 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-18 15:28:50,380 INFO wandb-AsyncioManager-main:3560821 [service_client.py:_forward_responses():94] Reached EOF. +2026-05-18 15:28:50,381 INFO wandb-AsyncioManager-main:3560821 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles. +2026-05-18 15:28:52,576 ERROR wandb-AsyncioManager-main:3560821 [asyncio_manager.py:fn_wrap_exceptions():183] Uncaught exception in run_soon callback. +Traceback (most recent call last): + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/wandb/sdk/lib/asyncio_manager.py", line 181, in fn_wrap_exceptions + await fn() + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 38, in publish + await self._send_server_request(request) + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/site-packages/wandb/sdk/lib/service/service_client.py", line 64, in _send_server_request + await self._writer.drain() + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/asyncio/streams.py", line 392, in drain + await self._protocol._drain_helper() + File "/home/seonho/miniconda3/envs/minje/lib/python3.12/asyncio/streams.py", line 166, in _drain_helper + raise ConnectionResetError('Connection lost') +ConnectionResetError: Connection lost diff --git a/wandb/run-20260518_152847-smy9mkg4/run-smy9mkg4.wandb b/wandb/run-20260518_152847-smy9mkg4/run-smy9mkg4.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c5122eb9017821593734fb10039e14c827c8e199 Binary files /dev/null and b/wandb/run-20260518_152847-smy9mkg4/run-smy9mkg4.wandb differ diff --git a/wandb/run-20260518_153314-yn3lgfkj/files/config.yaml b/wandb/run-20260518_153314-yn3lgfkj/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2ca05bc8e6b1e4217d6227c94fb1860d2bc57ec --- /dev/null +++ b/wandb/run-20260518_153314-yn3lgfkj/files/config.yaml @@ -0,0 +1,178 @@ +_wandb: + value: + cli_version: 0.24.2 + e: + ddeaytz4ga2t8xmz3osihmmadhyuvdau: + args: + - --action_dim=8 + - --batch_size=64 + - --camera_set=rgb + - --dataset.repo_id=/home/seonho/insight/insight_dataset + - --dataset.root=/home/seonho/insight/insight_dataset + - --dataset.video_backend=pyav + - --log_freq=100 + - --num_workers=4 + - --output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt + - --phase=phase2 + - --policy.base_model_path=nvidia/GR00T-N1.5-3B + - --policy.lora_rank=0 + - --policy.mgd_enabled=false + - --policy.mgd_loss_weight=0.0 + - --policy.mgd_trainable_mode=dit_only + - --policy.tune_llm=false + - --policy.tune_visual=false + - --policy.type=groot_processed_mgd + - --save_freq=50000 + - --state_dim=16 + - --steps=100000 + - --task_prompt_mode=non_guide + - --wandb.enable=true + - --wandb.entity=RwHlabs + - --wandb.group=insight_phase1_dit_only + - --wandb.name=insight_phase1_dit_only_groot + - --wandb.project=groot_insight_baseline_mgd + - --wandb.tags=mgd,phase2,groot_processed_mgd,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base + codePath: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + codePathLocal: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + cpu_count: 32 + cpu_count_logical: 64 + cudaVersion: "12.2" + disk: + /: + total: "3778130485248" + used: "3442052386816" + email: dankk98@gmail.com + executable: /home/seonho/miniconda3/envs/minje/bin/python3.12 + gpu: NVIDIA A100 80GB PCIe + gpu_count: 4 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-ca1c0319-e178-a3c3-ac41-386c2723deac + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-47b6036e-4fde-e399-5515-f73eed752c6d + host: worker1 + memory: + total: "1081620840448" + os: Linux-6.8.0-101-generic-x86_64-with-glibc2.35 + program: /home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + python: CPython 3.12.12 + root: /home/seonho/insight/outputs/dit_only_phase1_ckpt + startedAt: "2026-05-18T06:33:14.327419Z" + writerId: ddeaytz4ga2t8xmz3osihmmadhyuvdau + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "3": + - 13 + - 15 + - 16 + - 62 + "4": 3.12.12 + "5": 0.24.2 + "6": 5.3.0 + "12": 0.24.2 + "13": linux-x86_64 +action_dim: + value: 8 +batch_size: + value: 64 +camera_set: + value: rgb +cl_v2_fm_loss_weight: + value: null +cl_v2_loss_weight: + value: null +cl_v2_phase: + value: null +cl_v2_trainable_mode: + value: null +dataset_len: + value: 246793 +dataset_root: + value: /home/seonho/insight/insight_dataset +effective_batch_size: + value: 64 +gradient_checkpointing: + value: false +lora_alpha: + value: 16 +lora_dropout: + value: 0.1 +lora_rank: + value: 0 +lora_target: + value: llm +lr: + value: 0.0001 +method: + value: mgd +mgd_backprop_backbone: + value: true +mgd_enabled: + value: false +mgd_loss_weight: + value: 0 +mgd_trainable_mode: + value: dit_only +num_processes: + value: 1 +phase: + value: phase2 +repo_id: + value: /home/seonho/insight/insight_dataset +seed: + value: 42 +state_dim: + value: 16 +steps: + value: 100000 +task_prompt_mode: + value: non_guide +tune_diffusion_model: + value: true +tune_llm: + value: false +tune_projector: + value: true +tune_visual: + value: false +video_feature_keys: + value: "" diff --git a/wandb/run-20260518_153314-yn3lgfkj/files/output.log b/wandb/run-20260518_153314-yn3lgfkj/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ccd996e61ad451f06ada8fe510b60b6829ead6ed --- /dev/null +++ b/wandb/run-20260518_153314-yn3lgfkj/files/output.log @@ -0,0 +1,3 @@ +`use_fast` is set to `True` but the image processor class does not have a fast version. Falling back to the slow version. +The tokenizer you are loading from '/home/seonho/.cache/huggingface/lerobot/lerobot/eagle2hg-processor-groot-n1p5' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue. +The tokenizer you are loading from '/home/seonho/.cache/huggingface/lerobot/lerobot/eagle2hg-processor-groot-n1p5' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue. diff --git a/wandb/run-20260518_153314-yn3lgfkj/files/requirements.txt b/wandb/run-20260518_153314-yn3lgfkj/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8d89644db6c7aa4d72b923c55c62ae8132609a93 --- /dev/null +++ b/wandb/run-20260518_153314-yn3lgfkj/files/requirements.txt @@ -0,0 +1,209 @@ +attrs==25.4.0 +kiwisolver==1.5.0 +setuptools==80.10.2 +transformers==5.3.0 +pynndescent==0.6.0 +filelock==3.25.2 +numba==0.61.2 +gymnasium==1.2.3 +shellingham==1.5.4 +dill==0.3.8 +ninja==1.13.0 +hidapi==0.15.0 +scikit-image==0.26.0 +orderly-set==5.5.0 +mypy_extensions==1.1.0 +stringzilla==4.6.1 +tianshou==0.4.10 +numpy==2.2.5 +xxhash==3.6.0 +protobuf==3.19.6 +numpydantic==1.6.7 +pyserial==3.5 +omegaconf==2.3.0 +Farama-Notifications==0.0.4 +nvidia-nccl-cu12==2.26.2 +plotly==6.6.0 +platformdirs==4.9.4 +nvidia-cufft-cu12==11.3.0.4 +eval_type_backport==0.3.1 +packaging==25.0 +typing-inspect==0.9.0 +opencv-python-headless==4.12.0.88 +cmeel-assimp==6.0.2 +lerobot==0.5.1 +pyyaml-include==1.4.1 +urllib3==2.6.3 +lxml==6.1.0 +peft==0.18.1 +virtualenv==21.2.4 +cmeel==0.59.0 +pre_commit==4.6.0 +importlib_metadata==8.7.1 +click==8.3.1 +accelerate==1.13.0 +multiprocess==0.70.16 +distlib==0.4.0 +websockets==16.0 +albumentations==1.4.18 +tokenizers==0.22.2 +nvidia-cusparselt-cu12==0.6.3 +certifi==2026.2.25 +annotated-doc==0.0.4 +triton==3.3.1 +libpinocchio==3.9.0 +mpmath==1.3.0 +decord==0.6.0 +matplotlib==3.10.8 +llvmlite==0.44.0 +python-xlib==0.33 +regex==2026.2.28 +colored==2.3.2 +charset-normalizer==3.4.5 +mdurl==0.1.2 +GitPython==3.1.46 +gym==0.26.2 +av==15.1.0 +markdown-it-py==4.0.0 +glfw==2.10.0 +nvidia-cufile-cu12==1.11.1.6 +PyYAML==6.0.3 +python-dateutil==2.9.0.post0 +cmeel-tinyxml2==10.0.0 +blinker==1.9.0 +opencv-python==4.13.0.92 +cmake==4.1.3 +gym-notices==0.1.0 +typing-inspection==0.4.2 +timm==1.0.25 +torchcodec==0.5 +cmeel-octomap==1.10.0 +umap-learn==0.5.11 +MarkupSafe==3.0.3 +pluggy==1.6.0 +smmap==5.0.3 +fsspec==2025.3.0 +evdev==1.9.3 +python-discovery==1.2.2 +hf-xet==1.5.0 +scikit-learn==1.8.0 +h5py==3.16.0 +cmeel-console-bridge==1.0.2.3 +narwhals==2.18.1 +msgpack==1.1.2 +flash_attn==2.8.3 +ansitable==0.11.5 +pyarrow==23.0.1 +h11==0.16.0 +mergedeep==1.3.4 +tqdm==4.67.3 +vla-eval==0.1.1.dev27+g195d46338 +nvidia-nvjitlink-cu12==12.6.85 +cmeel-qhull==8.0.2.1 +libcoal==3.0.2 +cmeel-boost==1.89.0 +rtb-data==1.0.1 +mujoco==3.3.1 +nvidia-cuda-nvrtc-cu12==12.6.77 +PyOpenGL==3.1.10 +draccus==0.10.0 +pip==26.0.1 +threadpoolctl==3.6.0 +annotated-types==0.7.0 +anyio==4.12.1 +imageio-ffmpeg==0.6.0 +qpsolvers==4.11.0 +wrapt==2.1.2 +networkx==3.6.1 +contourpy==1.3.3 +Werkzeug==3.1.8 +tensorboard-data-server==0.7.2 +gitdb==4.0.12 +nvidia-cusparse-cu12==12.5.4.2 +typer==0.24.1 +toml==0.10.2 +tensorboard==2.20.0 +rerun-sdk==0.26.2 +diffusers==0.35.2 +albucore==0.0.17 +wandb==0.24.2 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-runtime-cu12==12.6.77 +cycler==0.12.1 +coal==3.0.2 +pydantic_core==2.27.2 +torch==2.7.1 +nvidia-cuda-cupti-cu12==12.6.80 +Pygments==2.19.2 +pytest==9.0.3 +ImageIO==2.37.3 +sentry-sdk==2.54.0 +antlr4-python3-runtime==4.9.3 +sympy==1.14.0 +safetensors==0.7.0 +cmeel-zlib==1.3.1 +einops==0.8.2 +spatialgeometry==1.1.0 +pygame==2.6.1 +absl-py==2.4.0 +pandas==3.0.1 +httpx==0.28.1 +robosuite==1.5.2 +fonttools==4.62.1 +progress==1.6.1 +nvidia-cusolver-cu12==11.7.1.2 +aiohttp==3.13.3 +frozenlist==1.8.0 +deepdiff==8.6.1 +yarl==1.23.0 +nvidia-cudnn-cu12==9.5.1.17 +torchvision==0.22.1 +idna==3.11 +lazy-loader==0.5 +termcolor==3.3.0 +nvidia-curand-cu12==10.3.7.77 +multidict==6.7.1 +Markdown==3.10.2 +pynput==1.8.1 +propcache==0.4.1 +zipp==3.23.0 +nvidia-nvtx-cu12==12.6.77 +Flask==3.1.3 +httpcore==1.0.9 +etils==1.14.0 +psutil==7.2.2 +pillow==12.1.1 +nodeenv==1.10.0 +pin==3.9.0 +quadprog==0.1.13 +eigenpy==3.12.0 +wheel==0.46.3 +simsimd==6.5.16 +itsdangerous==2.2.0 +aiohappyeyeballs==2.6.1 +cfgv==3.5.0 +cloudpickle==3.1.2 +iniconfig==2.3.0 +requests==2.32.5 +tifffile==2026.5.2 +joblib==1.5.3 +identify==2.6.19 +huggingface_hub==1.14.0 +datasets==4.8.4 +typing_extensions==4.15.0 +pydantic==2.10.6 +jsonlines==4.0.0 +swift-sim==1.1.0 +lazyregistry==0.4.0 +robocasa==1.0.0 +pgraph-python==0.6.3 +Jinja2==3.1.6 +aiosignal==1.4.0 +dm-tree==0.1.9 +six==1.17.0 +grpcio==1.80.0 +rich==14.3.3 +scipy==1.15.3 +cmeel-urdfdom==4.0.1 +pyparsing==3.3.2 +lerobot==0.5.1 diff --git a/wandb/run-20260518_153314-yn3lgfkj/files/wandb-metadata.json b/wandb/run-20260518_153314-yn3lgfkj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c8ac73618ded99544f60ee2a002bd1d5cbd5b5bd --- /dev/null +++ b/wandb/run-20260518_153314-yn3lgfkj/files/wandb-metadata.json @@ -0,0 +1,87 @@ +{ + "os": "Linux-6.8.0-101-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.12", + "startedAt": "2026-05-18T06:33:14.327419Z", + "args": [ + "--action_dim=8", + "--batch_size=64", + "--camera_set=rgb", + "--dataset.repo_id=/home/seonho/insight/insight_dataset", + "--dataset.root=/home/seonho/insight/insight_dataset", + "--dataset.video_backend=pyav", + "--log_freq=100", + "--num_workers=4", + "--output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "--phase=phase2", + "--policy.base_model_path=nvidia/GR00T-N1.5-3B", + "--policy.lora_rank=0", + "--policy.mgd_enabled=false", + "--policy.mgd_loss_weight=0.0", + "--policy.mgd_trainable_mode=dit_only", + "--policy.tune_llm=false", + "--policy.tune_visual=false", + "--policy.type=groot_processed_mgd", + "--save_freq=50000", + "--state_dim=16", + "--steps=100000", + "--task_prompt_mode=non_guide", + "--wandb.enable=true", + "--wandb.entity=RwHlabs", + "--wandb.group=insight_phase1_dit_only", + "--wandb.name=insight_phase1_dit_only_groot", + "--wandb.project=groot_insight_baseline_mgd", + "--wandb.tags=mgd,phase2,groot_processed_mgd,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + ], + "program": "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePath": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePathLocal": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "email": "dankk98@gmail.com", + "root": "/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "host": "worker1", + "executable": "/home/seonho/miniconda3/envs/minje/bin/python3.12", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 4, + "disk": { + "/": { + "total": "3778130485248", + "used": "3442052386816" + } + }, + "memory": { + "total": "1081620840448" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-ca1c0319-e178-a3c3-ac41-386c2723deac" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-47b6036e-4fde-e399-5515-f73eed752c6d" + } + ], + "cudaVersion": "12.2", + "writerId": "ddeaytz4ga2t8xmz3osihmmadhyuvdau" +} \ No newline at end of file diff --git a/wandb/run-20260518_153314-yn3lgfkj/files/wandb-summary.json b/wandb/run-20260518_153314-yn3lgfkj/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..cbd4427393093dddac5c9437731e350f442565e5 --- /dev/null +++ b/wandb/run-20260518_153314-yn3lgfkj/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":152,"params/whole_total":2427681217,"params/whole_trainable":840593408,"_wandb":{"runtime":152}} \ No newline at end of file diff --git a/wandb/run-20260518_153314-yn3lgfkj/logs/debug-core.log b/wandb/run-20260518_153314-yn3lgfkj/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..3f617bac2da9afe9f8f82466122b80d0545eedd6 --- /dev/null +++ b/wandb/run-20260518_153314-yn3lgfkj/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2026-05-18T15:33:14.368029784+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpt29ma1tu/port-3572879.txt","pid":3572879,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-18T15:33:14.368891132+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3572879} +{"time":"2026-05-18T15:33:14.368907866+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3572879-3579183-3991850837/socket","Net":"unix"}} +{"time":"2026-05-18T15:33:14.552175752+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-18T15:33:14.568692696+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"yn3lgfkj","id":"1(@)"} +{"time":"2026-05-18T15:33:14.955750856+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"yn3lgfkj","id":"1(@)"} +{"time":"2026-05-18T15:33:20.635218256+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"oxmrotilaa4a"} +{"time":"2026-05-18T15:35:48.036183042+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-18T15:35:48.036225497+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-18T15:35:48.036239979+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-18T15:35:48.036291142+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-18T15:35:48.036309848+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3572879-3579183-3991850837/socket","Net":"unix"}} diff --git a/wandb/run-20260518_153314-yn3lgfkj/logs/debug-internal.log b/wandb/run-20260518_153314-yn3lgfkj/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b4e1aece706a9c1fef1d1342653a561d42c7507c --- /dev/null +++ b/wandb/run-20260518_153314-yn3lgfkj/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-18T15:33:14.568787835+09:00","level":"INFO","msg":"stream: starting","core version":"0.24.2"} +{"time":"2026-05-18T15:33:14.955449815+09:00","level":"INFO","msg":"stream: created new stream","id":"yn3lgfkj"} +{"time":"2026-05-18T15:33:14.955563765+09:00","level":"INFO","msg":"handler: started","stream_id":"yn3lgfkj"} +{"time":"2026-05-18T15:33:14.955729554+09:00","level":"INFO","msg":"stream: started","id":"yn3lgfkj"} +{"time":"2026-05-18T15:33:14.955755456+09:00","level":"INFO","msg":"writer: started","stream_id":"yn3lgfkj"} +{"time":"2026-05-18T15:33:14.955762385+09:00","level":"INFO","msg":"sender: started","stream_id":"yn3lgfkj"} +{"time":"2026-05-18T15:35:48.03624071+09:00","level":"INFO","msg":"stream: closing","id":"yn3lgfkj"} +{"time":"2026-05-18T15:35:48.678372175+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-18T15:35:48.977410208+09:00","level":"INFO","msg":"handler: closed","stream_id":"yn3lgfkj"} +{"time":"2026-05-18T15:35:48.977614839+09:00","level":"INFO","msg":"sender: closed","stream_id":"yn3lgfkj"} +{"time":"2026-05-18T15:35:48.977668037+09:00","level":"INFO","msg":"stream: closed","id":"yn3lgfkj"} diff --git a/wandb/run-20260518_153314-yn3lgfkj/logs/debug.log b/wandb/run-20260518_153314-yn3lgfkj/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..92d7dc657fbbf5ce31647b3466597347dc1e12a2 --- /dev/null +++ b/wandb/run-20260518_153314-yn3lgfkj/logs/debug.log @@ -0,0 +1,21 @@ +2026-05-18 15:33:14,329 INFO MainThread:3572879 [wandb_setup.py:_flush():81] Current SDK version is 0.24.2 +2026-05-18 15:33:14,329 INFO MainThread:3572879 [wandb_setup.py:_flush():81] Configure stats pid to 3572879 +2026-05-18 15:33:14,329 INFO MainThread:3572879 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-18 15:33:14,329 INFO MainThread:3572879 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260518_153314-yn3lgfkj/logs/debug.log +2026-05-18 15:33:14,329 INFO MainThread:3572879 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260518_153314-yn3lgfkj/logs/debug-internal.log +2026-05-18 15:33:14,329 INFO MainThread:3572879 [wandb_init.py:init():844] calling init triggers +2026-05-18 15:33:14,329 INFO MainThread:3572879 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'repo_id': '/home/seonho/insight/insight_dataset', 'dataset_root': '/home/seonho/insight/insight_dataset', 'method': 'mgd', 'phase': 'phase2', 'steps': 100000, 'lr': 0.0001, 'batch_size': 64, 'num_processes': 1, 'effective_batch_size': 64, 'action_dim': 8, 'state_dim': 16, 'camera_set': 'rgb', 'video_feature_keys': '', 'task_prompt_mode': 'non_guide', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_target': 'llm', 'mgd_enabled': False, 'mgd_trainable_mode': 'dit_only', 'mgd_loss_weight': 0.0, 'mgd_backprop_backbone': True, 'cl_v2_phase': None, 'cl_v2_trainable_mode': None, 'cl_v2_loss_weight': None, 'cl_v2_fm_loss_weight': None, 'gradient_checkpointing': False, 'seed': 42, 'dataset_len': 246793, '_wandb': {}} +2026-05-18 15:33:14,329 INFO MainThread:3572879 [wandb_init.py:init():892] starting backend +2026-05-18 15:33:14,552 INFO MainThread:3572879 [wandb_init.py:init():895] sending inform_init request +2026-05-18 15:33:14,564 INFO MainThread:3572879 [wandb_init.py:init():903] backend started and connected +2026-05-18 15:33:14,569 INFO MainThread:3572879 [wandb_init.py:init():973] updated telemetry +2026-05-18 15:33:14,570 INFO MainThread:3572879 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-18 15:33:15,431 INFO MainThread:3572879 [wandb_init.py:init():1042] starting run threads in backend +2026-05-18 15:33:15,625 INFO MainThread:3572879 [wandb_run.py:_console_start():2529] atexit reg +2026-05-18 15:33:15,625 INFO MainThread:3572879 [wandb_run.py:_redirect():2377] redirect: wrap_raw +2026-05-18 15:33:15,625 INFO MainThread:3572879 [wandb_run.py:_redirect():2446] Wrapping output streams. +2026-05-18 15:33:15,626 INFO MainThread:3572879 [wandb_run.py:_redirect():2469] Redirects installed. +2026-05-18 15:33:15,630 INFO MainThread:3572879 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-18 15:35:48,036 INFO wandb-AsyncioManager-main:3572879 [service_client.py:_forward_responses():94] Reached EOF. +2026-05-18 15:35:48,036 INFO wandb-AsyncioManager-main:3572879 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20260518_153314-yn3lgfkj/run-yn3lgfkj.wandb b/wandb/run-20260518_153314-yn3lgfkj/run-yn3lgfkj.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3a8c818ba677dc5d139e4b1db3c562dc13d8ff3a Binary files /dev/null and b/wandb/run-20260518_153314-yn3lgfkj/run-yn3lgfkj.wandb differ diff --git a/wandb/run-20260518_153916-gnre0khg/files/config.yaml b/wandb/run-20260518_153916-gnre0khg/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10280a2c63afd20f78d68bca4ef1c8621bc06377 --- /dev/null +++ b/wandb/run-20260518_153916-gnre0khg/files/config.yaml @@ -0,0 +1,179 @@ +_wandb: + value: + cli_version: 0.24.2 + e: + eiejxwqk8ah7uyyz6snxn8nfcb9e0jru: + args: + - --action_dim=8 + - --batch_size=64 + - --camera_set=rgb + - --dataset.repo_id=/home/seonho/insight/insight_dataset + - --dataset.root=/home/seonho/insight/insight_dataset + - --dataset.video_backend=pyav + - --log_freq=100 + - --num_workers=4 + - --output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt + - --phase=phase2 + - --policy.base_model_path=nvidia/GR00T-N1.5-3B + - --policy.lora_rank=0 + - --policy.mgd_enabled=false + - --policy.mgd_loss_weight=0.0 + - --policy.mgd_trainable_mode=dit_only + - --policy.tune_llm=false + - --policy.tune_visual=false + - --policy.type=groot_processed_mgd + - --save_freq=50000 + - --state_dim=16 + - --steps=100000 + - --task_prompt_mode=non_guide + - --wandb.enable=true + - --wandb.entity=RwHlabs + - --wandb.group=insight_phase1_dit_only + - --wandb.name=insight_phase1_dit_only_groot + - --wandb.project=groot_insight_baseline_mgd + - --wandb.tags=mgd,phase2,groot_processed_mgd,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base + codePath: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + codePathLocal: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + cpu_count: 32 + cpu_count_logical: 64 + cudaVersion: "12.2" + disk: + /: + total: "3778130485248" + used: "3442053578752" + email: dankk98@gmail.com + executable: /home/seonho/miniconda3/envs/minje/bin/python3.12 + gpu: NVIDIA A100 80GB PCIe + gpu_count: 4 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-ca1c0319-e178-a3c3-ac41-386c2723deac + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-47b6036e-4fde-e399-5515-f73eed752c6d + host: worker1 + memory: + total: "1081620840448" + os: Linux-6.8.0-101-generic-x86_64-with-glibc2.35 + program: /home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + python: CPython 3.12.12 + root: /home/seonho/insight/outputs/dit_only_phase1_ckpt + startedAt: "2026-05-18T06:39:16.296301Z" + writerId: eiejxwqk8ah7uyyz6snxn8nfcb9e0jru + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "3": + - 13 + - 15 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.24.2 + "6": 5.3.0 + "12": 0.24.2 + "13": linux-x86_64 +action_dim: + value: 8 +batch_size: + value: 64 +camera_set: + value: rgb +cl_v2_fm_loss_weight: + value: null +cl_v2_loss_weight: + value: null +cl_v2_phase: + value: null +cl_v2_trainable_mode: + value: null +dataset_len: + value: 246793 +dataset_root: + value: /home/seonho/insight/insight_dataset +effective_batch_size: + value: 64 +gradient_checkpointing: + value: false +lora_alpha: + value: 16 +lora_dropout: + value: 0.1 +lora_rank: + value: 0 +lora_target: + value: llm +lr: + value: 0.0001 +method: + value: mgd +mgd_backprop_backbone: + value: true +mgd_enabled: + value: false +mgd_loss_weight: + value: 0 +mgd_trainable_mode: + value: dit_only +num_processes: + value: 1 +phase: + value: phase2 +repo_id: + value: /home/seonho/insight/insight_dataset +seed: + value: 42 +state_dim: + value: 16 +steps: + value: 100000 +task_prompt_mode: + value: non_guide +tune_diffusion_model: + value: true +tune_llm: + value: false +tune_projector: + value: true +tune_visual: + value: false +video_feature_keys: + value: "" diff --git a/wandb/run-20260518_153916-gnre0khg/files/output.log b/wandb/run-20260518_153916-gnre0khg/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ccd996e61ad451f06ada8fe510b60b6829ead6ed --- /dev/null +++ b/wandb/run-20260518_153916-gnre0khg/files/output.log @@ -0,0 +1,3 @@ +`use_fast` is set to `True` but the image processor class does not have a fast version. Falling back to the slow version. +The tokenizer you are loading from '/home/seonho/.cache/huggingface/lerobot/lerobot/eagle2hg-processor-groot-n1p5' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue. +The tokenizer you are loading from '/home/seonho/.cache/huggingface/lerobot/lerobot/eagle2hg-processor-groot-n1p5' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue. diff --git a/wandb/run-20260518_153916-gnre0khg/files/requirements.txt b/wandb/run-20260518_153916-gnre0khg/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8d89644db6c7aa4d72b923c55c62ae8132609a93 --- /dev/null +++ b/wandb/run-20260518_153916-gnre0khg/files/requirements.txt @@ -0,0 +1,209 @@ +attrs==25.4.0 +kiwisolver==1.5.0 +setuptools==80.10.2 +transformers==5.3.0 +pynndescent==0.6.0 +filelock==3.25.2 +numba==0.61.2 +gymnasium==1.2.3 +shellingham==1.5.4 +dill==0.3.8 +ninja==1.13.0 +hidapi==0.15.0 +scikit-image==0.26.0 +orderly-set==5.5.0 +mypy_extensions==1.1.0 +stringzilla==4.6.1 +tianshou==0.4.10 +numpy==2.2.5 +xxhash==3.6.0 +protobuf==3.19.6 +numpydantic==1.6.7 +pyserial==3.5 +omegaconf==2.3.0 +Farama-Notifications==0.0.4 +nvidia-nccl-cu12==2.26.2 +plotly==6.6.0 +platformdirs==4.9.4 +nvidia-cufft-cu12==11.3.0.4 +eval_type_backport==0.3.1 +packaging==25.0 +typing-inspect==0.9.0 +opencv-python-headless==4.12.0.88 +cmeel-assimp==6.0.2 +lerobot==0.5.1 +pyyaml-include==1.4.1 +urllib3==2.6.3 +lxml==6.1.0 +peft==0.18.1 +virtualenv==21.2.4 +cmeel==0.59.0 +pre_commit==4.6.0 +importlib_metadata==8.7.1 +click==8.3.1 +accelerate==1.13.0 +multiprocess==0.70.16 +distlib==0.4.0 +websockets==16.0 +albumentations==1.4.18 +tokenizers==0.22.2 +nvidia-cusparselt-cu12==0.6.3 +certifi==2026.2.25 +annotated-doc==0.0.4 +triton==3.3.1 +libpinocchio==3.9.0 +mpmath==1.3.0 +decord==0.6.0 +matplotlib==3.10.8 +llvmlite==0.44.0 +python-xlib==0.33 +regex==2026.2.28 +colored==2.3.2 +charset-normalizer==3.4.5 +mdurl==0.1.2 +GitPython==3.1.46 +gym==0.26.2 +av==15.1.0 +markdown-it-py==4.0.0 +glfw==2.10.0 +nvidia-cufile-cu12==1.11.1.6 +PyYAML==6.0.3 +python-dateutil==2.9.0.post0 +cmeel-tinyxml2==10.0.0 +blinker==1.9.0 +opencv-python==4.13.0.92 +cmake==4.1.3 +gym-notices==0.1.0 +typing-inspection==0.4.2 +timm==1.0.25 +torchcodec==0.5 +cmeel-octomap==1.10.0 +umap-learn==0.5.11 +MarkupSafe==3.0.3 +pluggy==1.6.0 +smmap==5.0.3 +fsspec==2025.3.0 +evdev==1.9.3 +python-discovery==1.2.2 +hf-xet==1.5.0 +scikit-learn==1.8.0 +h5py==3.16.0 +cmeel-console-bridge==1.0.2.3 +narwhals==2.18.1 +msgpack==1.1.2 +flash_attn==2.8.3 +ansitable==0.11.5 +pyarrow==23.0.1 +h11==0.16.0 +mergedeep==1.3.4 +tqdm==4.67.3 +vla-eval==0.1.1.dev27+g195d46338 +nvidia-nvjitlink-cu12==12.6.85 +cmeel-qhull==8.0.2.1 +libcoal==3.0.2 +cmeel-boost==1.89.0 +rtb-data==1.0.1 +mujoco==3.3.1 +nvidia-cuda-nvrtc-cu12==12.6.77 +PyOpenGL==3.1.10 +draccus==0.10.0 +pip==26.0.1 +threadpoolctl==3.6.0 +annotated-types==0.7.0 +anyio==4.12.1 +imageio-ffmpeg==0.6.0 +qpsolvers==4.11.0 +wrapt==2.1.2 +networkx==3.6.1 +contourpy==1.3.3 +Werkzeug==3.1.8 +tensorboard-data-server==0.7.2 +gitdb==4.0.12 +nvidia-cusparse-cu12==12.5.4.2 +typer==0.24.1 +toml==0.10.2 +tensorboard==2.20.0 +rerun-sdk==0.26.2 +diffusers==0.35.2 +albucore==0.0.17 +wandb==0.24.2 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-runtime-cu12==12.6.77 +cycler==0.12.1 +coal==3.0.2 +pydantic_core==2.27.2 +torch==2.7.1 +nvidia-cuda-cupti-cu12==12.6.80 +Pygments==2.19.2 +pytest==9.0.3 +ImageIO==2.37.3 +sentry-sdk==2.54.0 +antlr4-python3-runtime==4.9.3 +sympy==1.14.0 +safetensors==0.7.0 +cmeel-zlib==1.3.1 +einops==0.8.2 +spatialgeometry==1.1.0 +pygame==2.6.1 +absl-py==2.4.0 +pandas==3.0.1 +httpx==0.28.1 +robosuite==1.5.2 +fonttools==4.62.1 +progress==1.6.1 +nvidia-cusolver-cu12==11.7.1.2 +aiohttp==3.13.3 +frozenlist==1.8.0 +deepdiff==8.6.1 +yarl==1.23.0 +nvidia-cudnn-cu12==9.5.1.17 +torchvision==0.22.1 +idna==3.11 +lazy-loader==0.5 +termcolor==3.3.0 +nvidia-curand-cu12==10.3.7.77 +multidict==6.7.1 +Markdown==3.10.2 +pynput==1.8.1 +propcache==0.4.1 +zipp==3.23.0 +nvidia-nvtx-cu12==12.6.77 +Flask==3.1.3 +httpcore==1.0.9 +etils==1.14.0 +psutil==7.2.2 +pillow==12.1.1 +nodeenv==1.10.0 +pin==3.9.0 +quadprog==0.1.13 +eigenpy==3.12.0 +wheel==0.46.3 +simsimd==6.5.16 +itsdangerous==2.2.0 +aiohappyeyeballs==2.6.1 +cfgv==3.5.0 +cloudpickle==3.1.2 +iniconfig==2.3.0 +requests==2.32.5 +tifffile==2026.5.2 +joblib==1.5.3 +identify==2.6.19 +huggingface_hub==1.14.0 +datasets==4.8.4 +typing_extensions==4.15.0 +pydantic==2.10.6 +jsonlines==4.0.0 +swift-sim==1.1.0 +lazyregistry==0.4.0 +robocasa==1.0.0 +pgraph-python==0.6.3 +Jinja2==3.1.6 +aiosignal==1.4.0 +dm-tree==0.1.9 +six==1.17.0 +grpcio==1.80.0 +rich==14.3.3 +scipy==1.15.3 +cmeel-urdfdom==4.0.1 +pyparsing==3.3.2 +lerobot==0.5.1 diff --git a/wandb/run-20260518_153916-gnre0khg/files/wandb-metadata.json b/wandb/run-20260518_153916-gnre0khg/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3ecc67be368695db62eb8ae69390b64600a9696a --- /dev/null +++ b/wandb/run-20260518_153916-gnre0khg/files/wandb-metadata.json @@ -0,0 +1,87 @@ +{ + "os": "Linux-6.8.0-101-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.12", + "startedAt": "2026-05-18T06:39:16.296301Z", + "args": [ + "--action_dim=8", + "--batch_size=64", + "--camera_set=rgb", + "--dataset.repo_id=/home/seonho/insight/insight_dataset", + "--dataset.root=/home/seonho/insight/insight_dataset", + "--dataset.video_backend=pyav", + "--log_freq=100", + "--num_workers=4", + "--output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "--phase=phase2", + "--policy.base_model_path=nvidia/GR00T-N1.5-3B", + "--policy.lora_rank=0", + "--policy.mgd_enabled=false", + "--policy.mgd_loss_weight=0.0", + "--policy.mgd_trainable_mode=dit_only", + "--policy.tune_llm=false", + "--policy.tune_visual=false", + "--policy.type=groot_processed_mgd", + "--save_freq=50000", + "--state_dim=16", + "--steps=100000", + "--task_prompt_mode=non_guide", + "--wandb.enable=true", + "--wandb.entity=RwHlabs", + "--wandb.group=insight_phase1_dit_only", + "--wandb.name=insight_phase1_dit_only_groot", + "--wandb.project=groot_insight_baseline_mgd", + "--wandb.tags=mgd,phase2,groot_processed_mgd,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + ], + "program": "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePath": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePathLocal": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "email": "dankk98@gmail.com", + "root": "/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "host": "worker1", + "executable": "/home/seonho/miniconda3/envs/minje/bin/python3.12", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 4, + "disk": { + "/": { + "total": "3778130485248", + "used": "3442053578752" + } + }, + "memory": { + "total": "1081620840448" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-ca1c0319-e178-a3c3-ac41-386c2723deac" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-47b6036e-4fde-e399-5515-f73eed752c6d" + } + ], + "cudaVersion": "12.2", + "writerId": "eiejxwqk8ah7uyyz6snxn8nfcb9e0jru" +} \ No newline at end of file diff --git a/wandb/run-20260518_153916-gnre0khg/files/wandb-summary.json b/wandb/run-20260518_153916-gnre0khg/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..38c4f20899635b9a3d0504c8b94d681a64038095 --- /dev/null +++ b/wandb/run-20260518_153916-gnre0khg/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/mgd_loss":0,"train/step_s":2.814238720573485,"_wandb":{"runtime":109231},"_step":35200,"train/step_per_s":0.3553358827342907,"params/whole_trainable":840593408,"params/whole_total":2427681217,"train/mgd_token_mask_ratio_cfg":0.1,"train/total_step_s":2.814238720573485,"train/update_s":1.3307745764032006,"train/grad_norm":0.2792210876941681,"train/data_s":1.4834641441702843,"_timestamp":1.7791955259878259e+09,"train/policy_update_called":0,"train/lr":7.70699658915369e-05,"train/loss":0.02990889735519886,"_runtime":109231.984320202,"train/flow_matching_loss":0.02990889735519886} \ No newline at end of file diff --git a/wandb/run-20260518_153916-gnre0khg/logs/debug-core.log b/wandb/run-20260518_153916-gnre0khg/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..35fb6b5a42ae3968210e8c7fb81bafb4a154faeb --- /dev/null +++ b/wandb/run-20260518_153916-gnre0khg/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2026-05-18T15:39:16.32873076+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpdlzfx1u5/port-57400.txt","pid":57400,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-18T15:39:16.329166005+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":57400} +{"time":"2026-05-18T15:39:16.329149987+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-57400-66243-3803199700/socket","Net":"unix"}} +{"time":"2026-05-18T15:39:16.517417743+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-18T15:39:16.528153073+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"gnre0khg","id":"1(@)"} +{"time":"2026-05-18T15:39:16.99630513+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"gnre0khg","id":"1(@)"} +{"time":"2026-05-18T15:39:22.690960038+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"brn8qnywixlb"} +{"time":"2026-05-19T21:59:49.469269451+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-19T21:59:49.469329025+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-19T21:59:49.469318458+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-19T21:59:49.469424549+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-57400-66243-3803199700/socket","Net":"unix"}} +{"time":"2026-05-19T21:59:49.469479024+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} diff --git a/wandb/run-20260518_153916-gnre0khg/logs/debug-internal.log b/wandb/run-20260518_153916-gnre0khg/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a43baba476dea65306634523d1d9e8ad632d606e --- /dev/null +++ b/wandb/run-20260518_153916-gnre0khg/logs/debug-internal.log @@ -0,0 +1,13 @@ +{"time":"2026-05-18T15:39:16.528254116+09:00","level":"INFO","msg":"stream: starting","core version":"0.24.2"} +{"time":"2026-05-18T15:39:16.995905538+09:00","level":"INFO","msg":"stream: created new stream","id":"gnre0khg"} +{"time":"2026-05-18T15:39:16.996063345+09:00","level":"INFO","msg":"handler: started","stream_id":"gnre0khg"} +{"time":"2026-05-18T15:39:16.99628302+09:00","level":"INFO","msg":"stream: started","id":"gnre0khg"} +{"time":"2026-05-18T15:39:16.996311773+09:00","level":"INFO","msg":"writer: started","stream_id":"gnre0khg"} +{"time":"2026-05-18T15:39:16.996317275+09:00","level":"INFO","msg":"sender: started","stream_id":"gnre0khg"} +{"time":"2026-05-19T07:49:57.376272973+09:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_mgd/gnre0khg/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2026-05-19T17:57:47.189450009+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_mgd/gnre0khg/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-19T21:59:49.469321045+09:00","level":"INFO","msg":"stream: closing","id":"gnre0khg"} +{"time":"2026-05-19T21:59:50.721764721+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-19T21:59:51.039395154+09:00","level":"INFO","msg":"handler: closed","stream_id":"gnre0khg"} +{"time":"2026-05-19T21:59:51.039470981+09:00","level":"INFO","msg":"sender: closed","stream_id":"gnre0khg"} +{"time":"2026-05-19T21:59:51.039487806+09:00","level":"INFO","msg":"stream: closed","id":"gnre0khg"} diff --git a/wandb/run-20260518_153916-gnre0khg/logs/debug.log b/wandb/run-20260518_153916-gnre0khg/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..50fbb3bbf866ab1296d93b53ae429f0cdf759564 --- /dev/null +++ b/wandb/run-20260518_153916-gnre0khg/logs/debug.log @@ -0,0 +1,21 @@ +2026-05-18 15:39:16,297 INFO MainThread:57400 [wandb_setup.py:_flush():81] Current SDK version is 0.24.2 +2026-05-18 15:39:16,297 INFO MainThread:57400 [wandb_setup.py:_flush():81] Configure stats pid to 57400 +2026-05-18 15:39:16,297 INFO MainThread:57400 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-18 15:39:16,297 INFO MainThread:57400 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260518_153916-gnre0khg/logs/debug.log +2026-05-18 15:39:16,297 INFO MainThread:57400 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260518_153916-gnre0khg/logs/debug-internal.log +2026-05-18 15:39:16,297 INFO MainThread:57400 [wandb_init.py:init():844] calling init triggers +2026-05-18 15:39:16,298 INFO MainThread:57400 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'repo_id': '/home/seonho/insight/insight_dataset', 'dataset_root': '/home/seonho/insight/insight_dataset', 'method': 'mgd', 'phase': 'phase2', 'steps': 100000, 'lr': 0.0001, 'batch_size': 64, 'num_processes': 1, 'effective_batch_size': 64, 'action_dim': 8, 'state_dim': 16, 'camera_set': 'rgb', 'video_feature_keys': '', 'task_prompt_mode': 'non_guide', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_target': 'llm', 'mgd_enabled': False, 'mgd_trainable_mode': 'dit_only', 'mgd_loss_weight': 0.0, 'mgd_backprop_backbone': True, 'cl_v2_phase': None, 'cl_v2_trainable_mode': None, 'cl_v2_loss_weight': None, 'cl_v2_fm_loss_weight': None, 'gradient_checkpointing': False, 'seed': 42, 'dataset_len': 246793, '_wandb': {}} +2026-05-18 15:39:16,298 INFO MainThread:57400 [wandb_init.py:init():892] starting backend +2026-05-18 15:39:16,517 INFO MainThread:57400 [wandb_init.py:init():895] sending inform_init request +2026-05-18 15:39:16,526 INFO MainThread:57400 [wandb_init.py:init():903] backend started and connected +2026-05-18 15:39:16,531 INFO MainThread:57400 [wandb_init.py:init():973] updated telemetry +2026-05-18 15:39:16,532 INFO MainThread:57400 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-18 15:39:17,479 INFO MainThread:57400 [wandb_init.py:init():1042] starting run threads in backend +2026-05-18 15:39:17,682 INFO MainThread:57400 [wandb_run.py:_console_start():2529] atexit reg +2026-05-18 15:39:17,682 INFO MainThread:57400 [wandb_run.py:_redirect():2377] redirect: wrap_raw +2026-05-18 15:39:17,682 INFO MainThread:57400 [wandb_run.py:_redirect():2446] Wrapping output streams. +2026-05-18 15:39:17,682 INFO MainThread:57400 [wandb_run.py:_redirect():2469] Redirects installed. +2026-05-18 15:39:17,686 INFO MainThread:57400 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-19 21:59:49,469 INFO wandb-AsyncioManager-main:57400 [service_client.py:_forward_responses():94] Reached EOF. +2026-05-19 21:59:49,469 INFO wandb-AsyncioManager-main:57400 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20260518_153916-gnre0khg/run-gnre0khg.wandb b/wandb/run-20260518_153916-gnre0khg/run-gnre0khg.wandb new file mode 100644 index 0000000000000000000000000000000000000000..05da84d022f241bc74b8b0f462c0baaded928ab1 --- /dev/null +++ b/wandb/run-20260518_153916-gnre0khg/run-gnre0khg.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a5cc78f6301fe62e29bce469f3820e4d4b97a36e9675ac8e57c66b2d17048dd +size 15238877 diff --git a/wandb/run-20260519_220343-bgm9lwvb/files/config.yaml b/wandb/run-20260519_220343-bgm9lwvb/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98199dce2cfcd9d59586061f9890979e65651c37 --- /dev/null +++ b/wandb/run-20260519_220343-bgm9lwvb/files/config.yaml @@ -0,0 +1,179 @@ +_wandb: + value: + cli_version: 0.24.2 + e: + 2nphg9ukwpynd1goji2fvzir5qrvfux2: + args: + - --action_dim=8 + - --batch_size=64 + - --camera_set=rgb + - --dataset.repo_id=/home/seonho/insight/insight_dataset + - --dataset.root=/home/seonho/insight/insight_dataset + - --dataset.video_backend=pyav + - --log_freq=100 + - --num_workers=4 + - --output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt + - --phase=phase2 + - --policy.base_model_path=nvidia/GR00T-N1.5-3B + - --policy.lora_rank=0 + - --policy.mgd_enabled=false + - --policy.mgd_loss_weight=0.0 + - --policy.mgd_trainable_mode=dit_only + - --policy.tune_llm=false + - --policy.tune_visual=false + - --policy.type=groot_processed_mgd + - --save_freq=50000 + - --state_dim=16 + - --steps=100000 + - --task_prompt_mode=non_guide + - --wandb.enable=true + - --wandb.entity=RwHlabs + - --wandb.group=insight_phase1_dit_only + - --wandb.name=insight_phase1_dit_only_groot + - --wandb.project=groot_insight_baseline_mgd + - --wandb.tags=mgd,phase2,groot_processed_mgd,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base + codePath: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + codePathLocal: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + cpu_count: 32 + cpu_count_logical: 64 + cudaVersion: "12.2" + disk: + /: + total: "3778130485248" + used: "3366425817088" + email: dankk98@gmail.com + executable: /home/seonho/miniconda3/envs/minje/bin/python3.12 + gpu: NVIDIA A100 80GB PCIe + gpu_count: 4 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-ca1c0319-e178-a3c3-ac41-386c2723deac + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-47b6036e-4fde-e399-5515-f73eed752c6d + host: worker1 + memory: + total: "1081620840448" + os: Linux-6.8.0-101-generic-x86_64-with-glibc2.35 + program: /home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + python: CPython 3.12.12 + root: /home/seonho/insight/outputs/dit_only_phase1_ckpt + startedAt: "2026-05-19T13:03:43.486610Z" + writerId: 2nphg9ukwpynd1goji2fvzir5qrvfux2 + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "3": + - 13 + - 15 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.24.2 + "6": 5.3.0 + "12": 0.24.2 + "13": linux-x86_64 +action_dim: + value: 8 +batch_size: + value: 64 +camera_set: + value: rgb +cl_v2_fm_loss_weight: + value: null +cl_v2_loss_weight: + value: null +cl_v2_phase: + value: null +cl_v2_trainable_mode: + value: null +dataset_len: + value: 246793 +dataset_root: + value: /home/seonho/insight/insight_dataset +effective_batch_size: + value: 64 +gradient_checkpointing: + value: false +lora_alpha: + value: 16 +lora_dropout: + value: 0.1 +lora_rank: + value: 0 +lora_target: + value: llm +lr: + value: 0.0001 +method: + value: mgd +mgd_backprop_backbone: + value: true +mgd_enabled: + value: false +mgd_loss_weight: + value: 0 +mgd_trainable_mode: + value: dit_only +num_processes: + value: 1 +phase: + value: phase2 +repo_id: + value: /home/seonho/insight/insight_dataset +seed: + value: 42 +state_dim: + value: 16 +steps: + value: 100000 +task_prompt_mode: + value: non_guide +tune_diffusion_model: + value: true +tune_llm: + value: false +tune_projector: + value: true +tune_visual: + value: false +video_feature_keys: + value: "" diff --git a/wandb/run-20260519_220343-bgm9lwvb/files/output.log b/wandb/run-20260519_220343-bgm9lwvb/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ccd996e61ad451f06ada8fe510b60b6829ead6ed --- /dev/null +++ b/wandb/run-20260519_220343-bgm9lwvb/files/output.log @@ -0,0 +1,3 @@ +`use_fast` is set to `True` but the image processor class does not have a fast version. Falling back to the slow version. +The tokenizer you are loading from '/home/seonho/.cache/huggingface/lerobot/lerobot/eagle2hg-processor-groot-n1p5' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue. +The tokenizer you are loading from '/home/seonho/.cache/huggingface/lerobot/lerobot/eagle2hg-processor-groot-n1p5' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue. diff --git a/wandb/run-20260519_220343-bgm9lwvb/files/requirements.txt b/wandb/run-20260519_220343-bgm9lwvb/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8d89644db6c7aa4d72b923c55c62ae8132609a93 --- /dev/null +++ b/wandb/run-20260519_220343-bgm9lwvb/files/requirements.txt @@ -0,0 +1,209 @@ +attrs==25.4.0 +kiwisolver==1.5.0 +setuptools==80.10.2 +transformers==5.3.0 +pynndescent==0.6.0 +filelock==3.25.2 +numba==0.61.2 +gymnasium==1.2.3 +shellingham==1.5.4 +dill==0.3.8 +ninja==1.13.0 +hidapi==0.15.0 +scikit-image==0.26.0 +orderly-set==5.5.0 +mypy_extensions==1.1.0 +stringzilla==4.6.1 +tianshou==0.4.10 +numpy==2.2.5 +xxhash==3.6.0 +protobuf==3.19.6 +numpydantic==1.6.7 +pyserial==3.5 +omegaconf==2.3.0 +Farama-Notifications==0.0.4 +nvidia-nccl-cu12==2.26.2 +plotly==6.6.0 +platformdirs==4.9.4 +nvidia-cufft-cu12==11.3.0.4 +eval_type_backport==0.3.1 +packaging==25.0 +typing-inspect==0.9.0 +opencv-python-headless==4.12.0.88 +cmeel-assimp==6.0.2 +lerobot==0.5.1 +pyyaml-include==1.4.1 +urllib3==2.6.3 +lxml==6.1.0 +peft==0.18.1 +virtualenv==21.2.4 +cmeel==0.59.0 +pre_commit==4.6.0 +importlib_metadata==8.7.1 +click==8.3.1 +accelerate==1.13.0 +multiprocess==0.70.16 +distlib==0.4.0 +websockets==16.0 +albumentations==1.4.18 +tokenizers==0.22.2 +nvidia-cusparselt-cu12==0.6.3 +certifi==2026.2.25 +annotated-doc==0.0.4 +triton==3.3.1 +libpinocchio==3.9.0 +mpmath==1.3.0 +decord==0.6.0 +matplotlib==3.10.8 +llvmlite==0.44.0 +python-xlib==0.33 +regex==2026.2.28 +colored==2.3.2 +charset-normalizer==3.4.5 +mdurl==0.1.2 +GitPython==3.1.46 +gym==0.26.2 +av==15.1.0 +markdown-it-py==4.0.0 +glfw==2.10.0 +nvidia-cufile-cu12==1.11.1.6 +PyYAML==6.0.3 +python-dateutil==2.9.0.post0 +cmeel-tinyxml2==10.0.0 +blinker==1.9.0 +opencv-python==4.13.0.92 +cmake==4.1.3 +gym-notices==0.1.0 +typing-inspection==0.4.2 +timm==1.0.25 +torchcodec==0.5 +cmeel-octomap==1.10.0 +umap-learn==0.5.11 +MarkupSafe==3.0.3 +pluggy==1.6.0 +smmap==5.0.3 +fsspec==2025.3.0 +evdev==1.9.3 +python-discovery==1.2.2 +hf-xet==1.5.0 +scikit-learn==1.8.0 +h5py==3.16.0 +cmeel-console-bridge==1.0.2.3 +narwhals==2.18.1 +msgpack==1.1.2 +flash_attn==2.8.3 +ansitable==0.11.5 +pyarrow==23.0.1 +h11==0.16.0 +mergedeep==1.3.4 +tqdm==4.67.3 +vla-eval==0.1.1.dev27+g195d46338 +nvidia-nvjitlink-cu12==12.6.85 +cmeel-qhull==8.0.2.1 +libcoal==3.0.2 +cmeel-boost==1.89.0 +rtb-data==1.0.1 +mujoco==3.3.1 +nvidia-cuda-nvrtc-cu12==12.6.77 +PyOpenGL==3.1.10 +draccus==0.10.0 +pip==26.0.1 +threadpoolctl==3.6.0 +annotated-types==0.7.0 +anyio==4.12.1 +imageio-ffmpeg==0.6.0 +qpsolvers==4.11.0 +wrapt==2.1.2 +networkx==3.6.1 +contourpy==1.3.3 +Werkzeug==3.1.8 +tensorboard-data-server==0.7.2 +gitdb==4.0.12 +nvidia-cusparse-cu12==12.5.4.2 +typer==0.24.1 +toml==0.10.2 +tensorboard==2.20.0 +rerun-sdk==0.26.2 +diffusers==0.35.2 +albucore==0.0.17 +wandb==0.24.2 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-runtime-cu12==12.6.77 +cycler==0.12.1 +coal==3.0.2 +pydantic_core==2.27.2 +torch==2.7.1 +nvidia-cuda-cupti-cu12==12.6.80 +Pygments==2.19.2 +pytest==9.0.3 +ImageIO==2.37.3 +sentry-sdk==2.54.0 +antlr4-python3-runtime==4.9.3 +sympy==1.14.0 +safetensors==0.7.0 +cmeel-zlib==1.3.1 +einops==0.8.2 +spatialgeometry==1.1.0 +pygame==2.6.1 +absl-py==2.4.0 +pandas==3.0.1 +httpx==0.28.1 +robosuite==1.5.2 +fonttools==4.62.1 +progress==1.6.1 +nvidia-cusolver-cu12==11.7.1.2 +aiohttp==3.13.3 +frozenlist==1.8.0 +deepdiff==8.6.1 +yarl==1.23.0 +nvidia-cudnn-cu12==9.5.1.17 +torchvision==0.22.1 +idna==3.11 +lazy-loader==0.5 +termcolor==3.3.0 +nvidia-curand-cu12==10.3.7.77 +multidict==6.7.1 +Markdown==3.10.2 +pynput==1.8.1 +propcache==0.4.1 +zipp==3.23.0 +nvidia-nvtx-cu12==12.6.77 +Flask==3.1.3 +httpcore==1.0.9 +etils==1.14.0 +psutil==7.2.2 +pillow==12.1.1 +nodeenv==1.10.0 +pin==3.9.0 +quadprog==0.1.13 +eigenpy==3.12.0 +wheel==0.46.3 +simsimd==6.5.16 +itsdangerous==2.2.0 +aiohappyeyeballs==2.6.1 +cfgv==3.5.0 +cloudpickle==3.1.2 +iniconfig==2.3.0 +requests==2.32.5 +tifffile==2026.5.2 +joblib==1.5.3 +identify==2.6.19 +huggingface_hub==1.14.0 +datasets==4.8.4 +typing_extensions==4.15.0 +pydantic==2.10.6 +jsonlines==4.0.0 +swift-sim==1.1.0 +lazyregistry==0.4.0 +robocasa==1.0.0 +pgraph-python==0.6.3 +Jinja2==3.1.6 +aiosignal==1.4.0 +dm-tree==0.1.9 +six==1.17.0 +grpcio==1.80.0 +rich==14.3.3 +scipy==1.15.3 +cmeel-urdfdom==4.0.1 +pyparsing==3.3.2 +lerobot==0.5.1 diff --git a/wandb/run-20260519_220343-bgm9lwvb/files/wandb-metadata.json b/wandb/run-20260519_220343-bgm9lwvb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bbd844bb098d897cc52874cb62740ffda8b3a58d --- /dev/null +++ b/wandb/run-20260519_220343-bgm9lwvb/files/wandb-metadata.json @@ -0,0 +1,87 @@ +{ + "os": "Linux-6.8.0-101-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.12", + "startedAt": "2026-05-19T13:03:43.486610Z", + "args": [ + "--action_dim=8", + "--batch_size=64", + "--camera_set=rgb", + "--dataset.repo_id=/home/seonho/insight/insight_dataset", + "--dataset.root=/home/seonho/insight/insight_dataset", + "--dataset.video_backend=pyav", + "--log_freq=100", + "--num_workers=4", + "--output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "--phase=phase2", + "--policy.base_model_path=nvidia/GR00T-N1.5-3B", + "--policy.lora_rank=0", + "--policy.mgd_enabled=false", + "--policy.mgd_loss_weight=0.0", + "--policy.mgd_trainable_mode=dit_only", + "--policy.tune_llm=false", + "--policy.tune_visual=false", + "--policy.type=groot_processed_mgd", + "--save_freq=50000", + "--state_dim=16", + "--steps=100000", + "--task_prompt_mode=non_guide", + "--wandb.enable=true", + "--wandb.entity=RwHlabs", + "--wandb.group=insight_phase1_dit_only", + "--wandb.name=insight_phase1_dit_only_groot", + "--wandb.project=groot_insight_baseline_mgd", + "--wandb.tags=mgd,phase2,groot_processed_mgd,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + ], + "program": "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePath": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePathLocal": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "email": "dankk98@gmail.com", + "root": "/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "host": "worker1", + "executable": "/home/seonho/miniconda3/envs/minje/bin/python3.12", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 4, + "disk": { + "/": { + "total": "3778130485248", + "used": "3366425817088" + } + }, + "memory": { + "total": "1081620840448" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-ca1c0319-e178-a3c3-ac41-386c2723deac" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-47b6036e-4fde-e399-5515-f73eed752c6d" + } + ], + "cudaVersion": "12.2", + "writerId": "2nphg9ukwpynd1goji2fvzir5qrvfux2" +} \ No newline at end of file diff --git a/wandb/run-20260519_220343-bgm9lwvb/files/wandb-summary.json b/wandb/run-20260519_220343-bgm9lwvb/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e50028cfd2efbfb0d80ff4f27cc70d22cdcb05c8 --- /dev/null +++ b/wandb/run-20260519_220343-bgm9lwvb/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/step_per_s":0.37371609007549866,"train/flow_matching_loss":0.8621846437454224,"train/lr":2.0000000000000003e-06,"train/step_s":2.6758280592039227,"_step":100,"_runtime":541.832323407,"train/grad_norm":2.26335072517395,"train/policy_update_called":0,"train/update_s":1.3374275844544172,"params/whole_total":2427681217,"_timestamp":1.779196151028139e+09,"train/total_step_s":2.6758280592039227,"train/mgd_loss":0,"train/data_s":1.3384004747495055,"train/mgd_token_mask_ratio_cfg":0.1,"train/loss":0.8621846437454224,"params/whole_trainable":840593408,"_wandb":{"runtime":541}} \ No newline at end of file diff --git a/wandb/run-20260519_220343-bgm9lwvb/logs/debug-core.log b/wandb/run-20260519_220343-bgm9lwvb/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..ad23f34d55d31d0e4c3b95c11d7504461359ebbf --- /dev/null +++ b/wandb/run-20260519_220343-bgm9lwvb/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2026-05-19T22:03:43.519621576+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpqirxk_on/port-1549642.txt","pid":1549642,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-19T22:03:43.520097337+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":1549642} +{"time":"2026-05-19T22:03:43.520092462+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1549642-1555987-1519130990/socket","Net":"unix"}} +{"time":"2026-05-19T22:03:43.705660907+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-19T22:03:43.718970302+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"bgm9lwvb","id":"1(@)"} +{"time":"2026-05-19T22:03:44.22619333+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"bgm9lwvb","id":"1(@)"} +{"time":"2026-05-19T22:03:50.066872258+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"hui5y9h3x06n"} +{"time":"2026-05-19T22:12:46.699529098+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-19T22:12:46.699585939+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-19T22:12:46.699580621+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-19T22:12:46.699677433+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-19T22:12:46.699676307+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1549642-1555987-1519130990/socket","Net":"unix"}} diff --git a/wandb/run-20260519_220343-bgm9lwvb/logs/debug-internal.log b/wandb/run-20260519_220343-bgm9lwvb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8b92fe1f1666822984052499d171ec151cfd238e --- /dev/null +++ b/wandb/run-20260519_220343-bgm9lwvb/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2026-05-19T22:03:43.719258177+09:00","level":"INFO","msg":"stream: starting","core version":"0.24.2"} +{"time":"2026-05-19T22:03:44.225817762+09:00","level":"INFO","msg":"stream: created new stream","id":"bgm9lwvb"} +{"time":"2026-05-19T22:03:44.225945238+09:00","level":"INFO","msg":"handler: started","stream_id":"bgm9lwvb"} +{"time":"2026-05-19T22:03:44.226173885+09:00","level":"INFO","msg":"stream: started","id":"bgm9lwvb"} +{"time":"2026-05-19T22:03:44.226210237+09:00","level":"INFO","msg":"writer: started","stream_id":"bgm9lwvb"} +{"time":"2026-05-19T22:03:44.226206941+09:00","level":"INFO","msg":"sender: started","stream_id":"bgm9lwvb"} +{"time":"2026-05-19T22:12:46.699591081+09:00","level":"INFO","msg":"stream: closing","id":"bgm9lwvb"} +{"time":"2026-05-19T22:12:47.888667728+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-19T22:12:48.197154966+09:00","level":"INFO","msg":"handler: closed","stream_id":"bgm9lwvb"} +{"time":"2026-05-19T22:12:48.197316409+09:00","level":"INFO","msg":"sender: closed","stream_id":"bgm9lwvb"} +{"time":"2026-05-19T22:12:48.197360527+09:00","level":"INFO","msg":"stream: closed","id":"bgm9lwvb"} diff --git a/wandb/run-20260519_220343-bgm9lwvb/logs/debug.log b/wandb/run-20260519_220343-bgm9lwvb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..add98bf9755c52a21810021598bd3a418c737a17 --- /dev/null +++ b/wandb/run-20260519_220343-bgm9lwvb/logs/debug.log @@ -0,0 +1,21 @@ +2026-05-19 22:03:43,487 INFO MainThread:1549642 [wandb_setup.py:_flush():81] Current SDK version is 0.24.2 +2026-05-19 22:03:43,487 INFO MainThread:1549642 [wandb_setup.py:_flush():81] Configure stats pid to 1549642 +2026-05-19 22:03:43,487 INFO MainThread:1549642 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-19 22:03:43,487 INFO MainThread:1549642 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260519_220343-bgm9lwvb/logs/debug.log +2026-05-19 22:03:43,488 INFO MainThread:1549642 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260519_220343-bgm9lwvb/logs/debug-internal.log +2026-05-19 22:03:43,488 INFO MainThread:1549642 [wandb_init.py:init():844] calling init triggers +2026-05-19 22:03:43,488 INFO MainThread:1549642 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'repo_id': '/home/seonho/insight/insight_dataset', 'dataset_root': '/home/seonho/insight/insight_dataset', 'method': 'mgd', 'phase': 'phase2', 'steps': 100000, 'lr': 0.0001, 'batch_size': 64, 'num_processes': 1, 'effective_batch_size': 64, 'action_dim': 8, 'state_dim': 16, 'camera_set': 'rgb', 'video_feature_keys': '', 'task_prompt_mode': 'non_guide', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_target': 'llm', 'mgd_enabled': False, 'mgd_trainable_mode': 'dit_only', 'mgd_loss_weight': 0.0, 'mgd_backprop_backbone': True, 'cl_v2_phase': None, 'cl_v2_trainable_mode': None, 'cl_v2_loss_weight': None, 'cl_v2_fm_loss_weight': None, 'gradient_checkpointing': False, 'seed': 42, 'dataset_len': 246793, '_wandb': {}} +2026-05-19 22:03:43,488 INFO MainThread:1549642 [wandb_init.py:init():892] starting backend +2026-05-19 22:03:43,705 INFO MainThread:1549642 [wandb_init.py:init():895] sending inform_init request +2026-05-19 22:03:43,715 INFO MainThread:1549642 [wandb_init.py:init():903] backend started and connected +2026-05-19 22:03:43,719 INFO MainThread:1549642 [wandb_init.py:init():973] updated telemetry +2026-05-19 22:03:43,720 INFO MainThread:1549642 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-19 22:03:44,862 INFO MainThread:1549642 [wandb_init.py:init():1042] starting run threads in backend +2026-05-19 22:03:45,056 INFO MainThread:1549642 [wandb_run.py:_console_start():2529] atexit reg +2026-05-19 22:03:45,056 INFO MainThread:1549642 [wandb_run.py:_redirect():2377] redirect: wrap_raw +2026-05-19 22:03:45,056 INFO MainThread:1549642 [wandb_run.py:_redirect():2446] Wrapping output streams. +2026-05-19 22:03:45,057 INFO MainThread:1549642 [wandb_run.py:_redirect():2469] Redirects installed. +2026-05-19 22:03:45,062 INFO MainThread:1549642 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-19 22:12:46,699 INFO wandb-AsyncioManager-main:1549642 [service_client.py:_forward_responses():94] Reached EOF. +2026-05-19 22:12:46,700 INFO wandb-AsyncioManager-main:1549642 [mailbox.py:close():154] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20260519_220343-bgm9lwvb/run-bgm9lwvb.wandb b/wandb/run-20260519_220343-bgm9lwvb/run-bgm9lwvb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..470f52cb8c2e531e069f56aa76674c8d6556bdd0 Binary files /dev/null and b/wandb/run-20260519_220343-bgm9lwvb/run-bgm9lwvb.wandb differ diff --git a/wandb/run-20260519_221333-qz452p60/files/config.yaml b/wandb/run-20260519_221333-qz452p60/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6d7ba97c80adee1e3195385be0e99e032760a68 --- /dev/null +++ b/wandb/run-20260519_221333-qz452p60/files/config.yaml @@ -0,0 +1,177 @@ +_wandb: + value: + cli_version: 0.24.2 + e: + vmpevcmipoelskpc50yuob073od6op6p: + args: + - --batch_size=64 + - --camera_set=rgb + - --dataset.repo_id=/home/seonho/insight/insight_dataset + - --dataset.root=/home/seonho/insight/insight_dataset + - --dataset.video_backend=pyav + - --log_freq=100 + - --num_workers=8 + - --output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt + - --phase=phase2 + - --policy.base_model_path=nvidia/GR00T-N1.5-3B + - --policy.lora_rank=0 + - --policy.tune_diffusion_model=true + - --policy.tune_llm=false + - --policy.tune_projector=true + - --policy.tune_visual=false + - --policy.type=groot + - --save_freq=50000 + - --steps=100000 + - --task_prompt_mode=non_guide + - --wandb.enable=true + - --wandb.entity=RwHlabs + - --wandb.group=insight_phase1_dit_only + - --wandb.name=insight_phase1_dit_only_groot + - --wandb.project=groot_insight_baseline_base + - --wandb.tags=base,phase2,groot,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base + codePath: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + codePathLocal: clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + cpu_count: 32 + cpu_count_logical: 64 + cudaVersion: "12.2" + disk: + /: + total: "3778130485248" + used: "3366444920832" + email: dankk98@gmail.com + executable: /home/seonho/miniconda3/envs/minje/bin/python3.12 + gpu: NVIDIA A100 80GB PCIe + gpu_count: 4 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-ca1c0319-e178-a3c3-ac41-386c2723deac + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100 80GB PCIe + uuid: GPU-47b6036e-4fde-e399-5515-f73eed752c6d + host: worker1 + memory: + total: "1081620840448" + os: Linux-6.8.0-101-generic-x86_64-with-glibc2.35 + program: /home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py + python: CPython 3.12.12 + root: /home/seonho/insight/outputs/dit_only_phase1_ckpt + startedAt: "2026-05-19T13:13:33.529003Z" + writerId: vmpevcmipoelskpc50yuob073od6op6p + m: [] + python_version: 3.12.12 + t: + "1": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "2": + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 71 + - 83 + - 98 + "3": + - 2 + - 13 + - 15 + - 16 + - 61 + - 62 + "4": 3.12.12 + "5": 0.24.2 + "6": 5.3.0 + "12": 0.24.2 + "13": linux-x86_64 +action_dim: + value: 8 +batch_size: + value: 64 +camera_set: + value: rgb +cl_v2_fm_loss_weight: + value: null +cl_v2_loss_weight: + value: null +cl_v2_phase: + value: null +cl_v2_trainable_mode: + value: null +dataset_len: + value: 246793 +dataset_root: + value: /home/seonho/insight/insight_dataset +effective_batch_size: + value: 64 +gradient_checkpointing: + value: false +lora_alpha: + value: 16 +lora_dropout: + value: 0.1 +lora_rank: + value: 0 +lora_target: + value: null +lr: + value: 0.0001 +method: + value: base +mgd_backprop_backbone: + value: null +mgd_enabled: + value: null +mgd_loss_weight: + value: null +mgd_trainable_mode: + value: null +num_processes: + value: 1 +phase: + value: phase2 +repo_id: + value: /home/seonho/insight/insight_dataset +seed: + value: 42 +state_dim: + value: 16 +steps: + value: 100000 +task_prompt_mode: + value: non_guide +tune_diffusion_model: + value: true +tune_llm: + value: false +tune_projector: + value: true +tune_visual: + value: false +video_feature_keys: + value: "" diff --git a/wandb/run-20260519_221333-qz452p60/files/output.log b/wandb/run-20260519_221333-qz452p60/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..cddaff1b95f17a2933a92594354a9295337aab5b --- /dev/null +++ b/wandb/run-20260519_221333-qz452p60/files/output.log @@ -0,0 +1,5 @@ +`use_fast` is set to `True` but the image processor class does not have a fast version. Falling back to the slow version. +The tokenizer you are loading from '/home/seonho/.cache/huggingface/lerobot/lerobot/eagle2hg-processor-groot-n1p5' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue. +The tokenizer you are loading from '/home/seonho/.cache/huggingface/lerobot/lerobot/eagle2hg-processor-groot-n1p5' with an incorrect regex pattern: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/discussions/84#69121093e8b480e709447d5e. This will lead to incorrect tokenization. You should set the `fix_mistral_regex=True` flag when loading this tokenizer to fix this issue. +wandb: Adding directory to artifact (/home/seonho/insight/outputs/dit_only_phase1_ckpt/checkpoints/050000)... Done. 18.2s +wandb: Adding directory to artifact (/home/seonho/insight/outputs/dit_only_phase1_ckpt/checkpoints/100000)... Done. 25.2s diff --git a/wandb/run-20260519_221333-qz452p60/files/requirements.txt b/wandb/run-20260519_221333-qz452p60/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8d89644db6c7aa4d72b923c55c62ae8132609a93 --- /dev/null +++ b/wandb/run-20260519_221333-qz452p60/files/requirements.txt @@ -0,0 +1,209 @@ +attrs==25.4.0 +kiwisolver==1.5.0 +setuptools==80.10.2 +transformers==5.3.0 +pynndescent==0.6.0 +filelock==3.25.2 +numba==0.61.2 +gymnasium==1.2.3 +shellingham==1.5.4 +dill==0.3.8 +ninja==1.13.0 +hidapi==0.15.0 +scikit-image==0.26.0 +orderly-set==5.5.0 +mypy_extensions==1.1.0 +stringzilla==4.6.1 +tianshou==0.4.10 +numpy==2.2.5 +xxhash==3.6.0 +protobuf==3.19.6 +numpydantic==1.6.7 +pyserial==3.5 +omegaconf==2.3.0 +Farama-Notifications==0.0.4 +nvidia-nccl-cu12==2.26.2 +plotly==6.6.0 +platformdirs==4.9.4 +nvidia-cufft-cu12==11.3.0.4 +eval_type_backport==0.3.1 +packaging==25.0 +typing-inspect==0.9.0 +opencv-python-headless==4.12.0.88 +cmeel-assimp==6.0.2 +lerobot==0.5.1 +pyyaml-include==1.4.1 +urllib3==2.6.3 +lxml==6.1.0 +peft==0.18.1 +virtualenv==21.2.4 +cmeel==0.59.0 +pre_commit==4.6.0 +importlib_metadata==8.7.1 +click==8.3.1 +accelerate==1.13.0 +multiprocess==0.70.16 +distlib==0.4.0 +websockets==16.0 +albumentations==1.4.18 +tokenizers==0.22.2 +nvidia-cusparselt-cu12==0.6.3 +certifi==2026.2.25 +annotated-doc==0.0.4 +triton==3.3.1 +libpinocchio==3.9.0 +mpmath==1.3.0 +decord==0.6.0 +matplotlib==3.10.8 +llvmlite==0.44.0 +python-xlib==0.33 +regex==2026.2.28 +colored==2.3.2 +charset-normalizer==3.4.5 +mdurl==0.1.2 +GitPython==3.1.46 +gym==0.26.2 +av==15.1.0 +markdown-it-py==4.0.0 +glfw==2.10.0 +nvidia-cufile-cu12==1.11.1.6 +PyYAML==6.0.3 +python-dateutil==2.9.0.post0 +cmeel-tinyxml2==10.0.0 +blinker==1.9.0 +opencv-python==4.13.0.92 +cmake==4.1.3 +gym-notices==0.1.0 +typing-inspection==0.4.2 +timm==1.0.25 +torchcodec==0.5 +cmeel-octomap==1.10.0 +umap-learn==0.5.11 +MarkupSafe==3.0.3 +pluggy==1.6.0 +smmap==5.0.3 +fsspec==2025.3.0 +evdev==1.9.3 +python-discovery==1.2.2 +hf-xet==1.5.0 +scikit-learn==1.8.0 +h5py==3.16.0 +cmeel-console-bridge==1.0.2.3 +narwhals==2.18.1 +msgpack==1.1.2 +flash_attn==2.8.3 +ansitable==0.11.5 +pyarrow==23.0.1 +h11==0.16.0 +mergedeep==1.3.4 +tqdm==4.67.3 +vla-eval==0.1.1.dev27+g195d46338 +nvidia-nvjitlink-cu12==12.6.85 +cmeel-qhull==8.0.2.1 +libcoal==3.0.2 +cmeel-boost==1.89.0 +rtb-data==1.0.1 +mujoco==3.3.1 +nvidia-cuda-nvrtc-cu12==12.6.77 +PyOpenGL==3.1.10 +draccus==0.10.0 +pip==26.0.1 +threadpoolctl==3.6.0 +annotated-types==0.7.0 +anyio==4.12.1 +imageio-ffmpeg==0.6.0 +qpsolvers==4.11.0 +wrapt==2.1.2 +networkx==3.6.1 +contourpy==1.3.3 +Werkzeug==3.1.8 +tensorboard-data-server==0.7.2 +gitdb==4.0.12 +nvidia-cusparse-cu12==12.5.4.2 +typer==0.24.1 +toml==0.10.2 +tensorboard==2.20.0 +rerun-sdk==0.26.2 +diffusers==0.35.2 +albucore==0.0.17 +wandb==0.24.2 +nvidia-cublas-cu12==12.6.4.1 +nvidia-cuda-runtime-cu12==12.6.77 +cycler==0.12.1 +coal==3.0.2 +pydantic_core==2.27.2 +torch==2.7.1 +nvidia-cuda-cupti-cu12==12.6.80 +Pygments==2.19.2 +pytest==9.0.3 +ImageIO==2.37.3 +sentry-sdk==2.54.0 +antlr4-python3-runtime==4.9.3 +sympy==1.14.0 +safetensors==0.7.0 +cmeel-zlib==1.3.1 +einops==0.8.2 +spatialgeometry==1.1.0 +pygame==2.6.1 +absl-py==2.4.0 +pandas==3.0.1 +httpx==0.28.1 +robosuite==1.5.2 +fonttools==4.62.1 +progress==1.6.1 +nvidia-cusolver-cu12==11.7.1.2 +aiohttp==3.13.3 +frozenlist==1.8.0 +deepdiff==8.6.1 +yarl==1.23.0 +nvidia-cudnn-cu12==9.5.1.17 +torchvision==0.22.1 +idna==3.11 +lazy-loader==0.5 +termcolor==3.3.0 +nvidia-curand-cu12==10.3.7.77 +multidict==6.7.1 +Markdown==3.10.2 +pynput==1.8.1 +propcache==0.4.1 +zipp==3.23.0 +nvidia-nvtx-cu12==12.6.77 +Flask==3.1.3 +httpcore==1.0.9 +etils==1.14.0 +psutil==7.2.2 +pillow==12.1.1 +nodeenv==1.10.0 +pin==3.9.0 +quadprog==0.1.13 +eigenpy==3.12.0 +wheel==0.46.3 +simsimd==6.5.16 +itsdangerous==2.2.0 +aiohappyeyeballs==2.6.1 +cfgv==3.5.0 +cloudpickle==3.1.2 +iniconfig==2.3.0 +requests==2.32.5 +tifffile==2026.5.2 +joblib==1.5.3 +identify==2.6.19 +huggingface_hub==1.14.0 +datasets==4.8.4 +typing_extensions==4.15.0 +pydantic==2.10.6 +jsonlines==4.0.0 +swift-sim==1.1.0 +lazyregistry==0.4.0 +robocasa==1.0.0 +pgraph-python==0.6.3 +Jinja2==3.1.6 +aiosignal==1.4.0 +dm-tree==0.1.9 +six==1.17.0 +grpcio==1.80.0 +rich==14.3.3 +scipy==1.15.3 +cmeel-urdfdom==4.0.1 +pyparsing==3.3.2 +lerobot==0.5.1 diff --git a/wandb/run-20260519_221333-qz452p60/files/wandb-metadata.json b/wandb/run-20260519_221333-qz452p60/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5b74fb1d7ef8b9ec11bc8ac0e20e0f7aa16d8cbe --- /dev/null +++ b/wandb/run-20260519_221333-qz452p60/files/wandb-metadata.json @@ -0,0 +1,84 @@ +{ + "os": "Linux-6.8.0-101-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.12", + "startedAt": "2026-05-19T13:13:33.529003Z", + "args": [ + "--batch_size=64", + "--camera_set=rgb", + "--dataset.repo_id=/home/seonho/insight/insight_dataset", + "--dataset.root=/home/seonho/insight/insight_dataset", + "--dataset.video_backend=pyav", + "--log_freq=100", + "--num_workers=8", + "--output_dir=/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "--phase=phase2", + "--policy.base_model_path=nvidia/GR00T-N1.5-3B", + "--policy.lora_rank=0", + "--policy.tune_diffusion_model=true", + "--policy.tune_llm=false", + "--policy.tune_projector=true", + "--policy.tune_visual=false", + "--policy.type=groot", + "--save_freq=50000", + "--steps=100000", + "--task_prompt_mode=non_guide", + "--wandb.enable=true", + "--wandb.entity=RwHlabs", + "--wandb.group=insight_phase1_dit_only", + "--wandb.name=insight_phase1_dit_only_groot", + "--wandb.project=groot_insight_baseline_base", + "--wandb.tags=base,phase2,groot,rgb,non_guide,insight,insight_native,cartesian_8d,phase1,baseline,dit_only,nvidia_base" + ], + "program": "/home/seonho/clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePath": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "codePathLocal": "clvla/lerobot_cl/src/lerobot/scripts/train_groot_insight_yaml.py", + "email": "dankk98@gmail.com", + "root": "/home/seonho/insight/outputs/dit_only_phase1_ckpt", + "host": "worker1", + "executable": "/home/seonho/miniconda3/envs/minje/bin/python3.12", + "cpu_count": 32, + "cpu_count_logical": 64, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 4, + "disk": { + "/": { + "total": "3778130485248", + "used": "3366444920832" + } + }, + "memory": { + "total": "1081620840448" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-0dd6d491-2dd8-24ac-bb27-872483ab77e9" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-dbfb7925-aec1-50fb-039a-3a4c0a4b2caa" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-ca1c0319-e178-a3c3-ac41-386c2723deac" + }, + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-47b6036e-4fde-e399-5515-f73eed752c6d" + } + ], + "cudaVersion": "12.2", + "writerId": "vmpevcmipoelskpc50yuob073od6op6p" +} \ No newline at end of file diff --git a/wandb/run-20260519_221333-qz452p60/files/wandb-summary.json b/wandb/run-20260519_221333-qz452p60/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d63d57750a15dc944bd5f92d98b27f6d1320df04 --- /dev/null +++ b/wandb/run-20260519_221333-qz452p60/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/step_per_s":0.33431204184380825,"params/whole_total":2724163520,"_timestamp":1.779511947226162e+09,"_runtime":315596.070997787,"train/policy_update_called":0,"params/whole_trainable":1068806144,"_step":100000,"train/step_s":2.9912174101918936,"_wandb":{"runtime":315596},"train/total_step_s":2.9912174101918936,"train/lr":0,"train/grad_norm":1.2890625,"train/flow_matching_loss":0.06392817199230194,"train/loss":0.06392817199230194,"train/data_s":1.6900663105770946,"train/update_s":1.301151099614799} \ No newline at end of file diff --git a/wandb/run-20260519_221333-qz452p60/logs/debug-core.log b/wandb/run-20260519_221333-qz452p60/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..1878af307628e18113b8eda81aebad605d6142da --- /dev/null +++ b/wandb/run-20260519_221333-qz452p60/logs/debug-core.log @@ -0,0 +1,19 @@ +{"time":"2026-05-19T22:13:33.560255862+09:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpdz1_lm_i/port-3941155.txt","pid":3941155,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-05-19T22:13:33.561545668+09:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":3941155} +{"time":"2026-05-19T22:13:33.561517274+09:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3941155-3969559-3702025772/socket","Net":"unix"}} +{"time":"2026-05-19T22:13:33.746096923+09:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-05-19T22:13:33.757376606+09:00","level":"INFO","msg":"handleInformInit: received","streamId":"qz452p60","id":"1(@)"} +{"time":"2026-05-19T22:13:34.250141168+09:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"qz452p60","id":"1(@)"} +{"time":"2026-05-19T22:13:40.251100106+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"balup2j7cpqn"} +{"time":"2026-05-23T13:53:31.188193598+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"balup2j7cpqn"} +{"time":"2026-05-23T14:32:40.265804248+09:00","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"balup2j7cpqn"} +{"time":"2026-05-23T14:32:40.266542453+09:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"qz452p60","id":"1(@)"} +{"time":"2026-05-23T14:32:40.269150824+09:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"qz452p60","id":"1(@)"} +{"time":"2026-05-23T14:32:40.674864495+09:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-05-23T14:32:40.674904393+09:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-05-23T14:32:40.674912044+09:00","level":"INFO","msg":"server is shutting down"} +{"time":"2026-05-23T14:32:40.674915419+09:00","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-05-23T14:32:40.674981712+09:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-05-23T14:32:40.674992165+09:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-05-23T14:32:40.674975463+09:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3941155-3969559-3702025772/socket","Net":"unix"}} +{"time":"2026-05-23T14:32:40.675006382+09:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20260519_221333-qz452p60/logs/debug-internal.log b/wandb/run-20260519_221333-qz452p60/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..270ee03a66ab99851e9429a03aac4d728af8cc98 --- /dev/null +++ b/wandb/run-20260519_221333-qz452p60/logs/debug-internal.log @@ -0,0 +1,41 @@ +{"time":"2026-05-19T22:13:33.757508612+09:00","level":"INFO","msg":"stream: starting","core version":"0.24.2"} +{"time":"2026-05-19T22:13:34.249681253+09:00","level":"INFO","msg":"stream: created new stream","id":"qz452p60"} +{"time":"2026-05-19T22:13:34.249816267+09:00","level":"INFO","msg":"handler: started","stream_id":"qz452p60"} +{"time":"2026-05-19T22:13:34.250122028+09:00","level":"INFO","msg":"stream: started","id":"qz452p60"} +{"time":"2026-05-19T22:13:34.250159759+09:00","level":"INFO","msg":"sender: started","stream_id":"qz452p60"} +{"time":"2026-05-19T22:13:34.250156101+09:00","level":"INFO","msg":"writer: started","stream_id":"qz452p60"} +{"time":"2026-05-20T15:55:10.818156537+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T15:56:44.994984532+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T15:58:23.494572793+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T17:32:56.207023413+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T17:33:53.844718799+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T17:34:53.908902815+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T17:35:57.323694482+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:30:49.563054911+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:48:36.484690792+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:49:38.236616601+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:51:49.732015936+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T18:59:43.774277496+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:03:49.908408418+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:04:48.543201774+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:05:45.963581069+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:07:50.284610418+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:12:30.299479148+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:14:13.564300748+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:15:14.101427763+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:16:15.40790845+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:29:32.060104195+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:31:30.297052013+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:49:26.161299534+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T19:50:23.803656458+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T20:04:29.31970088+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T20:49:42.028043576+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-20T20:50:44.339074594+09:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/RwHlabs/groot_insight_baseline_base/qz452p60/file_stream","body":"{\"error\":\"context deadline exceeded\"}"} +{"time":"2026-05-23T13:56:03.488569868+09:00","level":"INFO","msg":"api: retrying error","error":"unexpected HTTP status 502 Bad Gateway"} +{"time":"2026-05-23T14:25:43.511326685+09:00","level":"INFO","msg":"api: retrying error","error":"unexpected HTTP status 502 Bad Gateway"} +{"time":"2026-05-23T14:32:39.967322696+09:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-05-23T14:32:40.260656129+09:00","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-05-23T14:32:40.266561008+09:00","level":"INFO","msg":"stream: closing","id":"qz452p60"} +{"time":"2026-05-23T14:32:40.266576703+09:00","level":"INFO","msg":"handler: closed","stream_id":"qz452p60"} +{"time":"2026-05-23T14:32:40.266689508+09:00","level":"INFO","msg":"sender: closed","stream_id":"qz452p60"} +{"time":"2026-05-23T14:32:40.266726133+09:00","level":"INFO","msg":"stream: closed","id":"qz452p60"} diff --git a/wandb/run-20260519_221333-qz452p60/logs/debug.log b/wandb/run-20260519_221333-qz452p60/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..87f2bab93a1a119919fbccca18966bebfc447aaa --- /dev/null +++ b/wandb/run-20260519_221333-qz452p60/logs/debug.log @@ -0,0 +1,24 @@ +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_setup.py:_flush():81] Current SDK version is 0.24.2 +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_setup.py:_flush():81] Configure stats pid to 3941155 +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260519_221333-qz452p60/logs/debug.log +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/seonho/insight/outputs/dit_only_phase1_ckpt/wandb/run-20260519_221333-qz452p60/logs/debug-internal.log +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:init():844] calling init triggers +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:init():849] wandb.init called with sweep_config: {} +config: {'repo_id': '/home/seonho/insight/insight_dataset', 'dataset_root': '/home/seonho/insight/insight_dataset', 'method': 'base', 'phase': 'phase2', 'steps': 100000, 'lr': 0.0001, 'batch_size': 64, 'num_processes': 1, 'effective_batch_size': 64, 'action_dim': 8, 'state_dim': 16, 'camera_set': 'rgb', 'video_feature_keys': '', 'task_prompt_mode': 'non_guide', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_target': None, 'mgd_enabled': None, 'mgd_trainable_mode': None, 'mgd_loss_weight': None, 'mgd_backprop_backbone': None, 'cl_v2_phase': None, 'cl_v2_trainable_mode': None, 'cl_v2_loss_weight': None, 'cl_v2_fm_loss_weight': None, 'gradient_checkpointing': False, 'seed': 42, 'dataset_len': 246793, '_wandb': {}} +2026-05-19 22:13:33,530 INFO MainThread:3941155 [wandb_init.py:init():892] starting backend +2026-05-19 22:13:33,746 INFO MainThread:3941155 [wandb_init.py:init():895] sending inform_init request +2026-05-19 22:13:33,753 INFO MainThread:3941155 [wandb_init.py:init():903] backend started and connected +2026-05-19 22:13:33,755 INFO MainThread:3941155 [wandb_init.py:init():973] updated telemetry +2026-05-19 22:13:33,756 INFO MainThread:3941155 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout +2026-05-19 22:13:35,113 INFO MainThread:3941155 [wandb_init.py:init():1042] starting run threads in backend +2026-05-19 22:13:35,241 INFO MainThread:3941155 [wandb_run.py:_console_start():2529] atexit reg +2026-05-19 22:13:35,241 INFO MainThread:3941155 [wandb_run.py:_redirect():2377] redirect: wrap_raw +2026-05-19 22:13:35,241 INFO MainThread:3941155 [wandb_run.py:_redirect():2446] Wrapping output streams. +2026-05-19 22:13:35,241 INFO MainThread:3941155 [wandb_run.py:_redirect():2469] Redirects installed. +2026-05-19 22:13:35,246 INFO MainThread:3941155 [wandb_init.py:init():1082] run started, returning control to user process +2026-05-23 13:53:31,185 INFO MainThread:3941155 [wandb_run.py:_finish():2295] finishing run RwHlabs/groot_insight_baseline_base/qz452p60 +2026-05-23 13:53:31,187 INFO MainThread:3941155 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0 +2026-05-23 13:53:31,187 INFO MainThread:3941155 [wandb_run.py:_restore():2476] restore +2026-05-23 13:53:31,187 INFO MainThread:3941155 [wandb_run.py:_restore():2482] restore done +2026-05-23 14:32:40,265 INFO MainThread:3941155 [wandb_run.py:_footer_sync_info():3871] logging synced files diff --git a/wandb/run-20260519_221333-qz452p60/run-qz452p60.wandb b/wandb/run-20260519_221333-qz452p60/run-qz452p60.wandb new file mode 100644 index 0000000000000000000000000000000000000000..cc2e188bc397d63dde517bc3dd1212e6cf5cce39 --- /dev/null +++ b/wandb/run-20260519_221333-qz452p60/run-qz452p60.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:995b1dc497f7e287888d27d0026d5d4933b87b94be3f26267219b050324d2a05 +size 43646553