diff --git a/Abs_6D + DeltaEE/Baseline.interrupted_reboot_20260719T051924/wandb/run-20260717_154331-qi6m7u1i/files/wandb-metadata.json b/Abs_6D + DeltaEE/Baseline.interrupted_reboot_20260719T051924/wandb/run-20260717_154331-qi6m7u1i/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..dfb40cfd78dabbba0b1191a4e96ff1d44511dead --- /dev/null +++ b/Abs_6D + DeltaEE/Baseline.interrupted_reboot_20260719T051924/wandb/run-20260717_154331-qi6m7u1i/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.13", + "startedAt": "2026-07-17T06:43:31.291595Z", + "args": [ + "--config_path=/tmp/insight_lerobot_cfg_s5s50f4t/baseline.yaml", + "--batch_size=128" + ], + "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", + "codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", + "codePathLocal": "my_scripts/lerobot_train_yaml.py", + "git": { + "remote": "git@github.com:paragon7060/clvla.git", + "commit": "a970d9f98f189faccce13cf2fb61ccf5d55921f3" + }, + "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/Baseline", + "host": "DGX-H200-01", + "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888425144320", + "used": "1493419012096" + } + }, + "memory": { + "total": "2164177248256" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c" + } + ], + "cudaVersion": "12.9", + "writerId": "e0110vxhdtw15tyv9b5bffn6whljiwiw" +} \ No newline at end of file diff --git a/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K.interrupted_reboot_20260719T051924/wandb/run-20260717_154331-e5avxg6w/files/output.log b/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K.interrupted_reboot_20260719T051924/wandb/run-20260717_154331-e5avxg6w/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..471f7c485acb051653e0b40c0362bf4c9f86284b --- /dev/null +++ b/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K.interrupted_reboot_20260719T051924/wandb/run-20260717_154331-e5avxg6w/files/output.log @@ -0,0 +1,217 @@ +INFO 2026-07-17 15:43:33 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-17 15:43:33 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/e5avxg6w +INFO 2026-07-17 15:43:33 ot_train.py:298 Creating dataset +INFO 2026-07-17 15:43:36 ot_train.py:332 Creating policy +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 1539.30it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 1211.10it/s] +INFO 2026-07-17 15:43:43 ot_train.py:405 Creating optimizer and scheduler +INFO 2026-07-17 15:43:43 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K +INFO 2026-07-17 15:43:43 ot_train.py:446 cfg.steps=60000 (60K) +INFO 2026-07-17 15:43:43 ot_train.py:447 dataset.num_frames=218367 (218K) +INFO 2026-07-17 15:43:43 ot_train.py:448 dataset.num_episodes=4930 +INFO 2026-07-17 15:43:43 ot_train.py:451 Effective batch size: 128 x 1 = 128 +INFO 2026-07-17 15:43:43 ot_train.py:452 num_learnable_params=1622090881 (2B) +INFO 2026-07-17 15:43:43 ot_train.py:453 num_total_params=3145712033 (3B) +Training: 0%| | 0/60000 [00:00, 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 6000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 6000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}} +2026-07-19 23:04:35,723 INFO MainThread:2036850 [wandb_init.py:init():814] starting backend +2026-07-19 23:04:35,947 INFO MainThread:2036850 [wandb_init.py:init():829] sending inform_init request +2026-07-19 23:04:36,641 INFO MainThread:2036850 [wandb_init.py:init():834] backend started and connected +2026-07-19 23:04:36,645 INFO MainThread:2036850 [wandb_init.py:init():904] updated telemetry +2026-07-19 23:04:36,657 INFO MainThread:2036850 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-19 23:04:37,058 INFO MainThread:2036850 [wandb_init.py:init():972] starting run threads in backend +2026-07-19 23:04:37,260 INFO MainThread:2036850 [wandb_run.py:_console_start():2523] atexit reg +2026-07-19 23:04:37,260 INFO MainThread:2036850 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-19 23:04:37,260 INFO MainThread:2036850 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-19 23:04:37,260 INFO MainThread:2036850 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-19 23:04:37,264 INFO MainThread:2036850 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/files/output.log b/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ee974197763fa333d05791e187b810f5e1cf7842 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K/wandb/run-20260719_230435-7ob3zmbd/files/output.log @@ -0,0 +1,17 @@ +INFO 2026-07-19 23:04:37 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-19 23:04:37 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/7ob3zmbd +INFO 2026-07-19 23:04:37 ot_train.py:298 Creating dataset +INFO 2026-07-19 23:04:41 ot_train.py:332 Creating policy +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 2422.64it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 2053.55it/s] +INFO 2026-07-19 23:04:46 ot_train.py:405 Creating optimizer and scheduler +INFO 2026-07-19 23:04:46 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_10K +INFO 2026-07-19 23:04:46 ot_train.py:446 cfg.steps=60000 (60K) +INFO 2026-07-19 23:04:46 ot_train.py:447 dataset.num_frames=218367 (218K) +INFO 2026-07-19 23:04:46 ot_train.py:448 dataset.num_episodes=4930 +INFO 2026-07-19 23:04:46 ot_train.py:451 Effective batch size: 128 x 1 = 128 +INFO 2026-07-19 23:04:46 ot_train.py:452 num_learnable_params=1622090881 (2B) +INFO 2026-07-19 23:04:46 ot_train.py:453 num_total_params=3145712033 (3B) +Training: 0%| | 0/60000 [00:00 https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/bvj1u4hd +INFO 2026-07-19 23:04:37 ot_train.py:298 Creating dataset +INFO 2026-07-19 23:04:41 ot_train.py:332 Creating policy +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 4662.26it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 1751.88it/s] +INFO 2026-07-19 23:04:46 ot_train.py:405 Creating optimizer and scheduler +INFO 2026-07-19 23:04:46 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/LAPstyle_linear_6K +INFO 2026-07-19 23:04:46 ot_train.py:446 cfg.steps=60000 (60K) +INFO 2026-07-19 23:04:46 ot_train.py:447 dataset.num_frames=218367 (218K) +INFO 2026-07-19 23:04:46 ot_train.py:448 dataset.num_episodes=4930 +INFO 2026-07-19 23:04:46 ot_train.py:451 Effective batch size: 128 x 1 = 128 +INFO 2026-07-19 23:04:46 ot_train.py:452 num_learnable_params=1622090881 (2B) +INFO 2026-07-19 23:04:46 ot_train.py:453 num_total_params=3145712033 (3B) +Training: 0%| | 0/60000 [00:00, 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}} +2026-07-19 23:04:35,707 INFO MainThread:2036849 [wandb_init.py:init():814] starting backend +2026-07-19 23:04:35,932 INFO MainThread:2036849 [wandb_init.py:init():829] sending inform_init request +2026-07-19 23:04:36,613 INFO MainThread:2036849 [wandb_init.py:init():834] backend started and connected +2026-07-19 23:04:36,615 INFO MainThread:2036849 [wandb_init.py:init():904] updated telemetry +2026-07-19 23:04:36,623 INFO MainThread:2036849 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-19 23:04:37,083 INFO MainThread:2036849 [wandb_init.py:init():972] starting run threads in backend +2026-07-19 23:04:37,250 INFO MainThread:2036849 [wandb_run.py:_console_start():2523] atexit reg +2026-07-19 23:04:37,250 INFO MainThread:2036849 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-19 23:04:37,250 INFO MainThread:2036849 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-19 23:04:37,251 INFO MainThread:2036849 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-19 23:04:37,254 INFO MainThread:2036849 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/run-20260719_230435-5e3ytmnu/files/output.log b/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/run-20260719_230435-5e3ytmnu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c3963e8af94e923892219de9799623ce248279e7 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K/wandb/run-20260719_230435-5e3ytmnu/files/output.log @@ -0,0 +1,17 @@ +INFO 2026-07-19 23:04:37 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-19 23:04:37 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/5e3ytmnu +INFO 2026-07-19 23:04:37 ot_train.py:298 Creating dataset +INFO 2026-07-19 23:04:39 ot_train.py:332 Creating policy +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 1496.60it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 1886.97it/s] +INFO 2026-07-19 23:04:45 ot_train.py:405 Creating optimizer and scheduler +INFO 2026-07-19 23:04:45 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_230419_rerun/RSCLstyle_cosine_60K +INFO 2026-07-19 23:04:45 ot_train.py:446 cfg.steps=60000 (60K) +INFO 2026-07-19 23:04:45 ot_train.py:447 dataset.num_frames=218367 (218K) +INFO 2026-07-19 23:04:45 ot_train.py:448 dataset.num_episodes=4930 +INFO 2026-07-19 23:04:45 ot_train.py:451 Effective batch size: 128 x 1 = 128 +INFO 2026-07-19 23:04:45 ot_train.py:452 num_learnable_params=1622090881 (2B) +INFO 2026-07-19 23:04:45 ot_train.py:453 num_total_params=3145712033 (3B) +Training: 0%| | 0/60000 [00:00, 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}} +2026-07-19 23:14:56,709 INFO MainThread:2499829 [wandb_init.py:init():814] starting backend +2026-07-19 23:14:56,933 INFO MainThread:2499829 [wandb_init.py:init():829] sending inform_init request +2026-07-19 23:14:57,546 INFO MainThread:2499829 [wandb_init.py:init():834] backend started and connected +2026-07-19 23:14:57,551 INFO MainThread:2499829 [wandb_init.py:init():904] updated telemetry +2026-07-19 23:14:57,560 INFO MainThread:2499829 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-19 23:14:57,962 INFO MainThread:2499829 [wandb_init.py:init():972] starting run threads in backend +2026-07-19 23:14:58,054 INFO MainThread:2499829 [wandb_run.py:_console_start():2523] atexit reg +2026-07-19 23:14:58,054 INFO MainThread:2499829 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-19 23:14:58,054 INFO MainThread:2499829 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-19 23:14:58,054 INFO MainThread:2499829 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-19 23:14:58,057 INFO MainThread:2499829 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/Baseline/wandb/run-20260719_231456-eexlcl2l/run-eexlcl2l.wandb b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/Baseline/wandb/run-20260719_231456-eexlcl2l/run-eexlcl2l.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e39f1734b7765e36516411f367ed00e80038e1bc Binary files /dev/null and b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/Baseline/wandb/run-20260719_231456-eexlcl2l/run-eexlcl2l.wandb differ diff --git a/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_10K/wandb/run-20260719_231457-45ifc2tf/files/output.log b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_10K/wandb/run-20260719_231457-45ifc2tf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..703e68cbd34c6d523412ea62966d90a194d05459 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_10K/wandb/run-20260719_231457-45ifc2tf/files/output.log @@ -0,0 +1,17 @@ +INFO 2026-07-19 23:14:58 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-19 23:14:58 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/45ifc2tf +INFO 2026-07-19 23:14:58 ot_train.py:298 Creating dataset +INFO 2026-07-19 23:15:01 ot_train.py:332 Creating policy +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 5772.86it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 2319.58it/s] +INFO 2026-07-19 23:15:12 ot_train.py:405 Creating optimizer and scheduler +INFO 2026-07-19 23:15:12 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_10K +INFO 2026-07-19 23:15:12 ot_train.py:446 cfg.steps=60000 (60K) +INFO 2026-07-19 23:15:12 ot_train.py:447 dataset.num_frames=218367 (218K) +INFO 2026-07-19 23:15:12 ot_train.py:448 dataset.num_episodes=4930 +INFO 2026-07-19 23:15:12 ot_train.py:451 Effective batch size: 128 x 1 = 128 +INFO 2026-07-19 23:15:12 ot_train.py:452 num_learnable_params=1622090881 (2B) +INFO 2026-07-19 23:15:12 ot_train.py:453 num_total_params=3145712033 (3B) +Training: 0%| | 0/60000 [00:00, 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 6000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 6000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}} +2026-07-19 23:14:55,951 INFO MainThread:2499832 [wandb_init.py:init():814] starting backend +2026-07-19 23:14:56,174 INFO MainThread:2499832 [wandb_init.py:init():829] sending inform_init request +2026-07-19 23:14:56,819 INFO MainThread:2499832 [wandb_init.py:init():834] backend started and connected +2026-07-19 23:14:56,821 INFO MainThread:2499832 [wandb_init.py:init():904] updated telemetry +2026-07-19 23:14:56,829 INFO MainThread:2499832 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-19 23:14:57,227 INFO MainThread:2499832 [wandb_init.py:init():972] starting run threads in backend +2026-07-19 23:14:57,421 INFO MainThread:2499832 [wandb_run.py:_console_start():2523] atexit reg +2026-07-19 23:14:57,421 INFO MainThread:2499832 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-19 23:14:57,422 INFO MainThread:2499832 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-19 23:14:57,422 INFO MainThread:2499832 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-19 23:14:57,427 INFO MainThread:2499832 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/files/requirements.txt b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3f670ee54ec46434506491c5853221aa2836efa4 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/files/requirements.txt @@ -0,0 +1,117 @@ +pyarrow==25.0.0 +importlib_metadata==9.0.0 +wheel==0.47.0 +cloudpickle==3.1.2 +nvidia-cusolver-cu12==11.7.3.90 +multiprocess==0.70.19 +torch==2.11.0+cu128 +nvidia-nvjitlink-cu12==12.8.93 +fsspec==2026.2.0 +pytz==2026.2 +hf-xet==1.5.1 +tokenizers==0.22.2 +opencv-python-headless==4.13.0.92 +cuda-pathfinder==1.2.2 +frozenlist==1.8.0 +smmap==5.0.3 +httpcore==1.0.9 +nvidia-nvshmem-cu12==3.4.5 +nvidia-cuda-nvrtc-cu12==12.8.93 +annotated-types==0.7.0 +platformdirs==4.10.0 +click==8.4.2 +pyyaml-include==1.4.1 +MarkupSafe==3.0.3 +Jinja2==3.1.6 +urllib3==2.7.0 +nvidia-curand-cu12==10.3.9.90 +certifi==2026.6.17 +nvidia-cuda-runtime-cu12==12.8.90 +torchcodec==0.11.1 +toml==0.10.2 +av==15.1.0 +nvidia-cufft-cu12==11.3.3.83 +attrs==26.1.0 +lerobot==0.6.0 +PyYAML==6.0.3 +safetensors==0.8.0 +diffusers==0.35.2 +requests==2.34.2 +nvidia-nvtx-cu12==12.8.90 +charset-normalizer==3.4.9 +networkx==3.6.1 +cmake==4.1.3 +pydantic_core==2.46.4 +timm==1.0.28 +dm-tree==0.1.10 +nvidia-cufile-cu12==1.13.1.3 +termcolor==3.3.0 +multidict==6.7.1 +filelock==3.29.0 +tzdata==2026.3 +aiohappyeyeballs==2.7.1 +httpx==0.28.1 +nvidia-nccl-cu12==2.28.9 +Pygments==2.20.0 +einops==0.8.2 +nvidia-cusparse-cu12==12.5.8.93 +torchvision==0.26.0+cu128 +dill==0.4.1 +GitPython==3.1.51 +nvidia-cuda-cupti-cu12==12.8.90 +decord==0.6.0 +sympy==1.14.0 +cuda-toolkit==12.8.1 +python-dateutil==2.9.0.post0 +shellingham==1.5.4 +propcache==0.5.2 +Farama-Notifications==0.0.6 +pillow==12.2.0 +cuda-bindings==12.9.4 +huggingface_hub==1.23.0 +pandas==2.3.3 +typing-inspect==0.9.0 +packaging==25.0 +rich==15.0.0 +nvidia-cusparselt-cu12==0.7.1 +setuptools==78.1.0 +wrapt==2.2.2 +mypy_extensions==1.1.0 +tqdm==4.68.4 +draccus==0.10.0 +aiosignal==1.4.0 +mpmath==1.3.0 +numpy==2.2.6 +typing_extensions==4.15.0 +psutil==7.2.2 +mdurl==0.1.2 +nvidia-cublas-cu12==12.8.4.1 +absl-py==2.5.0 +gitdb==4.0.12 +transformers==5.5.4 +accelerate==1.14.0 +pydantic==2.13.4 +pip==26.1.2 +markdown-it-py==4.2.0 +typer==0.26.8 +triton==3.6.0 +wandb==0.27.2 +regex==2026.7.10 +yarl==1.24.2 +jsonlines==4.0.0 +mergedeep==1.3.4 +aiohttp==3.14.1 +datasets==4.8.5 +gymnasium==1.3.0 +peft==0.19.1 +zipp==4.1.0 +idna==3.18 +nvidia-cudnn-cu12==9.19.0.56 +h11==0.16.0 +sentry-sdk==2.65.0 +annotated-doc==0.0.4 +protobuf==7.35.1 +typing-inspection==0.4.2 +anyio==4.14.2 +xxhash==3.8.1 +six==1.17.0 diff --git a/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/files/wandb-metadata.json b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a4dc2dbdeeac2142f43986afc2afdff551ad2e3f --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/files/wandb-metadata.json @@ -0,0 +1,91 @@ +{ + "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.13", + "startedAt": "2026-07-19T14:14:55.949118Z", + "args": [ + "--config_path=/tmp/insight_lerobot_cfg_q00gsgfc/LAPstyle_RKD_linear_decay_6K.yaml", + "--batch_size=128", + "--output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K" + ], + "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", + "git": { + "remote": "git@github.com:paragon7060/clvla.git", + "commit": "a9df08f7da2d8eb1fa7f28708b4eb6f6dc81b1f9" + }, + "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K", + "host": "DGX-H200-01", + "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888425144320", + "used": "814088617984" + } + }, + "memory": { + "total": "2164177309696" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c" + } + ], + "cudaVersion": "12.9", + "writerId": "9q8p3lipql5xeglk51zsoqxc6syi7cwe" +} \ No newline at end of file diff --git a/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/logs/debug-internal.log b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0944fa3c7ba7cbaf4cc5b50948c1e6c2536af1ae --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/logs/debug-internal.log @@ -0,0 +1,21 @@ +{"time":"2026-07-19T23:14:56.183101846+09:00","level":"INFO","msg":"wandb-core"} +{"time":"2026-07-19T23:14:56.183574889+09:00","level":"INFO","msg":"stream: starting","core version":"0.27.2"} +{"time":"2026-07-19T23:14:56.817463847+09:00","level":"INFO","msg":"stream: created new stream","id":"je3hccha"} +{"time":"2026-07-19T23:14:56.817628188+09:00","level":"INFO","msg":"handler: started"} +{"time":"2026-07-19T23:14:56.817725968+09:00","level":"INFO","msg":"stream: started"} +{"time":"2026-07-19T23:14:56.817748308+09:00","level":"INFO","msg":"writer: started","stream_id":"je3hccha"} +{"time":"2026-07-19T23:14:56.817777641+09:00","level":"INFO","msg":"sender: started"} +{"time":"2026-07-19T23:14:57.43025606+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1} +{"time":"2026-07-19T23:14:57.773819871+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-19T23:15:12.430460323+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":1,"console_lines":15,"uploaded_len":2} +{"time":"2026-07-19T23:15:12.841171644+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-19T23:15:27.43092437+09:00","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":1,"events_lines":2} +{"time":"2026-07-19T23:15:27.858025516+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-19T23:15:42.431234458+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":3,"events_lines":2,"console_offset":16,"console_lines":1} +{"time":"2026-07-19T23:15:42.726714207+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-19T23:15:57.43130328+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":5,"events_lines":2,"console_offset":16,"console_lines":1} +{"time":"2026-07-19T23:15:57.955844696+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-19T23:16:12.430569375+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":7,"events_lines":2,"console_offset":16,"console_lines":1} +{"time":"2026-07-19T23:16:12.826149518+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"} +{"time":"2026-07-19T23:16:27.430653119+09:00","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":9,"events_lines":2,"console_offset":16,"console_lines":1} +{"time":"2026-07-19T23:16:27.691305057+09:00","level":"INFO","msg":"filestream: request sent","status":"200 OK"} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/logs/debug.log b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e520d723a8731136ce178bdb0f04e70fffa82d5f --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/logs/debug.log @@ -0,0 +1,19 @@ +2026-07-19 23:14:55,950 INFO MainThread:2499832 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2 +2026-07-19 23:14:55,950 INFO MainThread:2499832 [wandb_setup.py:_flush():81] Configure stats pid to 2499832 +2026-07-19 23:14:55,950 INFO MainThread:2499832 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-07-19 23:14:55,950 INFO MainThread:2499832 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/logs/debug.log +2026-07-19 23:14:55,950 INFO MainThread:2499832 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K/wandb/run-20260719_231455-je3hccha/logs/debug-internal.log +2026-07-19 23:14:55,950 INFO MainThread:2499832 [wandb_init.py:init():766] calling init triggers +2026-07-19 23:14:55,950 INFO MainThread:2499832 [wandb_init.py:init():771] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 6000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/LAPstyle_linear_6K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 6000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}} +2026-07-19 23:14:55,951 INFO MainThread:2499832 [wandb_init.py:init():814] starting backend +2026-07-19 23:14:56,174 INFO MainThread:2499832 [wandb_init.py:init():829] sending inform_init request +2026-07-19 23:14:56,819 INFO MainThread:2499832 [wandb_init.py:init():834] backend started and connected +2026-07-19 23:14:56,821 INFO MainThread:2499832 [wandb_init.py:init():904] updated telemetry +2026-07-19 23:14:56,829 INFO MainThread:2499832 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-19 23:14:57,227 INFO MainThread:2499832 [wandb_init.py:init():972] starting run threads in backend +2026-07-19 23:14:57,421 INFO MainThread:2499832 [wandb_run.py:_console_start():2523] atexit reg +2026-07-19 23:14:57,421 INFO MainThread:2499832 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-19 23:14:57,422 INFO MainThread:2499832 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-19 23:14:57,422 INFO MainThread:2499832 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-19 23:14:57,427 INFO MainThread:2499832 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/RSCLstyle_cosine_60K/wandb/run-20260719_231456-ijigw64z/files/wandb-metadata.json b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/RSCLstyle_cosine_60K/wandb/run-20260719_231456-ijigw64z/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fe6e1926cf936b393e86d78064fff8ff95549ece --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/RSCLstyle_cosine_60K/wandb/run-20260719_231456-ijigw64z/files/wandb-metadata.json @@ -0,0 +1,91 @@ +{ + "os": "Linux-6.8.0-107-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.13", + "startedAt": "2026-07-19T14:14:56.096813Z", + "args": [ + "--config_path=/tmp/insight_lerobot_cfg__i6lm0cx/RSCLstyle_RKD_cosine_decay_60K.yaml", + "--batch_size=128", + "--output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/RSCLstyle_cosine_60K" + ], + "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", + "git": { + "remote": "git@github.com:paragon7060/clvla.git", + "commit": "a9df08f7da2d8eb1fa7f28708b4eb6f6dc81b1f9" + }, + "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231441_rerun_save20k40k60k/RSCLstyle_cosine_60K", + "host": "DGX-H200-01", + "executable": "/home/ext_minje/miniforge3/envs/lerobot060_groot/bin/python", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 8, + "disk": { + "/": { + "total": "1888425144320", + "used": "814088617984" + } + }, + "memory": { + "total": "2164177309696" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1e531a45-480e-3c1a-e9a0-2ba2cc27b0a3" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-5d3d5ae2-ffde-89d9-8818-dc19e11ad2fb" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-305c0dae-dcf5-d7bb-7dd3-5477d2194275" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-ed8b7920-8881-4e8f-3e88-be1bbecafa21" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-a91176db-55f2-b284-bbbf-56626596947d" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-922ff6d5-a948-205a-0426-dbee495fc3dd" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-1fde7c75-18fe-e3db-a4c1-85e67522f55f" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-a9b1c49b-d527-97d8-ce91-7e50bd02154c" + } + ], + "cudaVersion": "12.9", + "writerId": "8ord3b2hgfi9cnbrdyxoexlqdn89j67x" +} \ No newline at end of file diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/action_space_manifest.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/action_space_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..948b9f954edc763c7435e241e92f6269ea0e6bd8 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/action_space_manifest.json @@ -0,0 +1,205 @@ +{ + "schema_version": 2, + "action_space": "ee_abs_rot6d+ee_delta", + "action_views": [ + "ee_abs_rot6d", + "ee_delta" + ], + "source_feature": null, + "canonical_feature": "action", + "dtype": "float32", + "shape": [ + 16 + ], + "names": [ + "ee_abs_rot6d.x", + "ee_abs_rot6d.y", + "ee_abs_rot6d.z", + "ee_abs_rot6d.r_col0_x", + "ee_abs_rot6d.r_col0_y", + "ee_abs_rot6d.r_col0_z", + "ee_abs_rot6d.r_col1_x", + "ee_abs_rot6d.r_col1_y", + "ee_abs_rot6d.r_col1_z", + "ee_abs_rot6d.gripper", + "ee_delta.delta_x", + "ee_delta.delta_y", + "ee_delta.delta_z", + "ee_delta.delta_axis_angle_x", + "ee_delta.delta_axis_angle_y", + "ee_delta.delta_axis_angle_z" + ], + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "dataset_total_frames": 218367, + "dataset_total_episodes": 4930, + "dataset_fps": 10, + "stats_sha256": "75b95092e5fd9a1b16aa6dc7a478883927d7c7cce76c26b56b290b675ae2311a", + "stats": { + "min": [ + -0.21784618496894836, + -0.4474811851978302, + 0.1702737659215927, + -0.9999986290931702, + -0.9999991655349731, + -0.9999999403953552, + -0.9999990463256836, + -0.9999998211860657, + -0.9999988079071045, + 0.0, + -0.09827068448066711, + -0.13628354668617249, + -0.12191051989793777, + -2.431101083755493, + -0.3816019296646118, + -0.5094270706176758 + ], + "max": [ + 0.6283698678016663, + 0.4420371949672699, + 1.051027774810791, + 0.9999999403953552, + 0.9999995231628418, + 1.0, + 0.9999995231628418, + 1.0, + 0.9999996423721313, + 0.03999999910593033, + 0.11647262424230576, + 0.13701310753822327, + 0.13330328464508057, + 1.7823294401168823, + 0.5954466462135315, + 0.6865788102149963 + ], + "mean": [ + 0.4040228037735655, + -0.05247618696989652, + 0.6993459771218912, + 0.07011111991957117, + -0.03520558904460197, + 0.14470242846953205, + 0.05491166606739672, + -0.19073058893378805, + -0.06988117661871228, + 0.021167300266524632, + 0.0018730097199051647, + -0.0015708771114029225, + -0.001553903484685181, + 0.014420973441752116, + 0.00961362082315661, + -0.0017634798849202702 + ], + "std": [ + 0.11861927913357073, + 0.11318943557281885, + 0.17849160646987516, + 0.4812528791886511, + 0.7487196258736938, + 0.4251126804030219, + 0.47797229807769137, + 0.6267695346407992, + 0.5782954306815554, + 0.01996590431050817, + 0.015762732601462, + 0.010587306860662312, + 0.01180823404609393, + 0.10086023299718701, + 0.04747329757360547, + 0.14801097710214692 + ], + "count": [ + 218367 + ], + "q01": [ + 0.23988881827972064, + -0.10387461883034811, + 0.6297243923476871, + -0.43622841955353464, + -0.6885486527617446, + -0.14103555388039019, + -0.4327210063987595, + -0.9304790915716145, + -0.3465462258490417, + -1.000000013351432e-10, + -0.02190519006673929, + -0.02092386152118027, + -0.026422736164554015, + -0.11497312558112925, + -0.025678747253255536, + -0.17124583807766966 + ], + "q10": [ + 0.3042306436755216, + -0.09609991620622624, + 0.6394063072781145, + -0.3351306352998669, + -0.5751963015002924, + -0.08490621283307866, + -0.33849701676224275, + -0.8340557712529717, + -0.29760290790884514, + -1.000000013351432e-10, + -0.013015102506385306, + -0.011160732577261764, + -0.012954131824952583, + -0.056165101267083004, + -0.01548487039073523, + -0.15972996251089885 + ], + "q50": [ + 0.4160225263528291, + -0.06267459152989589, + 0.6811777491060801, + 0.0543536902230377, + -0.14012251058819217, + 0.04772394280004727, + 0.03838119138716219, + -0.14865169094195635, + -0.2073069401172425, + 0.020772594934870346, + 0.0005058764599893245, + 4.561956601090609e-06, + 4.945756498042436e-05, + 0.0001528964582560528, + 0.00017815767540355586, + -0.0002718872707462707 + ], + "q90": [ + 0.46641008755679864, + 0.020233143101890064, + 0.8095135405259374, + 0.48172401592474906, + 0.7398737043070319, + 0.5970841215636613, + 0.4674752875999555, + 0.36281754777672676, + 0.5033361883834496, + 0.039998362213352394, + 0.016358020650084733, + 0.003987531854121596, + 0.006191431456332113, + 0.11275584021509974, + 0.06100667096893492, + 0.15292656959583054 + ], + "q99": [ + 0.47320723171849244, + 0.04114262476135705, + 0.8467967259270389, + 0.5867781409980988, + 0.8529584859257228, + 0.7953936054341789, + 0.5768269858732034, + 0.49353349323389695, + 0.7168914440776504, + 0.03999983541667277, + 0.03626038003704021, + 0.008814053033445103, + 0.012299866693624668, + 0.17731576759283033, + 0.12117986440120965, + 0.16474415715691368 + ] + } +} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/prompt_manifest.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/prompt_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..21e3a43c712e5842c4616c53e544703bf52fb7ab --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/prompt_manifest.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "prompt_set": "detailed", + "source_file": "task_prompts.json", + "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc", + "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018", + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "task_count": 13, + "task_codes": [ + "5f", + "5g", + "1ext", + "3b", + "5b", + "3a", + "3c", + "5h", + "5d", + "3d", + "5e", + "5a", + "5c" + ], + "prompts": { + "5f": "Close the bottle in clockwise direction.", + "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.", + "1ext": "Find the arrow guide and open the indicated drawer.", + "3b": "Open the door, rotate counter-clockwise and push.", + "5b": "Open the bottle in counter-clockwise direction.", + "3a": "Open the door, rotate clockwise and push.", + "3c": "Open the door, rotate clockwise and pull.", + "5h": "Close the bottle in counter-clockwise direction.", + "5d": "Close the bottle in clockwise direction.", + "3d": "Open the door, rotate counter-clockwise and pull.", + "5e": "Close the bottle in counter-clockwise direction.", + "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.", + "5c": "Open the bottle in clockwise direction." + } +} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/source_config.yaml b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/source_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb1e7f8003e72c10c014b2b5c2e6e16623a9db63 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/source_config.yaml @@ -0,0 +1,63 @@ +# INSIGHT GR00T N1.7 training config +# INSIGHT wrapper-only fields are stripped before LeRobot config parsing. +insight: + # Options: joint_abs | ee_abs_quat | ee_abs_rot6d | ee_delta + action_space: ee_abs_rot6d+ee_delta + # Options: coarse | detailed + prompt_set: detailed + save_steps: ["60K"] + +dataset: + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2 + +policy: + type: groot + device: cuda + chunk_size: 16 + n_action_steps: 16 + push_to_hub: false + # Trainable scope. This baseline trains projector, diffusion, and VLLN; LLM/vision stay frozen. + # VLM + tune_llm: false + tune_visual: false + tune_top_llm_layers: 0 + # Action Expert + tune_projector: true + tune_diffusion_model: true + tune_vlln: true + + +seed: 42 +batch_size: 128 +steps: 60000 +log_freq: 200 +output_dir: "${HOME}/groot_insight/train_ckpt/Abs_6D + DeltaEE/Baseline" +job_name: INSIGHT_RD_baseline + +wandb: + enable: true + disable_artifact: true + +# Omitted because these match LeRobot/GR00T defaults or are unused in this setup: +# +# dataset: +# eval_split: 0.0 +# image_transforms: +# enable: false # default; set true only for train-image augmentation. +# +# policy: +# base_model_path: nvidia/GR00T-N1.7-3B +# embodiment_tag: new_embodiment +# use_relative_actions: false +# use_bf16: true +# repo_id: Whalswp/INSIGHT # unused while push_to_hub is false +# +# save_checkpoint: true +# use_policy_training_preset: true +# eval_steps: 0 +# env_eval_freq: 0 +# +# insight.prompt_set options: +# coarse: High-level task text that requires following the visual guide. +# detailed: Direction- and interaction-specific text, including rotation and push/pull details. diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/wandb/run-20260719_231648-h160hqm2/logs/debug.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/wandb/run-20260719_231648-h160hqm2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b58763d37c1d460be10444fc0dbcbcae67c42d13 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/wandb/run-20260719_231648-h160hqm2/logs/debug.log @@ -0,0 +1,19 @@ +2026-07-19 23:16:48,997 INFO MainThread:2591594 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2 +2026-07-19 23:16:48,997 INFO MainThread:2591594 [wandb_setup.py:_flush():81] Configure stats pid to 2591594 +2026-07-19 23:16:48,997 INFO MainThread:2591594 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-07-19 23:16:48,997 INFO MainThread:2591594 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/wandb/run-20260719_231648-h160hqm2/logs/debug.log +2026-07-19 23:16:48,997 INFO MainThread:2591594 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline/wandb/run-20260719_231648-h160hqm2/logs/debug-internal.log +2026-07-19 23:16:48,997 INFO MainThread:2591594 [wandb_init.py:init():766] calling init triggers +2026-07-19 23:16:48,997 INFO MainThread:2591594 [wandb_init.py:init():771] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot', 'n_obs_steps': 1, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': None, 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/Baseline', 'job_name': 'INSIGHT_RD_baseline', 'resume': False, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': None, '_wandb': {}} +2026-07-19 23:16:48,997 INFO MainThread:2591594 [wandb_init.py:init():814] starting backend +2026-07-19 23:16:49,214 INFO MainThread:2591594 [wandb_init.py:init():829] sending inform_init request +2026-07-19 23:16:49,747 INFO MainThread:2591594 [wandb_init.py:init():834] backend started and connected +2026-07-19 23:16:49,749 INFO MainThread:2591594 [wandb_init.py:init():904] updated telemetry +2026-07-19 23:16:49,755 INFO MainThread:2591594 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-19 23:16:50,147 INFO MainThread:2591594 [wandb_init.py:init():972] starting run threads in backend +2026-07-19 23:16:50,256 INFO MainThread:2591594 [wandb_run.py:_console_start():2523] atexit reg +2026-07-19 23:16:50,256 INFO MainThread:2591594 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-19 23:16:50,256 INFO MainThread:2591594 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-19 23:16:50,256 INFO MainThread:2591594 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-19 23:16:50,259 INFO MainThread:2591594 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/action_space_manifest.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/action_space_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..948b9f954edc763c7435e241e92f6269ea0e6bd8 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/action_space_manifest.json @@ -0,0 +1,205 @@ +{ + "schema_version": 2, + "action_space": "ee_abs_rot6d+ee_delta", + "action_views": [ + "ee_abs_rot6d", + "ee_delta" + ], + "source_feature": null, + "canonical_feature": "action", + "dtype": "float32", + "shape": [ + 16 + ], + "names": [ + "ee_abs_rot6d.x", + "ee_abs_rot6d.y", + "ee_abs_rot6d.z", + "ee_abs_rot6d.r_col0_x", + "ee_abs_rot6d.r_col0_y", + "ee_abs_rot6d.r_col0_z", + "ee_abs_rot6d.r_col1_x", + "ee_abs_rot6d.r_col1_y", + "ee_abs_rot6d.r_col1_z", + "ee_abs_rot6d.gripper", + "ee_delta.delta_x", + "ee_delta.delta_y", + "ee_delta.delta_z", + "ee_delta.delta_axis_angle_x", + "ee_delta.delta_axis_angle_y", + "ee_delta.delta_axis_angle_z" + ], + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "dataset_total_frames": 218367, + "dataset_total_episodes": 4930, + "dataset_fps": 10, + "stats_sha256": "75b95092e5fd9a1b16aa6dc7a478883927d7c7cce76c26b56b290b675ae2311a", + "stats": { + "min": [ + -0.21784618496894836, + -0.4474811851978302, + 0.1702737659215927, + -0.9999986290931702, + -0.9999991655349731, + -0.9999999403953552, + -0.9999990463256836, + -0.9999998211860657, + -0.9999988079071045, + 0.0, + -0.09827068448066711, + -0.13628354668617249, + -0.12191051989793777, + -2.431101083755493, + -0.3816019296646118, + -0.5094270706176758 + ], + "max": [ + 0.6283698678016663, + 0.4420371949672699, + 1.051027774810791, + 0.9999999403953552, + 0.9999995231628418, + 1.0, + 0.9999995231628418, + 1.0, + 0.9999996423721313, + 0.03999999910593033, + 0.11647262424230576, + 0.13701310753822327, + 0.13330328464508057, + 1.7823294401168823, + 0.5954466462135315, + 0.6865788102149963 + ], + "mean": [ + 0.4040228037735655, + -0.05247618696989652, + 0.6993459771218912, + 0.07011111991957117, + -0.03520558904460197, + 0.14470242846953205, + 0.05491166606739672, + -0.19073058893378805, + -0.06988117661871228, + 0.021167300266524632, + 0.0018730097199051647, + -0.0015708771114029225, + -0.001553903484685181, + 0.014420973441752116, + 0.00961362082315661, + -0.0017634798849202702 + ], + "std": [ + 0.11861927913357073, + 0.11318943557281885, + 0.17849160646987516, + 0.4812528791886511, + 0.7487196258736938, + 0.4251126804030219, + 0.47797229807769137, + 0.6267695346407992, + 0.5782954306815554, + 0.01996590431050817, + 0.015762732601462, + 0.010587306860662312, + 0.01180823404609393, + 0.10086023299718701, + 0.04747329757360547, + 0.14801097710214692 + ], + "count": [ + 218367 + ], + "q01": [ + 0.23988881827972064, + -0.10387461883034811, + 0.6297243923476871, + -0.43622841955353464, + -0.6885486527617446, + -0.14103555388039019, + -0.4327210063987595, + -0.9304790915716145, + -0.3465462258490417, + -1.000000013351432e-10, + -0.02190519006673929, + -0.02092386152118027, + -0.026422736164554015, + -0.11497312558112925, + -0.025678747253255536, + -0.17124583807766966 + ], + "q10": [ + 0.3042306436755216, + -0.09609991620622624, + 0.6394063072781145, + -0.3351306352998669, + -0.5751963015002924, + -0.08490621283307866, + -0.33849701676224275, + -0.8340557712529717, + -0.29760290790884514, + -1.000000013351432e-10, + -0.013015102506385306, + -0.011160732577261764, + -0.012954131824952583, + -0.056165101267083004, + -0.01548487039073523, + -0.15972996251089885 + ], + "q50": [ + 0.4160225263528291, + -0.06267459152989589, + 0.6811777491060801, + 0.0543536902230377, + -0.14012251058819217, + 0.04772394280004727, + 0.03838119138716219, + -0.14865169094195635, + -0.2073069401172425, + 0.020772594934870346, + 0.0005058764599893245, + 4.561956601090609e-06, + 4.945756498042436e-05, + 0.0001528964582560528, + 0.00017815767540355586, + -0.0002718872707462707 + ], + "q90": [ + 0.46641008755679864, + 0.020233143101890064, + 0.8095135405259374, + 0.48172401592474906, + 0.7398737043070319, + 0.5970841215636613, + 0.4674752875999555, + 0.36281754777672676, + 0.5033361883834496, + 0.039998362213352394, + 0.016358020650084733, + 0.003987531854121596, + 0.006191431456332113, + 0.11275584021509974, + 0.06100667096893492, + 0.15292656959583054 + ], + "q99": [ + 0.47320723171849244, + 0.04114262476135705, + 0.8467967259270389, + 0.5867781409980988, + 0.8529584859257228, + 0.7953936054341789, + 0.5768269858732034, + 0.49353349323389695, + 0.7168914440776504, + 0.03999983541667277, + 0.03626038003704021, + 0.008814053033445103, + 0.012299866693624668, + 0.17731576759283033, + 0.12117986440120965, + 0.16474415715691368 + ] + } +} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/phase_schedule.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/phase_schedule.json new file mode 100644 index 0000000000000000000000000000000000000000..8ba5c554b3132e132d81222d03914432be2666e3 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/phase_schedule.json @@ -0,0 +1,55 @@ +{ + "schema_version": 1, + "step_interval": "[start_step, until_step)", + "total_steps": 60000, + "optimizer_lifecycle": "single_continuous_optimizer", + "scheduler_lifecycle": "single_continuous_scheduler", + "phases": [ + { + "index": 0, + "name": "lap_rkd_linear_decay_10K", + "start_step": 0, + "until_step": 10000, + "policy": { + "rkd_enabled": true, + "rkd_fm_loss_weight": 1.0, + "rkd_loss_weight": 1.0, + "rkd_loss_weight_end": 0.0, + "rkd_loss_weight_schedule": "linear", + "rkd_distance_weight": 1.0, + "rkd_angle_weight": 2.0 + }, + "trainable": { + "tune_llm": false, + "tune_visual": false, + "tune_top_llm_layers": 0, + "tune_projector": true, + "tune_diffusion_model": true, + "tune_vlln": true + } + }, + { + "index": 1, + "name": "fm_only", + "start_step": 10000, + "until_step": 60000, + "policy": { + "rkd_enabled": false, + "rkd_fm_loss_weight": 1.0, + "rkd_loss_weight": 0.0, + "rkd_loss_weight_end": null, + "rkd_loss_weight_schedule": "linear", + "rkd_distance_weight": 1.0, + "rkd_angle_weight": 2.0 + }, + "trainable": { + "tune_llm": false, + "tune_visual": false, + "tune_top_llm_layers": 0, + "tune_projector": true, + "tune_diffusion_model": true, + "tune_vlln": true + } + } + ] +} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/prompt_manifest.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/prompt_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..21e3a43c712e5842c4616c53e544703bf52fb7ab --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/prompt_manifest.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "prompt_set": "detailed", + "source_file": "task_prompts.json", + "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc", + "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018", + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "task_count": 13, + "task_codes": [ + "5f", + "5g", + "1ext", + "3b", + "5b", + "3a", + "3c", + "5h", + "5d", + "3d", + "5e", + "5a", + "5c" + ], + "prompts": { + "5f": "Close the bottle in clockwise direction.", + "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.", + "1ext": "Find the arrow guide and open the indicated drawer.", + "3b": "Open the door, rotate counter-clockwise and push.", + "5b": "Open the bottle in counter-clockwise direction.", + "3a": "Open the door, rotate clockwise and push.", + "3c": "Open the door, rotate clockwise and pull.", + "5h": "Close the bottle in counter-clockwise direction.", + "5d": "Close the bottle in clockwise direction.", + "3d": "Open the door, rotate counter-clockwise and pull.", + "5e": "Close the bottle in counter-clockwise direction.", + "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.", + "5c": "Open the bottle in clockwise direction." + } +} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resolved_config.yaml b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resolved_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc32f32ae972fa3c1d7f8d41b6db2ffb8f2ac88a --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resolved_config.yaml @@ -0,0 +1,34 @@ +dataset: + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 +policy: + type: groot_rkd + device: cuda + chunk_size: 16 + n_action_steps: 16 + push_to_hub: false + warmup_steps: 6000 + tune_llm: false + tune_visual: false + tune_top_llm_layers: 0 + tune_projector: true + tune_diffusion_model: true + tune_vlln: true + rkd_enabled: true + rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84 + rkd_student_pooling: attention + rkd_student_attention_hidden_dim: 512 + rkd_student_attention_output_dim: 512 + rkd_fm_loss_weight: 1.0 + rkd_loss_weight: 1.0 + rkd_distance_weight: 1.0 + rkd_angle_weight: 2.0 +seed: 42 +batch_size: 128 +steps: 60000 +log_freq: 200 +output_dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_10K +job_name: INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K +wandb: + enable: true + disable_artifact: true diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_auth_gpu1.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_auth_gpu1.log new file mode 100644 index 0000000000000000000000000000000000000000..bec34948cf634ffcd3570bda6e7994dc8f68ac7b --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_auth_gpu1.log @@ -0,0 +1,214 @@ +INFO 2026-07-21 18:02:57 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'ny0tx2de'}} +INFO 2026-07-21 18:02:59 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:02:59 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/ny0tx2de +INFO 2026-07-21 18:02:59 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:03:00 ot_train.py:332 Creating policy +INFO 2026-07-21 18:03:00 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:03:00 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00 + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1424, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 397, in train + logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1197, in make_pre_post_processors + kwargs[override_key] = _project_nested_stats( + ^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 815, in _project_nested_stats + projected[key] = _project_nested_stats(item, source_key, composite_contract) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 815, in _project_nested_stats + projected[key] = _project_nested_stats(item, source_key, composite_contract) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 807, in _project_nested_stats + and all( + ^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 808, in + source in item +TypeError: argument of type 'PolicyFeature' is not iterable diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_fix2_gpu1.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_fix2_gpu1.log new file mode 100644 index 0000000000000000000000000000000000000000..9c40938d6a9023d0b9ba2c44b0afcd2d8fe26000 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_fix2_gpu1.log @@ -0,0 +1,199 @@ +INFO 2026-07-21 18:14:22 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'ny0tx2de'}} +INFO 2026-07-21 18:14:24 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:14:24 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/ny0tx2de +INFO 2026-07-21 18:14:24 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:14:26 ot_train.py:332 Creating policy +INFO 2026-07-21 18:14:26 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:14:26 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'ny0tx2de'}} +INFO 2026-07-21 18:08:00 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:08:00 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/ny0tx2de +INFO 2026-07-21 18:08:00 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:08:01 ot_train.py:332 Creating policy +INFO 2026-07-21 18:08:01 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:08:01 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00 + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1426, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 397, in train + logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1204, in make_pre_post_processors + return original_make_pre_post_processors(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/factory.py", line 323, in make_pre_post_processors + preprocessor = PolicyProcessorPipeline.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 738, in from_pretrained + cls._validate_overrides_used(validated_overrides, loaded_config) + File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 1254, in _validate_overrides_used + raise KeyError( +KeyError: "Override keys ['normalizer_processor'] do not match any step in the saved configuration. Available step keys: ['rename_observations_processor', 'to_batch_processor', 'rkd_preserve_raw_action', 'groot_n1_7_pack_inputs_v1', 'groot_n1_7_vlm_encode_v1', 'device_processor']. Make sure override keys match exact step class names or registry names." diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_gpu0_independent_nf69vc3o.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_gpu0_independent_nf69vc3o.log new file mode 100644 index 0000000000000000000000000000000000000000..ff888292238a1ac3bdc9456e06faee6a8a3b417c --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_gpu0_independent_nf69vc3o.log @@ -0,0 +1,200 @@ +INFO 2026-07-22 03:19:55 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'action_expert_num_layers': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'nf69vc3o'}} +INFO 2026-07-22 03:19:57 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 03:19:57 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/nf69vc3o +INFO 2026-07-22 03:19:57 ot_train.py:298 Creating dataset +INFO 2026-07-22 03:20:00 ot_train.py:332 Creating policy +INFO 2026-07-22 03:20:00 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model +INFO 2026-07-22 03:20:00 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'ny0tx2de'}} +INFO 2026-07-21 17:56:03 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 17:56:03 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/ny0tx2de +INFO 2026-07-21 17:56:03 ot_train.py:298 Creating dataset +INFO 2026-07-21 17:56:05 ot_train.py:332 Creating policy +INFO 2026-07-21 17:56:05 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model +INFO 2026-07-21 17:56:05 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 795, in hf_raise_for_status + response.raise_for_status() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/httpx/_models.py", line 829, in raise_for_status + raise HTTPStatusError(message, request=request, response=self) +httpx.HTTPStatusError: Client error '401 Unauthorized' for url 'https://huggingface.co/nvidia/GR00T-N1.7-3B/resolve/main/config.json' +For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401 + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 422, in cached_files + hf_hub_download( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1013, in hf_hub_download + return _hf_hub_download_to_cache_dir( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1165, in _hf_hub_download_to_cache_dir + _raise_on_head_call_error(head_call_error, force_download, local_files_only) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1864, in _raise_on_head_call_error + raise head_call_error + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1707, in _get_metadata_or_catch_error + metadata = get_hf_file_metadata( + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1615, in get_hf_file_metadata + response = _httpx_follow_relative_redirects_with_backoff( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 726, in _httpx_follow_relative_redirects_with_backoff + hf_raise_for_status(response) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 882, in hf_raise_for_status + raise _format(RepositoryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e +huggingface_hub.errors.RepositoryNotFoundError: 401 Client Error. (Request ID: Root=1-6a5fb2b5-6230713d52f06c4b0a188e0a;a13301f7-41f2-4c1a-aa6a-4018e0bdaaad) + +Repository Not Found for url: https://huggingface.co/nvidia/GR00T-N1.7-3B/resolve/main/config.json. +Please make sure you specified the correct `repo_id` and `repo_type`. +If you are trying to access a private or gated repo, make sure you are authenticated and your token has the required permissions. +For more details, see https://huggingface.co/docs/huggingface_hub/authentication +OAuth token signature verification failed: signature verification failed + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1428, in + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1424, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 333, in train + } + + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1170, in make_policy + policy = original_make_policy(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/factory.py", line 615, in make_policy + policy = policy_cls.from_pretrained(**kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 226, in from_pretrained + return super().from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/pretrained.py", line 194, in from_pretrained + instance = cls(config, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot_rkd.py", line 44, in __init__ + super().__init__(config, **kwargs) + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 79, in __init__ + self._groot_model = self._create_groot_model() + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 104, in _create_groot_model + model = GR00TN17.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/groot_n1_7.py", line 915, in from_pretrained + pretrained_model = super().from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/modeling_utils.py", line 3993, in from_pretrained + _adapter_model_path, pretrained_model_name_or_path, adapter_kwargs = maybe_load_adapters( + ^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/integrations/peft.py", line 973, in maybe_load_adapters + resolved_config_file = cached_file( + ^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 278, in cached_file + file = cached_files(path_or_repo_id=path_or_repo_id, filenames=[filename], **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 454, in cached_files + raise OSError( +OSError: nvidia/GR00T-N1.7-3B is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=` diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_independent_nf69vc3o_gpu1.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_independent_nf69vc3o_gpu1.log new file mode 100644 index 0000000000000000000000000000000000000000..7b458575072773ff4a0a403f59afcc1db925f261 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_independent_nf69vc3o_gpu1.log @@ -0,0 +1,400 @@ +INFO 2026-07-22 01:57:54 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'action_expert_num_layers': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'nf69vc3o'}} +INFO 2026-07-22 01:57:56 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 01:57:56 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/nf69vc3o +INFO 2026-07-22 01:57:56 ot_train.py:298 Creating dataset +INFO 2026-07-22 01:57:58 ot_train.py:332 Creating policy +INFO 2026-07-22 01:57:58 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model +INFO 2026-07-22 01:57:58 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'nf69vc3o'}} +INFO 2026-07-22 01:58:40 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 01:58:40 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/nf69vc3o +INFO 2026-07-22 01:58:40 ot_train.py:298 Creating dataset +INFO 2026-07-22 01:58:42 ot_train.py:332 Creating policy +INFO 2026-07-22 01:58:42 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model +INFO 2026-07-22 01:58:42 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'nf69vc3o'}} +INFO 2026-07-21 18:21:05 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:21:05 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/nf69vc3o +INFO 2026-07-21 18:21:05 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:21:07 ot_train.py:332 Creating policy +INFO 2026-07-21 18:21:07 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:21:07 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'nf69vc3o'}} +Traceback (most recent call last): + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1453, in + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1449, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 276, in train + if is_main_process: + ^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/common/wandb_utils.py", line 99, in __init__ + wandb.init( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1528, in init + get_sentry().reraise(e) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/analytics/sentry.py", line 191, in reraise + raise exc.with_traceback(tb) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1511, in init + run = wi.init(run_settings, run_config, run_printer) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 961, in init + raise error +wandb.errors.errors.UsageError: You provided an invalid value for the `resume` argument. The value 'must' is not a valid option for resuming the run (nf69vc3o) that has not been initialized. Please check your inputs and try again with a valid run ID. If you are trying to start a new run, please omit the `resume` argument or use `resume='allow'`. diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_restart_sep_nf69vc3o_gpu1.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_restart_sep_nf69vc3o_gpu1.log new file mode 100644 index 0000000000000000000000000000000000000000..44dead02ef3d2b07d64969afb08d3423c7f953f6 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/resume_dgx_20260722_restart_sep_nf69vc3o_gpu1.log @@ -0,0 +1,200 @@ +INFO 2026-07-22 01:56:01 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_10K', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'action_expert_num_layers': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'nf69vc3o'}} +INFO 2026-07-22 01:56:03 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 01:56:03 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/nf69vc3o +INFO 2026-07-22 01:56:03 ot_train.py:298 Creating dataset +INFO 2026-07-22 01:56:05 ot_train.py:332 Creating policy +INFO 2026-07-22 01:56:05 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_10K/checkpoints/030000/pretrained_model +INFO 2026-07-22 01:56:05 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'r3cz38bm'}} +INFO 2026-07-21 18:02:59 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:02:59 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/r3cz38bm +INFO 2026-07-21 18:02:59 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:03:01 ot_train.py:332 Creating policy +INFO 2026-07-21 18:03:01 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:03:01 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00 + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1424, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 397, in train + logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1197, in make_pre_post_processors + kwargs[override_key] = _project_nested_stats( + ^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 815, in _project_nested_stats + projected[key] = _project_nested_stats(item, source_key, composite_contract) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 815, in _project_nested_stats + projected[key] = _project_nested_stats(item, source_key, composite_contract) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 807, in _project_nested_stats + and all( + ^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 808, in + source in item +TypeError: argument of type 'PolicyFeature' is not iterable diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_fix2_gpu0.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_fix2_gpu0.log new file mode 100644 index 0000000000000000000000000000000000000000..37f78fb742c8875e2a08a76dec9ebd5bfd771b97 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_fix2_gpu0.log @@ -0,0 +1,199 @@ +INFO 2026-07-21 18:13:22 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'r3cz38bm'}} +INFO 2026-07-21 18:13:24 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:13:24 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/r3cz38bm +INFO 2026-07-21 18:13:24 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:13:26 ot_train.py:332 Creating policy +INFO 2026-07-21 18:13:26 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:13:26 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'r3cz38bm'}} +INFO 2026-07-21 18:07:59 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:07:59 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/r3cz38bm +INFO 2026-07-21 18:07:59 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:08:01 ot_train.py:332 Creating policy +INFO 2026-07-21 18:08:01 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:08:01 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00 + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1426, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 397, in train + logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1204, in make_pre_post_processors + return original_make_pre_post_processors(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/factory.py", line 323, in make_pre_post_processors + preprocessor = PolicyProcessorPipeline.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 738, in from_pretrained + cls._validate_overrides_used(validated_overrides, loaded_config) + File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 1254, in _validate_overrides_used + raise KeyError( +KeyError: "Override keys ['normalizer_processor'] do not match any step in the saved configuration. Available step keys: ['rename_observations_processor', 'to_batch_processor', 'rkd_preserve_raw_action', 'groot_n1_7_pack_inputs_v1', 'groot_n1_7_vlm_encode_v1', 'device_processor']. Make sure override keys match exact step class names or registry names." diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_gpu0.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_gpu0.log new file mode 100644 index 0000000000000000000000000000000000000000..808b2b0b07ed6000f890546832569581ba9bc656 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_gpu0.log @@ -0,0 +1,277 @@ +INFO 2026-07-21 17:56:01 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'r3cz38bm'}} +INFO 2026-07-21 17:56:03 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 17:56:03 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/r3cz38bm +INFO 2026-07-21 17:56:03 ot_train.py:298 Creating dataset +INFO 2026-07-21 17:56:05 ot_train.py:332 Creating policy +INFO 2026-07-21 17:56:05 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model +INFO 2026-07-21 17:56:05 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 795, in hf_raise_for_status + response.raise_for_status() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/httpx/_models.py", line 829, in raise_for_status + raise HTTPStatusError(message, request=request, response=self) +httpx.HTTPStatusError: Client error '401 Unauthorized' for url 'https://huggingface.co/nvidia/GR00T-N1.7-3B/resolve/main/config.json' +For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401 + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 422, in cached_files + hf_hub_download( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1013, in hf_hub_download + return _hf_hub_download_to_cache_dir( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1165, in _hf_hub_download_to_cache_dir + _raise_on_head_call_error(head_call_error, force_download, local_files_only) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1864, in _raise_on_head_call_error + raise head_call_error + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1707, in _get_metadata_or_catch_error + metadata = get_hf_file_metadata( + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1615, in get_hf_file_metadata + response = _httpx_follow_relative_redirects_with_backoff( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 726, in _httpx_follow_relative_redirects_with_backoff + hf_raise_for_status(response) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 882, in hf_raise_for_status + raise _format(RepositoryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e +huggingface_hub.errors.RepositoryNotFoundError: 401 Client Error. (Request ID: Root=1-6a5fb2b5-6824134c4e397387149d0cce;cac3eee4-7806-45b9-adfe-9e3e32a2e235) + +Repository Not Found for url: https://huggingface.co/nvidia/GR00T-N1.7-3B/resolve/main/config.json. +Please make sure you specified the correct `repo_id` and `repo_type`. +If you are trying to access a private or gated repo, make sure you are authenticated and your token has the required permissions. +For more details, see https://huggingface.co/docs/huggingface_hub/authentication +OAuth token signature verification failed: signature verification failed + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1428, in + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1424, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 333, in train + } + + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1170, in make_policy + policy = original_make_policy(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/factory.py", line 615, in make_policy + policy = policy_cls.from_pretrained(**kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 226, in from_pretrained + return super().from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/pretrained.py", line 194, in from_pretrained + instance = cls(config, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot_rkd.py", line 44, in __init__ + super().__init__(config, **kwargs) + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 79, in __init__ + self._groot_model = self._create_groot_model() + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 104, in _create_groot_model + model = GR00TN17.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/groot_n1_7.py", line 915, in from_pretrained + pretrained_model = super().from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/modeling_utils.py", line 3993, in from_pretrained + _adapter_model_path, pretrained_model_name_or_path, adapter_kwargs = maybe_load_adapters( + ^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/integrations/peft.py", line 973, in maybe_load_adapters + resolved_config_file = cached_file( + ^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 278, in cached_file + file = cached_files(path_or_repo_id=path_or_repo_id, filenames=[filename], **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 454, in cached_files + raise OSError( +OSError: nvidia/GR00T-N1.7-3B is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=` diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_newwandb2_a9vyia29_gpu0.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_newwandb2_a9vyia29_gpu0.log new file mode 100644 index 0000000000000000000000000000000000000000..e4aa63c8ebbb8a0b6cd8433a1f34378675d9b89e --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_newwandb2_a9vyia29_gpu0.log @@ -0,0 +1,279 @@ +INFO 2026-07-21 18:21:03 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'a9vyia29'}} +INFO 2026-07-21 18:21:05 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:21:05 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/a9vyia29 +INFO 2026-07-21 18:21:05 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:21:07 ot_train.py:332 Creating policy +INFO 2026-07-21 18:21:07 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:21:07 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00 + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1449, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 615, in train + wandb_logger.log_dict(wandb_log_dict, step) + ^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 299, in __call__ + transformed_transition = self._forward(transition) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 316, in _forward + transition = processor_step(transition) + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/groot/processor_groot.py", line 2158, in __call__ + sample_images = self._build_sample_images(video, batch_size, target_device) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/groot/processor_groot.py", line 2118, in _build_sample_images + video_t = video if torch.is_tensor(video) else torch.from_numpy(np.ascontiguousarray(video)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +KeyboardInterrupt diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_newwandb_a9vyia29_gpu0.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_newwandb_a9vyia29_gpu0.log new file mode 100644 index 0000000000000000000000000000000000000000..16c1a99f45bb883d8ddeccc9d88b222e5a0ed683 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/resume_dgx_20260722_newwandb_a9vyia29_gpu0.log @@ -0,0 +1,201 @@ +INFO 2026-07-21 18:19:52 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 6000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 6000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'a9vyia29'}} +Traceback (most recent call last): + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1453, in + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1449, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 276, in train + if is_main_process: + ^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/common/wandb_utils.py", line 99, in __init__ + wandb.init( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1528, in init + get_sentry().reraise(e) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/analytics/sentry.py", line 191, in reraise + raise exc.with_traceback(tb) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1511, in init + run = wi.init(run_settings, run_config, run_printer) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 961, in init + raise error +wandb.errors.errors.UsageError: You provided an invalid value for the `resume` argument. The value 'must' is not a valid option for resuming the run (a9vyia29) that has not been initialized. Please check your inputs and try again with a valid run ID. If you are trying to start a new run, please omit the `resume` argument or use `resume='allow'`. diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/source_config.yaml b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/source_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..543eb5102654db0f692dcb5b64b6816042d69080 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/LAPstyle_linear_6K/source_config.yaml @@ -0,0 +1,65 @@ +# INSIGHT GR00T N1.7 RKD training with LAP-style early RKD guidance. +# Common RKD values live in the top-level policy; phases contain overrides only. +insight: + # Composite policy action: ee_abs_rot6d 10D + ee_delta 6D (duplicate gripper removed) + action_space: ee_abs_rot6d+ee_delta + # Options: coarse | detailed + prompt_set: detailed + save_steps: ["60K"] + phases: + - name: lap_rkd_linear_decay_6K + until_step: 6K + policy: + # Inherits RKD=1.0 below and linearly decays it to zero. + rkd_loss_weight_end: 0.0 + + - name: fm_only + until_step: 60K + policy: + rkd_enabled: false + rkd_loss_weight: 0.0 + +dataset: + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + root: ${HOME}/INSIGHTfixposV4_filtered_multispace_v2 + +policy: + type: groot_rkd + device: cuda + chunk_size: 16 + n_action_steps: 16 + push_to_hub: false + # LAP Stage-2 warmup scaled to this 60K run: exactly 6K updates. + warmup_steps: 6000 + + # VLM + tune_llm: false + tune_visual: false + tune_top_llm_layers: 0 + # Action Expert + tune_projector: true + tune_diffusion_model: true + tune_vlln: true + + # Frozen RD TimewarpVAE teacher: ee_abs_rot6d 10D + ee_delta 6D (gripper deduplicated). + rkd_enabled: true + rkd_vae_checkpoint: ${HOME}/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84 + rkd_student_pooling: attention # attention | mean + rkd_student_attention_hidden_dim: 512 + rkd_student_attention_output_dim: 512 + rkd_fm_loss_weight: 1.0 + rkd_loss_weight: 1.0 + rkd_distance_weight: 1.0 + rkd_angle_weight: 2.0 + +seed: 42 +# Single GPU/process, so this is also the global batch size. +batch_size: 128 +steps: 60000 +log_freq: 200 +output_dir: "${HOME}/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/LAPstyle_linear_6K" +job_name: INSIGHT_RD_RKD_TimewarpVAE_LAPstyle_6K + +wandb: + enable: true + disable_artifact: true diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/action_space_manifest.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/action_space_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..948b9f954edc763c7435e241e92f6269ea0e6bd8 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/action_space_manifest.json @@ -0,0 +1,205 @@ +{ + "schema_version": 2, + "action_space": "ee_abs_rot6d+ee_delta", + "action_views": [ + "ee_abs_rot6d", + "ee_delta" + ], + "source_feature": null, + "canonical_feature": "action", + "dtype": "float32", + "shape": [ + 16 + ], + "names": [ + "ee_abs_rot6d.x", + "ee_abs_rot6d.y", + "ee_abs_rot6d.z", + "ee_abs_rot6d.r_col0_x", + "ee_abs_rot6d.r_col0_y", + "ee_abs_rot6d.r_col0_z", + "ee_abs_rot6d.r_col1_x", + "ee_abs_rot6d.r_col1_y", + "ee_abs_rot6d.r_col1_z", + "ee_abs_rot6d.gripper", + "ee_delta.delta_x", + "ee_delta.delta_y", + "ee_delta.delta_z", + "ee_delta.delta_axis_angle_x", + "ee_delta.delta_axis_angle_y", + "ee_delta.delta_axis_angle_z" + ], + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "dataset_total_frames": 218367, + "dataset_total_episodes": 4930, + "dataset_fps": 10, + "stats_sha256": "75b95092e5fd9a1b16aa6dc7a478883927d7c7cce76c26b56b290b675ae2311a", + "stats": { + "min": [ + -0.21784618496894836, + -0.4474811851978302, + 0.1702737659215927, + -0.9999986290931702, + -0.9999991655349731, + -0.9999999403953552, + -0.9999990463256836, + -0.9999998211860657, + -0.9999988079071045, + 0.0, + -0.09827068448066711, + -0.13628354668617249, + -0.12191051989793777, + -2.431101083755493, + -0.3816019296646118, + -0.5094270706176758 + ], + "max": [ + 0.6283698678016663, + 0.4420371949672699, + 1.051027774810791, + 0.9999999403953552, + 0.9999995231628418, + 1.0, + 0.9999995231628418, + 1.0, + 0.9999996423721313, + 0.03999999910593033, + 0.11647262424230576, + 0.13701310753822327, + 0.13330328464508057, + 1.7823294401168823, + 0.5954466462135315, + 0.6865788102149963 + ], + "mean": [ + 0.4040228037735655, + -0.05247618696989652, + 0.6993459771218912, + 0.07011111991957117, + -0.03520558904460197, + 0.14470242846953205, + 0.05491166606739672, + -0.19073058893378805, + -0.06988117661871228, + 0.021167300266524632, + 0.0018730097199051647, + -0.0015708771114029225, + -0.001553903484685181, + 0.014420973441752116, + 0.00961362082315661, + -0.0017634798849202702 + ], + "std": [ + 0.11861927913357073, + 0.11318943557281885, + 0.17849160646987516, + 0.4812528791886511, + 0.7487196258736938, + 0.4251126804030219, + 0.47797229807769137, + 0.6267695346407992, + 0.5782954306815554, + 0.01996590431050817, + 0.015762732601462, + 0.010587306860662312, + 0.01180823404609393, + 0.10086023299718701, + 0.04747329757360547, + 0.14801097710214692 + ], + "count": [ + 218367 + ], + "q01": [ + 0.23988881827972064, + -0.10387461883034811, + 0.6297243923476871, + -0.43622841955353464, + -0.6885486527617446, + -0.14103555388039019, + -0.4327210063987595, + -0.9304790915716145, + -0.3465462258490417, + -1.000000013351432e-10, + -0.02190519006673929, + -0.02092386152118027, + -0.026422736164554015, + -0.11497312558112925, + -0.025678747253255536, + -0.17124583807766966 + ], + "q10": [ + 0.3042306436755216, + -0.09609991620622624, + 0.6394063072781145, + -0.3351306352998669, + -0.5751963015002924, + -0.08490621283307866, + -0.33849701676224275, + -0.8340557712529717, + -0.29760290790884514, + -1.000000013351432e-10, + -0.013015102506385306, + -0.011160732577261764, + -0.012954131824952583, + -0.056165101267083004, + -0.01548487039073523, + -0.15972996251089885 + ], + "q50": [ + 0.4160225263528291, + -0.06267459152989589, + 0.6811777491060801, + 0.0543536902230377, + -0.14012251058819217, + 0.04772394280004727, + 0.03838119138716219, + -0.14865169094195635, + -0.2073069401172425, + 0.020772594934870346, + 0.0005058764599893245, + 4.561956601090609e-06, + 4.945756498042436e-05, + 0.0001528964582560528, + 0.00017815767540355586, + -0.0002718872707462707 + ], + "q90": [ + 0.46641008755679864, + 0.020233143101890064, + 0.8095135405259374, + 0.48172401592474906, + 0.7398737043070319, + 0.5970841215636613, + 0.4674752875999555, + 0.36281754777672676, + 0.5033361883834496, + 0.039998362213352394, + 0.016358020650084733, + 0.003987531854121596, + 0.006191431456332113, + 0.11275584021509974, + 0.06100667096893492, + 0.15292656959583054 + ], + "q99": [ + 0.47320723171849244, + 0.04114262476135705, + 0.8467967259270389, + 0.5867781409980988, + 0.8529584859257228, + 0.7953936054341789, + 0.5768269858732034, + 0.49353349323389695, + 0.7168914440776504, + 0.03999983541667277, + 0.03626038003704021, + 0.008814053033445103, + 0.012299866693624668, + 0.17731576759283033, + 0.12117986440120965, + 0.16474415715691368 + ] + } +} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/phase_schedule.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/phase_schedule.json new file mode 100644 index 0000000000000000000000000000000000000000..f7e4d4be997c76539eafba9602aecab4d33e137e --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/phase_schedule.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "step_interval": "[start_step, until_step)", + "total_steps": 60000, + "optimizer_lifecycle": "single_continuous_optimizer", + "scheduler_lifecycle": "single_continuous_scheduler", + "phases": [ + { + "index": 0, + "name": "rscl_rkd_cosine_decay_60K", + "start_step": 0, + "until_step": 60000, + "policy": { + "rkd_enabled": true, + "rkd_fm_loss_weight": 1.0, + "rkd_loss_weight": 1.0, + "rkd_loss_weight_end": 0.0, + "rkd_loss_weight_schedule": "cosine", + "rkd_distance_weight": 1.0, + "rkd_angle_weight": 2.0 + }, + "trainable": { + "tune_llm": false, + "tune_visual": false, + "tune_top_llm_layers": 0, + "tune_projector": true, + "tune_diffusion_model": true, + "tune_vlln": true + } + } + ] +} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/prompt_manifest.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/prompt_manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..21e3a43c712e5842c4616c53e544703bf52fb7ab --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/prompt_manifest.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "prompt_set": "detailed", + "source_file": "task_prompts.json", + "source_sha256": "0ba48fb372da9f49dcfcd7dcc2da8cce6ce991d3661060e664a104778d1ffacc", + "mapping_sha256": "f71954786e94cdeceb135dde09fdc6b65db0b87ab3b09d25060129d0f0fde018", + "dataset_repo_id": "Whalswp/INSIGHTfixposV4_filtered_multispace_v2", + "dataset_revision": "v3.0", + "task_count": 13, + "task_codes": [ + "5f", + "5g", + "1ext", + "3b", + "5b", + "3a", + "3c", + "5h", + "5d", + "3d", + "5e", + "5a", + "5c" + ], + "prompts": { + "5f": "Close the bottle in clockwise direction.", + "5g": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in clockwise direction.", + "1ext": "Find the arrow guide and open the indicated drawer.", + "3b": "Open the door, rotate counter-clockwise and push.", + "5b": "Open the bottle in counter-clockwise direction.", + "3a": "Open the door, rotate clockwise and push.", + "3c": "Open the door, rotate clockwise and pull.", + "5h": "Close the bottle in counter-clockwise direction.", + "5d": "Close the bottle in clockwise direction.", + "3d": "Open the door, rotate counter-clockwise and pull.", + "5e": "Close the bottle in counter-clockwise direction.", + "5a": "Grip the cap on the sides indicated by the 'squeeze' arrow and open the bottle in counter-clockwise direction.", + "5c": "Open the bottle in clockwise direction." + } +} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resolved_config.yaml b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resolved_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad0b9df058f3070d8e2df9dde77d9608e4eca917 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resolved_config.yaml @@ -0,0 +1,34 @@ +dataset: + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 +policy: + type: groot_rkd + device: cuda + chunk_size: 16 + n_action_steps: 16 + push_to_hub: false + warmup_steps: 3000 + tune_llm: false + tune_visual: false + tune_top_llm_layers: 0 + tune_projector: true + tune_diffusion_model: true + tune_vlln: true + rkd_enabled: true + rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84 + rkd_student_pooling: attention + rkd_student_attention_hidden_dim: 512 + rkd_student_attention_output_dim: 512 + rkd_fm_loss_weight: 1.0 + rkd_loss_weight: 1.0 + rkd_distance_weight: 1.0 + rkd_angle_weight: 2.0 +seed: 42 +batch_size: 128 +steps: 60000 +log_freq: 200 +output_dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/RKD_TimewarpVAE/RSCLstyle_cosine_60K +job_name: INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60 +wandb: + enable: true + disable_artifact: true diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_auth_gpu1.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_auth_gpu1.log new file mode 100644 index 0000000000000000000000000000000000000000..8823bfdb3596bea83449e98d86114b774f3b7994 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_auth_gpu1.log @@ -0,0 +1,214 @@ +INFO 2026-07-21 18:02:57 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'bepgb8uu'}} +INFO 2026-07-21 18:02:59 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:02:59 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/bepgb8uu +INFO 2026-07-21 18:02:59 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:03:01 ot_train.py:332 Creating policy +INFO 2026-07-21 18:03:01 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:03:01 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00 + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1424, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 397, in train + logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1197, in make_pre_post_processors + kwargs[override_key] = _project_nested_stats( + ^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 815, in _project_nested_stats + projected[key] = _project_nested_stats(item, source_key, composite_contract) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 815, in _project_nested_stats + projected[key] = _project_nested_stats(item, source_key, composite_contract) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 807, in _project_nested_stats + and all( + ^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 808, in + source in item +TypeError: argument of type 'PolicyFeature' is not iterable diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_fix2_gpu1.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_fix2_gpu1.log new file mode 100644 index 0000000000000000000000000000000000000000..388993ec5646ae64ce97bfded0e0cd9ee623dcca --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_fix2_gpu1.log @@ -0,0 +1,198 @@ +INFO 2026-07-21 18:14:22 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'bepgb8uu'}} +INFO 2026-07-21 18:14:25 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:14:25 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/bepgb8uu +INFO 2026-07-21 18:14:25 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:14:26 ot_train.py:332 Creating policy +INFO 2026-07-21 18:14:26 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:14:26 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'bepgb8uu'}} +INFO 2026-07-21 18:07:59 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:07:59 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/bepgb8uu +INFO 2026-07-21 18:07:59 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:08:01 ot_train.py:332 Creating policy +INFO 2026-07-21 18:08:01 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:08:01 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00 + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1426, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 397, in train + logging.info(colored("Output dir:", "yellow", attrs=["bold"]) + f" {cfg.output_dir}") + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1204, in make_pre_post_processors + return original_make_pre_post_processors(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/factory.py", line 323, in make_pre_post_processors + preprocessor = PolicyProcessorPipeline.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 738, in from_pretrained + cls._validate_overrides_used(validated_overrides, loaded_config) + File "/home/ext_minje/clvla/lerobot/src/lerobot/processor/pipeline.py", line 1254, in _validate_overrides_used + raise KeyError( +KeyError: "Override keys ['normalizer_processor'] do not match any step in the saved configuration. Available step keys: ['rename_observations_processor', 'to_batch_processor', 'rkd_preserve_raw_action', 'groot_n1_7_pack_inputs_v1', 'groot_n1_7_vlm_encode_v1', 'device_processor']. Make sure override keys match exact step class names or registry names." diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_gpu1.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_gpu1.log new file mode 100644 index 0000000000000000000000000000000000000000..a53a187064ed24e0157ceed707e83befba3321b2 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_gpu1.log @@ -0,0 +1,277 @@ +INFO 2026-07-21 17:56:01 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': 'bepgb8uu'}} +INFO 2026-07-21 17:56:03 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 17:56:03 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/bepgb8uu +INFO 2026-07-21 17:56:03 ot_train.py:298 Creating dataset +INFO 2026-07-21 17:56:05 ot_train.py:332 Creating policy +INFO 2026-07-21 17:56:05 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-21 17:56:05 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 795, in hf_raise_for_status + response.raise_for_status() + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/httpx/_models.py", line 829, in raise_for_status + raise HTTPStatusError(message, request=request, response=self) +httpx.HTTPStatusError: Client error '401 Unauthorized' for url 'https://huggingface.co/nvidia/GR00T-N1.7-3B/resolve/main/config.json' +For more information check: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401 + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 422, in cached_files + hf_hub_download( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1013, in hf_hub_download + return _hf_hub_download_to_cache_dir( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1165, in _hf_hub_download_to_cache_dir + _raise_on_head_call_error(head_call_error, force_download, local_files_only) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1864, in _raise_on_head_call_error + raise head_call_error + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1707, in _get_metadata_or_catch_error + metadata = get_hf_file_metadata( + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_validators.py", line 88, in _inner_fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/file_download.py", line 1615, in get_hf_file_metadata + response = _httpx_follow_relative_redirects_with_backoff( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 726, in _httpx_follow_relative_redirects_with_backoff + hf_raise_for_status(response) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/huggingface_hub/utils/_http.py", line 882, in hf_raise_for_status + raise _format(RepositoryNotFoundError, message, response, repo_type=repo_type, repo_id=repo_id) from e +huggingface_hub.errors.RepositoryNotFoundError: 401 Client Error. (Request ID: Root=1-6a5fb2b5-6e644f103ef022560085cd44;f53fa2ef-017a-4e77-8ad8-455daaf34c1f) + +Repository Not Found for url: https://huggingface.co/nvidia/GR00T-N1.7-3B/resolve/main/config.json. +Please make sure you specified the correct `repo_id` and `repo_type`. +If you are trying to access a private or gated repo, make sure you are authenticated and your token has the required permissions. +For more details, see https://huggingface.co/docs/huggingface_hub/authentication +OAuth token signature verification failed: signature verification failed + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1428, in + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1424, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 333, in train + } + + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1170, in make_policy + policy = original_make_policy(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/factory.py", line 615, in make_policy + policy = policy_cls.from_pretrained(**kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 226, in from_pretrained + return super().from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/policies/pretrained.py", line 194, in from_pretrained + instance = cls(config, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot_rkd.py", line 44, in __init__ + super().__init__(config, **kwargs) + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 79, in __init__ + self._groot_model = self._create_groot_model() + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/modeling_groot.py", line 104, in _create_groot_model + model = GR00TN17.from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_policies/groot_RKD/groot_n1_7.py", line 915, in from_pretrained + pretrained_model = super().from_pretrained( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/modeling_utils.py", line 3993, in from_pretrained + _adapter_model_path, pretrained_model_name_or_path, adapter_kwargs = maybe_load_adapters( + ^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/integrations/peft.py", line 973, in maybe_load_adapters + resolved_config_file = cached_file( + ^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 278, in cached_file + file = cached_files(path_or_repo_id=path_or_repo_id, filenames=[filename], **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/transformers/utils/hub.py", line 454, in cached_files + raise OSError( +OSError: nvidia/GR00T-N1.7-3B is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models' +If this is a private repository, make sure to pass a token having permission to this repo either by logging in with `hf auth login` or by passing `token=` diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_gpu1_independent_4bcz2tkp.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_gpu1_independent_4bcz2tkp.log new file mode 100644 index 0000000000000000000000000000000000000000..487c715be62845a112aaa7fd3c1d0ee8dbf9da70 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_gpu1_independent_4bcz2tkp.log @@ -0,0 +1,199 @@ +INFO 2026-07-22 03:20:05 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'action_expert_num_layers': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': '4bcz2tkp'}} +INFO 2026-07-22 03:20:07 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 03:20:07 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/4bcz2tkp +INFO 2026-07-22 03:20:07 ot_train.py:298 Creating dataset +INFO 2026-07-22 03:20:09 ot_train.py:332 Creating policy +INFO 2026-07-22 03:20:09 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-22 03:20:09 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': '4bcz2tkp'}} +INFO 2026-07-22 01:58:04 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 01:58:04 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/4bcz2tkp +INFO 2026-07-22 01:58:04 ot_train.py:298 Creating dataset +INFO 2026-07-22 01:58:06 ot_train.py:332 Creating policy +INFO 2026-07-22 01:58:06 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-22 01:58:06 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': '4bcz2tkp'}} +INFO 2026-07-22 01:58:49 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 01:58:49 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/4bcz2tkp +INFO 2026-07-22 01:58:49 ot_train.py:298 Creating dataset +INFO 2026-07-22 01:58:50 ot_train.py:332 Creating policy +INFO 2026-07-22 01:58:50 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-22 01:58:50 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': '4bcz2tkp'}} +INFO 2026-07-21 18:21:06 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-21 18:21:06 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/4bcz2tkp +INFO 2026-07-21 18:21:06 ot_train.py:298 Creating dataset +INFO 2026-07-21 18:21:08 ot_train.py:332 Creating policy +INFO 2026-07-21 18:21:08 ng_groot.py:191 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-21 18:21:08 ng_groot.py:225 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00}, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': '4bcz2tkp'}} +Traceback (most recent call last): + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1453, in + main() + File "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", line 1449, in main + train_module.main() + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 816, in main + File "/home/ext_minje/clvla/lerobot/src/lerobot/configs/parser.py", line 320, in wrapper_inner + response = fn(cfg, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/scripts/lerobot_train.py", line 276, in train + if is_main_process: + ^^^^^ + File "/home/ext_minje/clvla/lerobot/src/lerobot/common/wandb_utils.py", line 99, in __init__ + wandb.init( + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1528, in init + get_sentry().reraise(e) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/analytics/sentry.py", line 191, in reraise + raise exc.with_traceback(tb) + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 1511, in init + run = wi.init(run_settings, run_config, run_printer) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/ext_minje/miniconda3/envs/lerobot060_groot/lib/python3.12/site-packages/wandb/sdk/wandb_init.py", line 961, in init + raise error +wandb.errors.errors.UsageError: You provided an invalid value for the `resume` argument. The value 'must' is not a valid option for resuming the run (4bcz2tkp) that has not been initialized. Please check your inputs and try again with a valid run ID. If you are trying to start a new run, please omit the `resume` argument or use `resume='allow'`. diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_restart_sep_4bcz2tkp_gpu1.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_restart_sep_4bcz2tkp_gpu1.log new file mode 100644 index 0000000000000000000000000000000000000000..8476c0ec1ca94f6f311456bf9930f25d17559235 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/resume_dgx_20260722_restart_sep_4bcz2tkp_gpu1.log @@ -0,0 +1,199 @@ +INFO 2026-07-22 01:56:01 ot_train.py:272 {'batch_size': 128, + 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000', + 'cudnn_deterministic': False, + 'dataset': {'depth_output_unit': 'mm', + 'episodes': None, + 'eval_split': 0.0, + 'image_transforms': {'enable': False, + 'max_num_transforms': 3, + 'random_order': False, + 'tfs': {'affine': {'kwargs': {'degrees': [-5.0, + 5.0], + 'translate': [0.05, + 0.05]}, + 'type': 'RandomAffine', + 'weight': 1.0}, + 'brightness': {'kwargs': {'brightness': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'contrast': {'kwargs': {'contrast': [0.8, + 1.2]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'hue': {'kwargs': {'hue': [-0.05, + 0.05]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'saturation': {'kwargs': {'saturation': [0.5, + 1.5]}, + 'type': 'ColorJitter', + 'weight': 1.0}, + 'sharpness': {'kwargs': {'sharpness': [0.5, + 1.5]}, + 'type': 'SharpnessJitter', + 'weight': 1.0}}}, + 'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', + 'return_uint8': False, + 'revision': None, + 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', + 'streaming': False, + 'use_imagenet_stats': True, + 'video_backend': 'torchcodec'}, + 'env': None, + 'env_eval_freq': 20000, + 'eval': {'batch_size': 50, + 'n_episodes': 50, + 'recording': False, + 'recording_private': False, + 'recording_repo_id': None, + 'use_async_envs': True}, + 'eval_steps': 0, + 'job': {'detach': False, + 'image': 'huggingface/lerobot-gpu:latest', + 'tags': [], + 'target': None, + 'timeout': '2d'}, + 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', + 'log_freq': 200, + 'max_eval_samples': 0, + 'num_workers': 4, + 'optimizer': {'betas': [0.9, 0.999], + 'eps': 1e-08, + 'grad_clip_norm': 1.0, + 'lr': 0.0001, + 'type': 'adamw', + 'weight_decay': 1e-05}, + 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K', + 'peft': None, + 'persistent_workers': True, + 'policy': {'action_decode_transform': None, + 'action_expert_num_layers': None, + 'balance_dataset_weights': True, + 'balance_trajectory_weights': True, + 'base_model_path': 'nvidia/GR00T-N1.7-3B', + 'batch_size': 32, + 'chunk_size': 16, + 'dataloader_num_workers': 8, + 'dataset_paths': None, + 'device': 'cuda', + 'embodiment_tag': 'new_embodiment', + 'image_size': [256, 256], + 'input_features': {'observation.images.guide': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.right_shoulder': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.images.wrist': {'shape': [3, + 224, + 224], + 'type': }, + 'observation.state': {'shape': [16], + 'type': }}, + 'license': None, + 'lora_alpha': 16, + 'lora_dropout': 0.1, + 'lora_full_model': False, + 'lora_rank': 0, + 'max_action_dim': 132, + 'max_state_dim': 132, + 'max_steps': 10000, + 'model_params_fp32': True, + 'n_action_steps': 16, + 'n_obs_steps': 1, + 'normalization_mapping': {'ACTION': , + 'STATE': , + 'VISUAL': }, + 'num_inference_timesteps': None, + 'optimizer_betas': [0.9, 0.999], + 'optimizer_eps': 1e-08, + 'optimizer_lr': 0.0001, + 'optimizer_weight_decay': 1e-05, + 'output_dir': './tmp/gr00t', + 'output_features': {'action': {'shape': [16], + 'type': }}, + 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D ' + '+ ' + 'DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', + 'pretrained_revision': None, + 'private': None, + 'push_to_hub': False, + 'relative_exclude_joints': [], + 'repo_id': None, + 'report_to': 'wandb', + 'resume': False, + 'rkd_angle_weight': 2.0, + 'rkd_distance_weight': 1.0, + 'rkd_enabled': True, + 'rkd_fm_loss_weight': 1.0, + 'rkd_loss_weight': 1.0, + 'rkd_student_attention_hidden_dim': 512, + 'rkd_student_attention_output_dim': 512, + 'rkd_student_pooling': 'attention', + 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', + 'rtc_ramp_rate': None, + 'save_steps': 1000, + 'tags': None, + 'tokenizer_assets_repo': None, + 'tune_diffusion_model': True, + 'tune_llm': False, + 'tune_projector': True, + 'tune_top_llm_layers': 0, + 'tune_visual': False, + 'tune_vlln': True, + 'type': 'groot_rkd', + 'use_amp': False, + 'use_bf16': True, + 'use_flash_attention': False, + 'use_peft': False, + 'use_relative_actions': False, + 'video_backend': 'decord', + 'warmup_ratio': 0.05, + 'warmup_steps': 3000}, + 'prefetch_factor': 4, + 'rename_map': {}, + 'resume': True, + 'reward_model': None, + 'sample_weighting': None, + 'save_checkpoint': True, + 'save_checkpoint_to_hub': False, + 'save_freq': 20000, + 'scheduler': {'name': 'cosine', 'num_warmup_steps': 3000, 'type': 'diffuser'}, + 'seed': 42, + 'steps': 60000, + 'tolerance_s': 0.0001, + 'use_policy_training_preset': True, + 'wandb': {'add_tags': True, + 'disable_artifact': True, + 'enable': True, + 'entity': None, + 'mode': None, + 'notes': None, + 'project': 'lerobot', + 'run_id': '4bcz2tkp'}} +INFO 2026-07-22 01:56:03 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 01:56:03 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/4bcz2tkp +INFO 2026-07-22 01:56:03 ot_train.py:298 Creating dataset +INFO 2026-07-22 01:56:05 ot_train.py:332 Creating policy +INFO 2026-07-22 01:56:05 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-22 01:56:05 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... + Fetching 27 files: 0%| | 0/27 [00:00, 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [16]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': None, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '4bcz2tkp', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000', '_wandb': {}} +2026-07-22 03:20:05,794 INFO MainThread:2622885 [wandb_init.py:init():814] starting backend +2026-07-22 03:20:06,010 INFO MainThread:2622885 [wandb_init.py:init():829] sending inform_init request +2026-07-22 03:20:06,497 INFO MainThread:2622885 [wandb_init.py:init():834] backend started and connected +2026-07-22 03:20:06,499 INFO MainThread:2622885 [wandb_init.py:init():904] updated telemetry +2026-07-22 03:20:06,503 INFO MainThread:2622885 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-22 03:20:07,517 INFO MainThread:2622885 [wandb_init.py:init():967] run resumed +2026-07-22 03:20:07,519 INFO MainThread:2622885 [wandb_init.py:init():972] starting run threads in backend +2026-07-22 03:20:07,577 INFO MainThread:2622885 [wandb_run.py:_console_start():2523] atexit reg +2026-07-22 03:20:07,577 INFO MainThread:2622885 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-22 03:20:07,578 INFO MainThread:2622885 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-22 03:20:07,578 INFO MainThread:2622885 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-22 03:20:07,579 INFO MainThread:2622885 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_175602-bepgb8uu/files/config.yaml b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_175602-bepgb8uu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cab0bf33b62c4340dcfc24c9275f8a3e00630428 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_175602-bepgb8uu/files/config.yaml @@ -0,0 +1,333 @@ +_wandb: + value: + cli_version: 0.27.2 + e: + e7cqx8h2t71p0rei3xmd9d75oaf2n9n8: + args: + - --config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model/train_config.json + - --resume=true + - --dataset.root=/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 + - --output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K + codePath: benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + codePathLocal: my_scripts/lerobot_train_yaml.py + cpu_count: 112 + cpu_count_logical: 224 + cudaVersion: "12.8" + disk: + /: + total: "1888425144320" + used: "1054655565824" + executable: /home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python + git: + commit: c1ff610ae4b02827f1c984cef8bbf28a7889bd6f + remote: git@github.com:paragon7060/clvla.git + gpu: NVIDIA H200 + gpu_count: 2 + gpu_nvidia: + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991 + - architecture: Hopper + cudaCores: 16896 + memoryTotal: "150754820096" + name: NVIDIA H200 + uuid: GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc + host: ext_minje-box + memory: + total: "2164176568320" + os: Linux-6.8.0-55-generic-x86_64-with-glibc2.39 + program: /home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py + python: CPython 3.12.13 + root: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K + startedAt: "2026-07-21T17:56:02.220495Z" + writerId: e7cqx8h2t71p0rei3xmd9d75oaf2n9n8 + m: [] + python_version: 3.12.13 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + - 83 + - 98 + "3": + - 5 + - 13 + - 14 + - 15 + - 16 + - 41 + - 62 + "4": 3.12.13 + "5": 0.27.2 + "6": 5.5.4 + "10": + - 21 + "12": 0.27.2 + "13": linux-x86_64 +batch_size: + value: 128 +checkpoint_path: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000 +cudnn_deterministic: + value: false +dataset: + value: + depth_output_unit: mm + episodes: null + eval_split: 0 + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + affine: + kwargs: + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + type: RandomAffine + weight: 1 + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: Whalswp/INSIGHTfixposV4_filtered_multispace_v2 + return_uint8: false + revision: null + root: /home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2 + streaming: false + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +env_eval_freq: + value: 20000 +eval: + value: + batch_size: 50 + n_episodes: 50 + recording: false + recording_private: false + recording_repo_id: null + use_async_envs: true +eval_steps: + value: 0 +job: + value: + detach: false + image: huggingface/lerobot-gpu:latest + tags: [] + target: null + timeout: 2d +job_name: + value: INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60 +log_freq: + value: 200 +max_eval_samples: + value: 0 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 1 + lr: 0.0001 + type: adamw + weight_decay: 1e-05 +output_dir: + value: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K +peft: + value: null +persistent_workers: + value: true +policy: + value: + action_decode_transform: null + balance_dataset_weights: true + balance_trajectory_weights: true + base_model_path: nvidia/GR00T-N1.7-3B + batch_size: 32 + chunk_size: 16 + dataloader_num_workers: 8 + dataset_paths: null + device: cuda + embodiment_tag: new_embodiment + image_size: + - 256 + - 256 + input_features: + observation.images.guide: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.right_shoulder: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.images.wrist: + shape: + - 3 + - 224 + - 224 + type: VISUAL + observation.state: + shape: + - 16 + type: STATE + license: null + lora_alpha: 16 + lora_dropout: 0.1 + lora_full_model: false + lora_rank: 0 + max_action_dim: 132 + max_state_dim: 132 + max_steps: 10000 + model_params_fp32: true + n_action_steps: 16 + n_obs_steps: 1 + normalization_mapping: + ACTION: IDENTITY + STATE: IDENTITY + VISUAL: IDENTITY + num_inference_timesteps: null + optimizer_betas: + - 0.9 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-05 + output_dir: ./tmp/gr00t + output_features: + action: + shape: + - 16 + type: ACTION + pretrained_path: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model + pretrained_revision: null + private: null + push_to_hub: false + relative_exclude_joints: [] + repo_id: null + report_to: wandb + resume: false + rkd_angle_weight: 2 + rkd_distance_weight: 1 + rkd_enabled: true + rkd_fm_loss_weight: 1 + rkd_loss_weight: 1 + rkd_student_attention_hidden_dim: 512 + rkd_student_attention_output_dim: 512 + rkd_student_pooling: attention + rkd_vae_checkpoint: /home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84 + rtc_ramp_rate: null + save_steps: 1000 + tags: null + tokenizer_assets_repo: null + tune_diffusion_model: true + tune_llm: false + tune_projector: true + tune_top_llm_layers: 0 + tune_visual: false + tune_vlln: true + type: groot_rkd + use_amp: false + use_bf16: true + use_flash_attention: false + use_peft: false + use_relative_actions: false + video_backend: decord + warmup_ratio: 0.05 + warmup_steps: 3000 +prefetch_factor: + value: 4 +resume: + value: true +reward_model: + value: null +sample_weighting: + value: null +save_checkpoint: + value: true +save_checkpoint_to_hub: + value: false +save_freq: + value: 20000 +scheduler: + value: + name: cosine + num_warmup_steps: 3000 + type: diffuser +seed: + value: 42 +steps: + value: 60000 +tolerance_s: + value: 0.0001 +use_policy_training_preset: + value: true +wandb: + value: + add_tags: true + disable_artifact: true + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: bepgb8uu diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_175602-bepgb8uu/run-bepgb8uu.wandb b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_175602-bepgb8uu/run-bepgb8uu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f8abf9a9124e3fc99e23ffd8d5dfc54054fdefd2 Binary files /dev/null and b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_175602-bepgb8uu/run-bepgb8uu.wandb differ diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_180257-bepgb8uu/run-bepgb8uu.wandb b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_180257-bepgb8uu/run-bepgb8uu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f9a44aaa7548b08ffce0691c2d727c5ab183f3cb Binary files /dev/null and b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_180257-bepgb8uu/run-bepgb8uu.wandb differ diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_180758-bepgb8uu/run-bepgb8uu.wandb b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_180758-bepgb8uu/run-bepgb8uu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d8968e1ef7eb90bc2f81445cee190e643f7f7f7d Binary files /dev/null and b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_180758-bepgb8uu/run-bepgb8uu.wandb differ diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181423-bepgb8uu/run-bepgb8uu.wandb b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181423-bepgb8uu/run-bepgb8uu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a6fa9ed97f4670c43cd260ba03c431f4d2ced4b3 Binary files /dev/null and b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181423-bepgb8uu/run-bepgb8uu.wandb differ diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/files/wandb-summary.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1d476fc88692f959c7a899096787abbc21a55dbc --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":0,"_wandb":{"runtime":0}} \ No newline at end of file diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/logs/debug-core.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b95b7f7dad100db11f72fa0cbe69674cc01b2bba --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/logs/debug-core.log @@ -0,0 +1,15 @@ +{"time":"2026-07-21T18:19:49.021160945Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpd4hkix9o/port-169828.txt","pid":169828,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-07-21T18:19:49.022059982Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":169828} +{"time":"2026-07-21T18:19:49.022047922Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-169828-170916-58554531/socket","Net":"unix"}} +{"time":"2026-07-21T18:19:49.206942091Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-07-21T18:19:49.214366175Z","level":"INFO","msg":"handleInformInit: received","streamId":"4bcz2tkp","id":"1(@)"} +{"time":"2026-07-21T18:19:49.67434026Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"4bcz2tkp","id":"1(@)"} +{"time":"2026-07-21T18:19:49.978026902Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2026-07-21T18:19:49.978182197Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-21T18:19:49.978199384Z","level":"INFO","msg":"server is shutting down"} +{"time":"2026-07-21T18:19:49.978189604Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-21T18:19:49.978236356Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-21T18:19:49.978299586Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-169828-170916-58554531/socket","Net":"unix"}} +{"time":"2026-07-21T18:19:49.979319337Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2026-07-21T18:19:49.979335173Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-21T18:19:49.979340406Z","level":"INFO","msg":"server is closed"} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/logs/debug.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..25a3c75b24eefc4e155395586c96c66e3317ac83 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/logs/debug.log @@ -0,0 +1,13 @@ +2026-07-21 18:19:48,989 INFO MainThread:169828 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2 +2026-07-21 18:19:48,989 INFO MainThread:169828 [wandb_setup.py:_flush():81] Configure stats pid to 169828 +2026-07-21 18:19:48,989 INFO MainThread:169828 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-07-21 18:19:48,989 INFO MainThread:169828 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/logs/debug.log +2026-07-21 18:19:48,990 INFO MainThread:169828 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/logs/debug-internal.log +2026-07-21 18:19:48,990 INFO MainThread:169828 [wandb_init.py:init():766] calling init triggers +2026-07-21 18:19:48,990 INFO MainThread:169828 [wandb_init.py:init():771] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': , 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [16]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '4bcz2tkp', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000', '_wandb': {}} +2026-07-21 18:19:48,990 INFO MainThread:169828 [wandb_init.py:init():814] starting backend +2026-07-21 18:19:49,207 INFO MainThread:169828 [wandb_init.py:init():829] sending inform_init request +2026-07-21 18:19:49,675 INFO MainThread:169828 [wandb_init.py:init():834] backend started and connected +2026-07-21 18:19:49,677 INFO MainThread:169828 [wandb_init.py:init():904] updated telemetry +2026-07-21 18:19:49,681 INFO MainThread:169828 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/run-4bcz2tkp.wandb b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/run-4bcz2tkp.wandb new file mode 100644 index 0000000000000000000000000000000000000000..bfd219bc017a88e34169c9127adce95bab8bf259 Binary files /dev/null and b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260721_181948-4bcz2tkp/run-4bcz2tkp.wandb differ diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015601-4bcz2tkp/files/output.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015601-4bcz2tkp/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..dcd480b68dcee7f19adf4446ffcf1eb0259723bd --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015601-4bcz2tkp/files/output.log @@ -0,0 +1,21 @@ +INFO 2026-07-22 01:56:03 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 01:56:03 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/4bcz2tkp +INFO 2026-07-22 01:56:03 ot_train.py:298 Creating dataset +INFO 2026-07-22 01:56:05 ot_train.py:332 Creating policy +INFO 2026-07-22 01:56:05 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-22 01:56:05 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 4363.85it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 1122.12it/s] +Loading weights from local directory +INFO 2026-07-22 01:56:18 ot_train.py:405 Creating optimizer and scheduler +INFO 2026-07-22 01:56:21 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K +INFO 2026-07-22 01:56:21 ot_train.py:446 cfg.steps=60000 (60K) +INFO 2026-07-22 01:56:21 ot_train.py:447 dataset.num_frames=218367 (218K) +INFO 2026-07-22 01:56:21 ot_train.py:448 dataset.num_episodes=4930 +INFO 2026-07-22 01:56:21 ot_train.py:451 Effective batch size: 128 x 1 = 128 +INFO 2026-07-22 01:56:21 ot_train.py:452 num_learnable_params=1622090881 (2B) +INFO 2026-07-22 01:56:21 ot_train.py:453 num_total_params=3145712033 (3B) +INFO 2026-07-22 01:56:21 ot_train.py:494 Resuming data order at epoch 26, sample 82688 +Training: 0%| | 0/30000 [00:00 https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/4bcz2tkp +INFO 2026-07-22 01:58:04 ot_train.py:298 Creating dataset +INFO 2026-07-22 01:58:06 ot_train.py:332 Creating policy +INFO 2026-07-22 01:58:06 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-22 01:58:06 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 11690.53it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 1820.65it/s] +Loading weights from local directory diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/files/requirements.txt b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..23f950ae0bcfce36eb60fb82209b4a99657840c6 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/files/requirements.txt @@ -0,0 +1,117 @@ +tzdata==2026.3 +nvidia-cufile-cu12==1.13.1.3 +wandb==0.27.2 +six==1.17.0 +peft==0.19.1 +nvidia-curand-cu12==10.3.9.90 +nvidia-cuda-runtime-cu12==12.8.90 +datasets==4.8.5 +attrs==26.1.0 +cloudpickle==3.1.2 +packaging==25.0 +nvidia-cublas-cu12==12.8.4.1 +cmake==4.1.3 +psutil==7.2.2 +pydantic==2.13.4 +GitPython==3.1.51 +pillow==12.2.0 +pytz==2026.2 +cuda-pathfinder==1.2.2 +nvidia-cusparse-cu12==12.5.8.93 +typing_extensions==4.15.0 +urllib3==2.7.0 +pydantic_core==2.46.4 +draccus==0.10.0 +mypy_extensions==1.1.0 +dill==0.4.1 +typer==0.26.8 +decord==0.6.0 +tqdm==4.68.4 +triton==3.6.0 +idna==3.18 +requests==2.34.2 +nvidia-cudnn-cu12==9.19.0.56 +mdurl==0.1.2 +shellingham==1.5.4 +zipp==4.1.0 +timm==1.0.28 +torch==2.11.0+cu128 +tokenizers==0.22.2 +nvidia-cusparselt-cu12==0.7.1 +nvidia-cufft-cu12==11.3.3.83 +nvidia-cusolver-cu12==11.7.3.90 +opencv-python-headless==4.13.0.92 +httpx==0.28.1 +Jinja2==3.1.6 +einops==0.8.2 +mergedeep==1.3.4 +platformdirs==4.10.0 +nvidia-nvjitlink-cu12==12.8.93 +lerobot==0.6.0 +PyYAML==6.0.3 +accelerate==1.14.0 +anyio==4.14.2 +gymnasium==1.3.0 +pip==26.1.2 +huggingface_hub==1.23.0 +diffusers==0.35.2 +annotated-doc==0.0.4 +pyarrow==25.0.0 +python-dateutil==2.9.0.post0 +termcolor==3.3.0 +nvidia-nccl-cu12==2.28.9 +sentry-sdk==2.65.0 +torchcodec==0.11.1 +pyyaml-include==1.4.1 +multiprocess==0.70.19 +wrapt==2.2.2 +typing-inspection==0.4.2 +nvidia-cuda-nvrtc-cu12==12.8.93 +Pygments==2.20.0 +nvidia-cuda-cupti-cu12==12.8.90 +certifi==2026.6.17 +setuptools==78.1.0 +charset-normalizer==3.4.9 +propcache==0.5.2 +multidict==6.7.1 +regex==2026.7.10 +frozenlist==1.8.0 +smmap==5.0.3 +hf-xet==1.5.1 +torchvision==0.26.0+cu128 +gitdb==4.0.12 +transformers==5.5.4 +rich==15.0.0 +cuda-bindings==12.9.4 +protobuf==7.35.1 +aiosignal==1.4.0 +jsonlines==4.0.0 +yarl==1.24.2 +filelock==3.29.0 +av==15.1.0 +networkx==3.6.1 +sympy==1.14.0 +wheel==0.47.0 +nvidia-nvtx-cu12==12.8.90 +xxhash==3.8.1 +toml==0.10.2 +MarkupSafe==3.0.3 +pandas==2.3.3 +safetensors==0.8.0 +click==8.4.2 +fsspec==2026.2.0 +dm-tree==0.1.10 +absl-py==2.5.0 +Farama-Notifications==0.0.6 +mpmath==1.3.0 +importlib_metadata==9.0.0 +markdown-it-py==4.2.0 +h11==0.16.0 +typing-inspect==0.9.0 +httpcore==1.0.9 +annotated-types==0.7.0 +numpy==2.2.6 +nvidia-nvshmem-cu12==3.4.5 +aiohappyeyeballs==2.7.1 +cuda-toolkit==12.8.1 +aiohttp==3.14.1 diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/files/wandb-metadata.json b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0b54d3e919c56111d7374f4f70b273c0d02637fd --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/files/wandb-metadata.json @@ -0,0 +1,53 @@ +{ + "os": "Linux-6.8.0-55-generic-x86_64-with-glibc2.39", + "python": "CPython 3.12.13", + "startedAt": "2026-07-22T01:58:03.114527Z", + "args": [ + "--config_path=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model/train_config.json", + "--resume=true", + "--dataset.root=/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2", + "--output_dir=/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K", + "--wandb.run_id=4bcz2tkp" + ], + "program": "/home/ext_minje/clvla/benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", + "codePath": "benchmarks/INSIGHT/my_scripts/lerobot_train_yaml.py", + "codePathLocal": "my_scripts/lerobot_train_yaml.py", + "git": { + "remote": "git@github.com:paragon7060/clvla.git", + "commit": "eed2ce4044cadf00b8e6de57043553236a21b84e" + }, + "root": "/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K", + "host": "ext_minje-box", + "executable": "/home/ext_minje/miniconda3/envs/lerobot060_groot/bin/python", + "cpu_count": 112, + "cpu_count_logical": 224, + "gpu": "NVIDIA H200", + "gpu_count": 2, + "disk": { + "/": { + "total": "1888425144320", + "used": "1113150959616" + } + }, + "memory": { + "total": "2164176568320" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-86be6e5b-fb48-fa1c-9bd7-83d336cb2991" + }, + { + "name": "NVIDIA H200", + "memoryTotal": "150754820096", + "cudaCores": 16896, + "architecture": "Hopper", + "uuid": "GPU-24eb1ba2-7313-ff93-1435-748768ebdbdc" + } + ], + "cudaVersion": "12.8", + "writerId": "5hs1hma1hzp155ydnru132wuadindbw8" +} \ No newline at end of file diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug-core.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0896313323bdfbbc24002ecb7c426bad9eb27da9 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug-core.log @@ -0,0 +1,12 @@ +{"time":"2026-07-22T01:58:03.147104Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpk5qq9p2n/port-4143355.txt","pid":4143355,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2026-07-22T01:58:03.147727281Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":4143355} +{"time":"2026-07-22T01:58:03.147683983Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4143355-4144231-4243726196/socket","Net":"unix"}} +{"time":"2026-07-22T01:58:03.332497197Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2026-07-22T01:58:03.338967354Z","level":"INFO","msg":"handleInformInit: received","streamId":"4bcz2tkp","id":"1(@)"} +{"time":"2026-07-22T01:58:03.704276959Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"4bcz2tkp","id":"1(@)"} +{"time":"2026-07-22T01:58:09.463435777Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"lb53a64eev1p"} +{"time":"2026-07-22T01:58:17.586558841Z","level":"INFO","msg":"processOutgoingData: finished","id":"1(@)"} +{"time":"2026-07-22T01:58:17.586557591Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2026-07-22T01:58:17.586645125Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2026-07-22T01:58:17.586649976Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2026-07-22T01:58:18.568633007Z","level":"INFO","msg":"server: parent process exited, terminating service process"} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug-internal.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..001dd6fb358cb5b322183ff20012dfcdca60b7c9 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug-internal.log @@ -0,0 +1,9 @@ +{"time":"2026-07-22T01:58:03.339107585Z","level":"INFO","msg":"wandb-core"} +{"time":"2026-07-22T01:58:03.339305367Z","level":"INFO","msg":"stream: starting","core version":"0.27.2"} +{"time":"2026-07-22T01:58:03.704070853Z","level":"INFO","msg":"stream: created new stream","id":"4bcz2tkp"} +{"time":"2026-07-22T01:58:03.704180326Z","level":"INFO","msg":"handler: started"} +{"time":"2026-07-22T01:58:03.70427158Z","level":"INFO","msg":"stream: started"} +{"time":"2026-07-22T01:58:03.70446682Z","level":"INFO","msg":"sender: started"} +{"time":"2026-07-22T01:58:03.70446492Z","level":"INFO","msg":"writer: started","stream_id":"4bcz2tkp"} +{"time":"2026-07-22T01:58:04.40415234Z","level":"INFO","msg":"filestream: sending request","total_files":1} +{"time":"2026-07-22T01:58:05.027836924Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"} diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5802a340ec6936373eb3687050e677b63f7edc5d --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug.log @@ -0,0 +1,20 @@ +2026-07-22 01:58:03,115 INFO MainThread:4143355 [wandb_setup.py:_flush():81] Current SDK version is 0.27.2 +2026-07-22 01:58:03,115 INFO MainThread:4143355 [wandb_setup.py:_flush():81] Configure stats pid to 4143355 +2026-07-22 01:58:03,115 INFO MainThread:4143355 [wandb_setup.py:_flush():81] Loading settings from environment variables +2026-07-22 01:58:03,115 INFO MainThread:4143355 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug.log +2026-07-22 01:58:03,115 INFO MainThread:4143355 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/logs/debug-internal.log +2026-07-22 01:58:03,115 INFO MainThread:4143355 [wandb_init.py:init():766] calling init triggers +2026-07-22 01:58:03,115 INFO MainThread:4143355 [wandb_init.py:init():771] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'Whalswp/INSIGHTfixposV4_filtered_multispace_v2', 'root': '/home/ext_minje/INSIGHTfixposV4_filtered_multispace_v2', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'affine': {'weight': 1.0, 'type': 'RandomAffine', 'kwargs': {'degrees': [-5.0, 5.0], 'translate': [0.05, 0.05]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec', 'return_uint8': False, 'depth_output_unit': 'mm', 'streaming': False, 'eval_split': 0.0}, 'env': None, 'policy': {'type': 'groot_rkd', 'n_obs_steps': 1, 'input_features': {'observation.state': {'type': , 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [16]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': None, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '4bcz2tkp', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000', '_wandb': {}} +2026-07-22 01:58:03,115 INFO MainThread:4143355 [wandb_init.py:init():814] starting backend +2026-07-22 01:58:03,332 INFO MainThread:4143355 [wandb_init.py:init():829] sending inform_init request +2026-07-22 01:58:03,705 INFO MainThread:4143355 [wandb_init.py:init():834] backend started and connected +2026-07-22 01:58:03,707 INFO MainThread:4143355 [wandb_init.py:init():904] updated telemetry +2026-07-22 01:58:03,711 INFO MainThread:4143355 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-22 01:58:04,401 INFO MainThread:4143355 [wandb_init.py:init():967] run resumed +2026-07-22 01:58:04,403 INFO MainThread:4143355 [wandb_init.py:init():972] starting run threads in backend +2026-07-22 01:58:04,457 INFO MainThread:4143355 [wandb_run.py:_console_start():2523] atexit reg +2026-07-22 01:58:04,457 INFO MainThread:4143355 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-22 01:58:04,457 INFO MainThread:4143355 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-22 01:58:04,457 INFO MainThread:4143355 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-22 01:58:04,459 INFO MainThread:4143355 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/run-4bcz2tkp.wandb b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/run-4bcz2tkp.wandb new file mode 100644 index 0000000000000000000000000000000000000000..30977402b08eb894463b83d7de7a007255879a32 Binary files /dev/null and b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015803-4bcz2tkp/run-4bcz2tkp.wandb differ diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015846-4bcz2tkp/files/output.log b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015846-4bcz2tkp/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d8f0d8b72f98b8d307db547410435d76a05ba380 --- /dev/null +++ b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_015846-4bcz2tkp/files/output.log @@ -0,0 +1,22 @@ +INFO 2026-07-22 01:58:49 db_utils.py:121 Logs will be synced with wandb. +INFO 2026-07-22 01:58:49 db_utils.py:122 Track this run --> https://wandb.ai/minje227_hyu-hanyang-university/lerobot/runs/4bcz2tkp +INFO 2026-07-22 01:58:49 ot_train.py:298 Creating dataset +INFO 2026-07-22 01:58:50 ot_train.py:332 Creating policy +INFO 2026-07-22 01:58:50 ng_groot.py:193 The Groot policy wraps NVIDIA's GR00T n1.7 model. Loading pretrained model from: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model +INFO 2026-07-22 01:58:50 ng_groot.py:227 Detected fine-tuned LeRobot checkpoint, loading with state dict... +Fetching 27 files: 100%|██████████| 27/27 [00:00<00:00, 5164.93it/s] +`torch_dtype` is deprecated! Use `dtype` instead! +Loading weights: 100%|██████████| 1031/1031 [00:00<00:00, 2035.22it/s] +Loading weights from local directory +INFO 2026-07-22 01:59:02 ot_train.py:405 Creating optimizer and scheduler +INFO 2026-07-22 01:59:04 ot_train.py:439 Output dir: /home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K +INFO 2026-07-22 01:59:04 ot_train.py:446 cfg.steps=60000 (60K) +INFO 2026-07-22 01:59:04 ot_train.py:447 dataset.num_frames=218367 (218K) +INFO 2026-07-22 01:59:04 ot_train.py:448 dataset.num_episodes=4930 +INFO 2026-07-22 01:59:04 ot_train.py:451 Effective batch size: 128 x 1 = 128 +INFO 2026-07-22 01:59:04 ot_train.py:452 num_learnable_params=1622090881 (2B) +INFO 2026-07-22 01:59:04 ot_train.py:453 num_total_params=3145712033 (3B) +INFO 2026-07-22 01:59:04 ot_train.py:494 Resuming data order at epoch 26, sample 82688 +Training: 0%| | 0/30000 [00:00, 'shape': [16]}, 'observation.images.wrist': {'type': , 'shape': [3, 224, 224]}, 'observation.images.right_shoulder': {'type': , 'shape': [3, 224, 224]}, 'observation.images.guide': {'type': , 'shape': [3, 224, 224]}}, 'output_features': {'action': {'type': , 'shape': [16]}}, 'device': 'cuda', 'use_amp': False, 'use_peft': False, 'push_to_hub': False, 'repo_id': None, 'private': None, 'tags': None, 'license': None, 'pretrained_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000/pretrained_model', 'pretrained_revision': None, 'chunk_size': 16, 'n_action_steps': 16, 'max_state_dim': 132, 'max_action_dim': 132, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'base_model_path': 'nvidia/GR00T-N1.7-3B', 'action_decode_transform': None, 'embodiment_tag': 'new_embodiment', 'tune_llm': False, 'tune_visual': False, 'tune_projector': True, 'tune_diffusion_model': True, 'tune_vlln': True, 'tune_top_llm_layers': 0, 'num_inference_timesteps': None, 'rtc_ramp_rate': None, 'use_flash_attention': False, 'use_relative_actions': False, 'relative_exclude_joints': [], 'optimizer_lr': 0.0001, 'optimizer_betas': [0.9, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-05, 'warmup_steps': 3000, 'warmup_ratio': 0.05, 'use_bf16': True, 'model_params_fp32': True, 'image_size': [256, 256], 'tokenizer_assets_repo': None, 'lora_rank': 0, 'lora_alpha': 16, 'lora_dropout': 0.1, 'lora_full_model': False, 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'dataset_paths': None, 'output_dir': './tmp/gr00t', 'save_steps': 1000, 'max_steps': 10000, 'batch_size': 32, 'dataloader_num_workers': 8, 'report_to': 'wandb', 'resume': False, 'action_expert_num_layers': None, 'rkd_enabled': True, 'rkd_vae_checkpoint': '/home/ext_minje/groot_insight/TimewarpVAE_ckpt/multispace_gripper_dedup_batch_ablation_v1/b512/RD/K08_z16_g4_h256/checkpoint-step-20000-epoch-46.84', 'rkd_student_pooling': 'attention', 'rkd_student_attention_hidden_dim': 512, 'rkd_student_attention_output_dim': 512, 'rkd_fm_loss_weight': 1.0, 'rkd_loss_weight': 1.0, 'rkd_distance_weight': 1.0, 'rkd_angle_weight': 2.0}, 'reward_model': None, 'output_dir': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K', 'job_name': 'INSIGHT_RD_RKD_TimewarpVAE_RSCLstyle_C60', 'resume': True, 'seed': 42, 'cudnn_deterministic': False, 'num_workers': 4, 'batch_size': 128, 'prefetch_factor': 4, 'persistent_workers': True, 'steps': 60000, 'env_eval_freq': 20000, 'log_freq': 200, 'eval_steps': 0, 'max_eval_samples': 0, 'tolerance_s': 0.0001, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 0.0001, 'weight_decay': 1e-05, 'grad_clip_norm': 1.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 3000, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': True, 'recording': False, 'recording_repo_id': None, 'recording_private': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': '4bcz2tkp', 'mode': None, 'add_tags': True}, 'peft': None, 'job': {'target': None, 'image': 'huggingface/lerobot-gpu:latest', 'timeout': '2d', 'detach': False, 'tags': []}, 'save_checkpoint_to_hub': False, 'sample_weighting': None, 'rename_map': {}, 'checkpoint_path': '/home/ext_minje/groot_insight/train_ckpt/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/checkpoints/030000', '_wandb': {}} +2026-07-22 01:58:46,819 INFO MainThread:4148992 [wandb_init.py:init():814] starting backend +2026-07-22 01:58:47,036 INFO MainThread:4148992 [wandb_init.py:init():829] sending inform_init request +2026-07-22 01:58:47,507 INFO MainThread:4148992 [wandb_init.py:init():834] backend started and connected +2026-07-22 01:58:47,509 INFO MainThread:4148992 [wandb_init.py:init():904] updated telemetry +2026-07-22 01:58:47,513 INFO MainThread:4148992 [wandb_init.py:init():927] communicating run to backend with 90.0 second timeout +2026-07-22 01:58:49,000 INFO MainThread:4148992 [wandb_init.py:init():967] run resumed +2026-07-22 01:58:49,002 INFO MainThread:4148992 [wandb_init.py:init():972] starting run threads in backend +2026-07-22 01:58:49,058 INFO MainThread:4148992 [wandb_run.py:_console_start():2523] atexit reg +2026-07-22 01:58:49,058 INFO MainThread:4148992 [wandb_run.py:_redirect():2373] redirect: wrap_raw +2026-07-22 01:58:49,059 INFO MainThread:4148992 [wandb_run.py:_redirect():2442] Wrapping output streams. +2026-07-22 01:58:49,059 INFO MainThread:4148992 [wandb_run.py:_redirect():2465] Redirects installed. +2026-07-22 01:58:49,061 INFO MainThread:4148992 [wandb_init.py:init():1010] run started, returning control to user process diff --git a/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_032005-4bcz2tkp/run-4bcz2tkp.wandb b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_032005-4bcz2tkp/run-4bcz2tkp.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5881b0afd5685da6ddd79cb244f06bb729b26d13 Binary files /dev/null and b/Abs_6D + DeltaEE/restarts/20260719_231636_rerun_save30k60k/RSCLstyle_cosine_60K/wandb/run-20260722_032005-4bcz2tkp/run-4bcz2tkp.wandb differ