diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..5b04e598c5b1b9776a4d97bf9f904f0222559962 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +wandb/latest-run/run-7fd56jqi.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20250907_001603-7fd56jqi/run-7fd56jqi.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/checkpoints/001000/pretrained_model/config.json b/checkpoints/001000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb0a3f97346fc76f276c18f3fa34df929d6302b --- /dev/null +++ b/checkpoints/001000/pretrained_model/config.json @@ -0,0 +1,65 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/checkpoints/001000/pretrained_model/model.safetensors b/checkpoints/001000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f66b9cbf1720d16414a6e9a72db263e1f02ce429 --- /dev/null +++ b/checkpoints/001000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:020d2fc9c027a05260fd70c93381488e0aa0f917a6520ab13059c1946f641c07 +size 206463376 diff --git a/checkpoints/001000/pretrained_model/train_config.json b/checkpoints/001000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..97ec2a98ed20ecee7d0d67136b2f303ead343232 --- /dev/null +++ b/checkpoints/001000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "pusht_60_v5", + "root": null, + "episodes": null, + "image_transforms": { + "enable": true, + "max_num_transforms": 3, + "random_order": true, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "perspective": { + "weight": 1.0, + "type": "RandomPerspective", + "kwargs": { + "distortion_scale": 0.025, + "p": 1.0 + } + }, + "rotation": { + "weight": 1.0, + "type": "RandomRotation", + "kwargs": { + "degrees": [ + -5, + 5 + ] + } + }, + "crop": { + "weight": 1.0, + "type": "RandomCrop", + "kwargs": { + "size": [ + 200, + 200 + ] + } + } + }, + "downsample": { + "resize": { + "weight": 1.0, + "type": "Resize", + "kwargs": { + "size": [ + 240, + 240 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "job_name": "2025-09-07_00-15-58_act_abs_v5", + "resume": false, + "seed": 100000, + "num_workers": 4, + "batch_size": 512, + "steps": 20000, + "eval_freq": 2000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 1000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/001000/training_state/optimizer_param_groups.json b/checkpoints/001000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/checkpoints/001000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/checkpoints/001000/training_state/optimizer_state.safetensors b/checkpoints/001000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c433be2da8a05badb261929d853a805e65aaa511 --- /dev/null +++ b/checkpoints/001000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ef6b5e680cfd1b392bb2953656eac5ceb104e8f4c590387f11e1ca073aff780 +size 412547284 diff --git a/checkpoints/001000/training_state/rng_state.safetensors b/checkpoints/001000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..59241b0044c7efc7f4f4f354047caf10792b7f9c --- /dev/null +++ b/checkpoints/001000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6eadfa1f674d3ccccf26430052dd368b44f382d356ab0d97567e56373456b966 +size 15708 diff --git a/checkpoints/001000/training_state/training_step.json b/checkpoints/001000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d98c94f5b78238bf495ac68b9f9fb446cfac5c07 --- /dev/null +++ b/checkpoints/001000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 1000 +} \ No newline at end of file diff --git a/checkpoints/002000/pretrained_model/config.json b/checkpoints/002000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb0a3f97346fc76f276c18f3fa34df929d6302b --- /dev/null +++ b/checkpoints/002000/pretrained_model/config.json @@ -0,0 +1,65 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/checkpoints/002000/pretrained_model/model.safetensors b/checkpoints/002000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6eb4d998d80f42fc53f08a9c64bad4dc9913ecc1 --- /dev/null +++ b/checkpoints/002000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:729b5b6b4b434182720ffb94099495c6bc36b0c530269be4268e2bbe7c321255 +size 206463376 diff --git a/checkpoints/002000/pretrained_model/train_config.json b/checkpoints/002000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..97ec2a98ed20ecee7d0d67136b2f303ead343232 --- /dev/null +++ b/checkpoints/002000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "pusht_60_v5", + "root": null, + "episodes": null, + "image_transforms": { + "enable": true, + "max_num_transforms": 3, + "random_order": true, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "perspective": { + "weight": 1.0, + "type": "RandomPerspective", + "kwargs": { + "distortion_scale": 0.025, + "p": 1.0 + } + }, + "rotation": { + "weight": 1.0, + "type": "RandomRotation", + "kwargs": { + "degrees": [ + -5, + 5 + ] + } + }, + "crop": { + "weight": 1.0, + "type": "RandomCrop", + "kwargs": { + "size": [ + 200, + 200 + ] + } + } + }, + "downsample": { + "resize": { + "weight": 1.0, + "type": "Resize", + "kwargs": { + "size": [ + 240, + 240 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "job_name": "2025-09-07_00-15-58_act_abs_v5", + "resume": false, + "seed": 100000, + "num_workers": 4, + "batch_size": 512, + "steps": 20000, + "eval_freq": 2000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 1000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/002000/training_state/optimizer_param_groups.json b/checkpoints/002000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/checkpoints/002000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/checkpoints/002000/training_state/optimizer_state.safetensors b/checkpoints/002000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0aedf33740602de6f515c973eb08581d08ca6f5f --- /dev/null +++ b/checkpoints/002000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:422c68efe76edc51735c9315416c8ccb39a6c28ec644449afea69db32af1f8bd +size 412547284 diff --git a/checkpoints/002000/training_state/rng_state.safetensors b/checkpoints/002000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..11c69ab940ca268f9d2fbbba3c3cc2b29a934080 --- /dev/null +++ b/checkpoints/002000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db04b4dbf8a133b35c0130f41ed5e18e7b1c12a89d4707de501fdc16fb7818d1 +size 15708 diff --git a/checkpoints/002000/training_state/training_step.json b/checkpoints/002000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..39090bbb986edb821e1602990d19357dcdb5d2ae --- /dev/null +++ b/checkpoints/002000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 2000 +} \ No newline at end of file diff --git a/checkpoints/003000/pretrained_model/config.json b/checkpoints/003000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb0a3f97346fc76f276c18f3fa34df929d6302b --- /dev/null +++ b/checkpoints/003000/pretrained_model/config.json @@ -0,0 +1,65 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/checkpoints/003000/pretrained_model/model.safetensors b/checkpoints/003000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..96a9b02676d3957fdfc431d4c151ead9330e73bf --- /dev/null +++ b/checkpoints/003000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:992ed2f8d512f33e2004e68316834ea8dc1dd767e0a639d5208067f62f021fd9 +size 206463376 diff --git a/checkpoints/003000/pretrained_model/train_config.json b/checkpoints/003000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..97ec2a98ed20ecee7d0d67136b2f303ead343232 --- /dev/null +++ b/checkpoints/003000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "pusht_60_v5", + "root": null, + "episodes": null, + "image_transforms": { + "enable": true, + "max_num_transforms": 3, + "random_order": true, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "perspective": { + "weight": 1.0, + "type": "RandomPerspective", + "kwargs": { + "distortion_scale": 0.025, + "p": 1.0 + } + }, + "rotation": { + "weight": 1.0, + "type": "RandomRotation", + "kwargs": { + "degrees": [ + -5, + 5 + ] + } + }, + "crop": { + "weight": 1.0, + "type": "RandomCrop", + "kwargs": { + "size": [ + 200, + 200 + ] + } + } + }, + "downsample": { + "resize": { + "weight": 1.0, + "type": "Resize", + "kwargs": { + "size": [ + 240, + 240 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "job_name": "2025-09-07_00-15-58_act_abs_v5", + "resume": false, + "seed": 100000, + "num_workers": 4, + "batch_size": 512, + "steps": 20000, + "eval_freq": 2000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 1000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/003000/training_state/optimizer_param_groups.json b/checkpoints/003000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/checkpoints/003000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/checkpoints/003000/training_state/optimizer_state.safetensors b/checkpoints/003000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3a424ed6753f7bfd2387addde2b5098095d019fd --- /dev/null +++ b/checkpoints/003000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc1f2d283d11471e346b731c314dcc4bbac1f620a4d854d279f9b3e4265e9497 +size 412547284 diff --git a/checkpoints/003000/training_state/rng_state.safetensors b/checkpoints/003000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0f41e4855207c61c3fe678fd33bc503b91f9c925 --- /dev/null +++ b/checkpoints/003000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd4bf23d64701cdc36136ad7da04d728eff26b25aa73b0389cfdeead16322434 +size 15708 diff --git a/checkpoints/003000/training_state/training_step.json b/checkpoints/003000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..c1a44127b7dfea653fd776d529fa83c55d32081c --- /dev/null +++ b/checkpoints/003000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 3000 +} \ No newline at end of file diff --git a/checkpoints/004000/pretrained_model/config.json b/checkpoints/004000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb0a3f97346fc76f276c18f3fa34df929d6302b --- /dev/null +++ b/checkpoints/004000/pretrained_model/config.json @@ -0,0 +1,65 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/checkpoints/004000/pretrained_model/model.safetensors b/checkpoints/004000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8788e8ba494c8f8e14ff55fbadf7f1e9ab33a7c --- /dev/null +++ b/checkpoints/004000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f2b915036430d64363d79301d4e285e37e5e0556f8298731d537a0d5a34def +size 206463376 diff --git a/checkpoints/004000/pretrained_model/train_config.json b/checkpoints/004000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..97ec2a98ed20ecee7d0d67136b2f303ead343232 --- /dev/null +++ b/checkpoints/004000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "pusht_60_v5", + "root": null, + "episodes": null, + "image_transforms": { + "enable": true, + "max_num_transforms": 3, + "random_order": true, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "perspective": { + "weight": 1.0, + "type": "RandomPerspective", + "kwargs": { + "distortion_scale": 0.025, + "p": 1.0 + } + }, + "rotation": { + "weight": 1.0, + "type": "RandomRotation", + "kwargs": { + "degrees": [ + -5, + 5 + ] + } + }, + "crop": { + "weight": 1.0, + "type": "RandomCrop", + "kwargs": { + "size": [ + 200, + 200 + ] + } + } + }, + "downsample": { + "resize": { + "weight": 1.0, + "type": "Resize", + "kwargs": { + "size": [ + 240, + 240 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "job_name": "2025-09-07_00-15-58_act_abs_v5", + "resume": false, + "seed": 100000, + "num_workers": 4, + "batch_size": 512, + "steps": 20000, + "eval_freq": 2000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 1000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/004000/training_state/optimizer_param_groups.json b/checkpoints/004000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/checkpoints/004000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/checkpoints/004000/training_state/optimizer_state.safetensors b/checkpoints/004000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd1cc11478f025ca19ffa40d1cc54cc21dff4c95 --- /dev/null +++ b/checkpoints/004000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:facf43c01c61c9544c1a2909980219676a7227db836b73fc5267ed7a0d79f113 +size 412547284 diff --git a/checkpoints/004000/training_state/rng_state.safetensors b/checkpoints/004000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8250e91d5fae8bd618ad77b92f07e6c3d0776a35 --- /dev/null +++ b/checkpoints/004000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dea718e10ce112ecb1819309f2e726844aa0072a3f5e9a9d97223efb9be0b8cc +size 15708 diff --git a/checkpoints/004000/training_state/training_step.json b/checkpoints/004000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..75d870521c191f77dd9eaa4d83486eab6e768f69 --- /dev/null +++ b/checkpoints/004000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 4000 +} \ No newline at end of file diff --git a/checkpoints/005000/pretrained_model/config.json b/checkpoints/005000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb0a3f97346fc76f276c18f3fa34df929d6302b --- /dev/null +++ b/checkpoints/005000/pretrained_model/config.json @@ -0,0 +1,65 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/checkpoints/005000/pretrained_model/model.safetensors b/checkpoints/005000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..87beb739110fc74d2b9f152705a96a3736f6c7d6 --- /dev/null +++ b/checkpoints/005000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1518f08ff3c9ab2187d8dc74fb37f4a0e06979ce2d1466c929508d96b28a0240 +size 206463376 diff --git a/checkpoints/005000/pretrained_model/train_config.json b/checkpoints/005000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..97ec2a98ed20ecee7d0d67136b2f303ead343232 --- /dev/null +++ b/checkpoints/005000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "pusht_60_v5", + "root": null, + "episodes": null, + "image_transforms": { + "enable": true, + "max_num_transforms": 3, + "random_order": true, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "perspective": { + "weight": 1.0, + "type": "RandomPerspective", + "kwargs": { + "distortion_scale": 0.025, + "p": 1.0 + } + }, + "rotation": { + "weight": 1.0, + "type": "RandomRotation", + "kwargs": { + "degrees": [ + -5, + 5 + ] + } + }, + "crop": { + "weight": 1.0, + "type": "RandomCrop", + "kwargs": { + "size": [ + 200, + 200 + ] + } + } + }, + "downsample": { + "resize": { + "weight": 1.0, + "type": "Resize", + "kwargs": { + "size": [ + 240, + 240 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "job_name": "2025-09-07_00-15-58_act_abs_v5", + "resume": false, + "seed": 100000, + "num_workers": 4, + "batch_size": 512, + "steps": 20000, + "eval_freq": 2000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 1000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/005000/training_state/optimizer_param_groups.json b/checkpoints/005000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/checkpoints/005000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/checkpoints/005000/training_state/optimizer_state.safetensors b/checkpoints/005000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2b92f11cc3aa2cc4d1db29ec8ca40b41e5b264c3 --- /dev/null +++ b/checkpoints/005000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d9de02b2d02ff90245ce78756fdcfcae45d367962eaa72c322fc24cea0ccf5a +size 412547284 diff --git a/checkpoints/005000/training_state/rng_state.safetensors b/checkpoints/005000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..85bd9d8c29cc9c3e6699fd959127a87cc20ab5d1 --- /dev/null +++ b/checkpoints/005000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab7d2c7ec5a229efcbb32ca18fbc91117053c07b01d5f1311fc0225ef5f367ae +size 15708 diff --git a/checkpoints/005000/training_state/training_step.json b/checkpoints/005000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..592449d3fc8b35c30c604eb1dabe60537e8224a0 --- /dev/null +++ b/checkpoints/005000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 5000 +} \ No newline at end of file diff --git a/checkpoints/006000/pretrained_model/config.json b/checkpoints/006000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb0a3f97346fc76f276c18f3fa34df929d6302b --- /dev/null +++ b/checkpoints/006000/pretrained_model/config.json @@ -0,0 +1,65 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/checkpoints/006000/pretrained_model/model.safetensors b/checkpoints/006000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..59391d055cb388587a9d0f100808b3438a29129b --- /dev/null +++ b/checkpoints/006000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d29ec89439c2316e82b1587a19a71e040074bf06c6fcffc6f2e87775ec3b54d +size 206463376 diff --git a/checkpoints/006000/pretrained_model/train_config.json b/checkpoints/006000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..97ec2a98ed20ecee7d0d67136b2f303ead343232 --- /dev/null +++ b/checkpoints/006000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "pusht_60_v5", + "root": null, + "episodes": null, + "image_transforms": { + "enable": true, + "max_num_transforms": 3, + "random_order": true, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "perspective": { + "weight": 1.0, + "type": "RandomPerspective", + "kwargs": { + "distortion_scale": 0.025, + "p": 1.0 + } + }, + "rotation": { + "weight": 1.0, + "type": "RandomRotation", + "kwargs": { + "degrees": [ + -5, + 5 + ] + } + }, + "crop": { + "weight": 1.0, + "type": "RandomCrop", + "kwargs": { + "size": [ + 200, + 200 + ] + } + } + }, + "downsample": { + "resize": { + "weight": 1.0, + "type": "Resize", + "kwargs": { + "size": [ + 240, + 240 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "job_name": "2025-09-07_00-15-58_act_abs_v5", + "resume": false, + "seed": 100000, + "num_workers": 4, + "batch_size": 512, + "steps": 20000, + "eval_freq": 2000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 1000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/006000/training_state/optimizer_param_groups.json b/checkpoints/006000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/checkpoints/006000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/checkpoints/006000/training_state/optimizer_state.safetensors b/checkpoints/006000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..363c58cd26256cd6cad070bec4586d1b74d3c4f5 --- /dev/null +++ b/checkpoints/006000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98d308855d2e14b97cbd43c8796d04d0ed701366edc48c242bb06d2ded52c1e3 +size 412547284 diff --git a/checkpoints/006000/training_state/rng_state.safetensors b/checkpoints/006000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dd206f2e3bf9887de0c5b7979052327684921bdc --- /dev/null +++ b/checkpoints/006000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:468da71908520580c7c1bdcded078acaa040698a397131f1e01dd22f729ad463 +size 15708 diff --git a/checkpoints/006000/training_state/training_step.json b/checkpoints/006000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..e267ac589be64705f8674638b9f5099c886778da --- /dev/null +++ b/checkpoints/006000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 6000 +} \ No newline at end of file diff --git a/checkpoints/007000/pretrained_model/config.json b/checkpoints/007000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb0a3f97346fc76f276c18f3fa34df929d6302b --- /dev/null +++ b/checkpoints/007000/pretrained_model/config.json @@ -0,0 +1,65 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/checkpoints/007000/pretrained_model/model.safetensors b/checkpoints/007000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..073011b794c28224a51de9a08e37d20fd74087a7 --- /dev/null +++ b/checkpoints/007000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ab8605806082228ba0ac215d7cfda0c1d6f14dbf77a80eb7eb5c3927f8e8930 +size 206463376 diff --git a/checkpoints/007000/pretrained_model/train_config.json b/checkpoints/007000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..97ec2a98ed20ecee7d0d67136b2f303ead343232 --- /dev/null +++ b/checkpoints/007000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "pusht_60_v5", + "root": null, + "episodes": null, + "image_transforms": { + "enable": true, + "max_num_transforms": 3, + "random_order": true, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "perspective": { + "weight": 1.0, + "type": "RandomPerspective", + "kwargs": { + "distortion_scale": 0.025, + "p": 1.0 + } + }, + "rotation": { + "weight": 1.0, + "type": "RandomRotation", + "kwargs": { + "degrees": [ + -5, + 5 + ] + } + }, + "crop": { + "weight": 1.0, + "type": "RandomCrop", + "kwargs": { + "size": [ + 200, + 200 + ] + } + } + }, + "downsample": { + "resize": { + "weight": 1.0, + "type": "Resize", + "kwargs": { + "size": [ + 240, + 240 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "job_name": "2025-09-07_00-15-58_act_abs_v5", + "resume": false, + "seed": 100000, + "num_workers": 4, + "batch_size": 512, + "steps": 20000, + "eval_freq": 2000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 1000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/007000/training_state/optimizer_param_groups.json b/checkpoints/007000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/checkpoints/007000/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/checkpoints/007000/training_state/optimizer_state.safetensors b/checkpoints/007000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ee3c218a720897fe8a96b227a5caa230612e0ea5 --- /dev/null +++ b/checkpoints/007000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:311d74b61d32243c0989ffac3592202cbc7d9a918301ab674439e861dc658425 +size 412547284 diff --git a/checkpoints/007000/training_state/rng_state.safetensors b/checkpoints/007000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7cf2d92b2c807c4fd4d482cea3f55315e35b2efd --- /dev/null +++ b/checkpoints/007000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc8724a606eabf9298bbd5aa5ed4288663c5e24de08aef1a5807e61ebff2bc0c +size 15708 diff --git a/checkpoints/007000/training_state/training_step.json b/checkpoints/007000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..1f4c730c4d88ece076ae9d4e01ed7fbe60b0415d --- /dev/null +++ b/checkpoints/007000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 7000 +} \ No newline at end of file diff --git a/checkpoints/last/pretrained_model/config.json b/checkpoints/last/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb0a3f97346fc76f276c18f3fa34df929d6302b --- /dev/null +++ b/checkpoints/last/pretrained_model/config.json @@ -0,0 +1,65 @@ +{ + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 +} \ No newline at end of file diff --git a/checkpoints/last/pretrained_model/model.safetensors b/checkpoints/last/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..073011b794c28224a51de9a08e37d20fd74087a7 --- /dev/null +++ b/checkpoints/last/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ab8605806082228ba0ac215d7cfda0c1d6f14dbf77a80eb7eb5c3927f8e8930 +size 206463376 diff --git a/checkpoints/last/pretrained_model/train_config.json b/checkpoints/last/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..97ec2a98ed20ecee7d0d67136b2f303ead343232 --- /dev/null +++ b/checkpoints/last/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "pusht_60_v5", + "root": null, + "episodes": null, + "image_transforms": { + "enable": true, + "max_num_transforms": 3, + "random_order": true, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + }, + "perspective": { + "weight": 1.0, + "type": "RandomPerspective", + "kwargs": { + "distortion_scale": 0.025, + "p": 1.0 + } + }, + "rotation": { + "weight": 1.0, + "type": "RandomRotation", + "kwargs": { + "degrees": [ + -5, + 5 + ] + } + }, + "crop": { + "weight": 1.0, + "type": "RandomCrop", + "kwargs": { + "size": [ + 200, + 200 + ] + } + } + }, + "downsample": { + "resize": { + "weight": 1.0, + "type": "Resize", + "kwargs": { + "size": [ + 240, + 240 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "act", + "n_obs_steps": 1, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MEAN_STD", + "ACTION": "MEAN_STD" + }, + "relative_actions": false, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 2 + ] + }, + "observation.images.front": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + }, + "observation.images.wrist": { + "type": "VISUAL", + "shape": [ + 3, + 240, + 240 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 2 + ] + } + }, + "device": "cuda", + "use_amp": false, + "chunk_size": 50, + "n_action_steps": 50, + "vision_backbone": "resnet18", + "pretrained_backbone_weights": "ResNet18_Weights.IMAGENET1K_V1", + "replace_final_stride_with_dilation": 0, + "pre_norm": false, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "temporal_ensemble_coeff": null, + "dropout": 0.1, + "kl_weight": 10.0, + "optimizer_lr": 1e-05, + "optimizer_weight_decay": 0.0001, + "optimizer_lr_backbone": 1e-05 + }, + "output_dir": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "job_name": "2025-09-07_00-15-58_act_abs_v5", + "resume": false, + "seed": 100000, + "num_workers": 4, + "batch_size": 512, + "steps": 20000, + "eval_freq": 2000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 1000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adamw", + "lr": 1e-05, + "weight_decay": 0.0001, + "grad_clip_norm": 10.0, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": null, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": true, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/checkpoints/last/training_state/optimizer_param_groups.json b/checkpoints/last/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..33506df097b87c64b2aa15765ab89706780c0dc0 --- /dev/null +++ b/checkpoints/last/training_state/optimizer_param_groups.json @@ -0,0 +1,191 @@ +[ + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132 + ] + }, + { + "lr": 1e-05, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 0.0001, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "decoupled_weight_decay": true, + "params": [ + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152 + ] + } +] \ No newline at end of file diff --git a/checkpoints/last/training_state/optimizer_state.safetensors b/checkpoints/last/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ee3c218a720897fe8a96b227a5caa230612e0ea5 --- /dev/null +++ b/checkpoints/last/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:311d74b61d32243c0989ffac3592202cbc7d9a918301ab674439e861dc658425 +size 412547284 diff --git a/checkpoints/last/training_state/rng_state.safetensors b/checkpoints/last/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7cf2d92b2c807c4fd4d482cea3f55315e35b2efd --- /dev/null +++ b/checkpoints/last/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc8724a606eabf9298bbd5aa5ed4288663c5e24de08aef1a5807e61ebff2bc0c +size 15708 diff --git a/checkpoints/last/training_state/training_step.json b/checkpoints/last/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..1f4c730c4d88ece076ae9d4e01ed7fbe60b0415d --- /dev/null +++ b/checkpoints/last/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 7000 +} \ No newline at end of file diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ebc2d2a58835f0787067a9bb4095819f798697ce --- /dev/null +++ b/wandb/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2025-09-07T00:16:03.541599108Z","level":"INFO","msg":"stream: starting","core version":"0.21.0"} +{"time":"2025-09-07T00:16:03.717802463Z","level":"INFO","msg":"stream: created new stream","id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717830474Z","level":"INFO","msg":"stream: started","id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717843784Z","level":"INFO","msg":"handler: started","stream_id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717862125Z","level":"INFO","msg":"sender: started","stream_id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717879775Z","level":"INFO","msg":"writer: Do: started","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.47360162Z","level":"INFO","msg":"stream: closing","id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.766374646Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-07T04:10:26.87005929Z","level":"INFO","msg":"handler: closed","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.870105072Z","level":"INFO","msg":"sender: closed","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.870094282Z","level":"INFO","msg":"writer: Close: closed","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.870214416Z","level":"INFO","msg":"stream: closed","id":"7fd56jqi"} diff --git a/wandb/debug.log b/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b184fb9bfad3a85f5e7ee81fdd7f00e5059cd921 --- /dev/null +++ b/wandb/debug.log @@ -0,0 +1,22 @@ +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0 +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Configure stats pid to 6396 +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/projects/policy/wandb/settings +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5/wandb/run-20250907_001603-7fd56jqi/logs/debug.log +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5/wandb/run-20250907_001603-7fd56jqi/logs/debug-internal.log +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:init():830] calling init triggers +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'pusht_60_v5', 'root': None, 'episodes': None, 'image_transforms': {'enable': True, 'max_num_transforms': 3, 'random_order': True, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'perspective': {'weight': 1.0, 'type': 'RandomPerspective', 'kwargs': {'distortion_scale': 0.025, 'p': 1.0}}, 'rotation': {'weight': 1.0, 'type': 'RandomRotation', 'kwargs': {'degrees': [-5, 5]}}, 'crop': {'weight': 1.0, 'type': 'RandomCrop', 'kwargs': {'size': [200, 200]}}}, 'downsample': {'resize': {'weight': 1.0, 'type': 'Resize', 'kwargs': {'size': [240, 240]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'relative_actions': False, 'input_features': {'observation.image.side': {'type': , 'shape': [3, 240, 240]}, 'observation.image.wrist': {'type': , 'shape': [3, 240, 240]}, 'observation.state': {'type': , 'shape': [2]}}, 'output_features': {'action': {'type': , 'shape': [2]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 50, 'n_action_steps': 50, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': 0, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5', 'job_name': '2025-09-07_00-15-58_act_abs_v5', 'resume': False, 'seed': 100000, 'num_workers': 4, 'batch_size': 512, 'steps': 20000, 'eval_freq': 2000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 1000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:init():871] starting backend +2025-09-07 00:16:03,536 INFO MainThread:6396 [wandb_init.py:init():874] sending inform_init request +2025-09-07 00:16:03,539 INFO MainThread:6396 [wandb_init.py:init():882] backend started and connected +2025-09-07 00:16:03,541 INFO MainThread:6396 [wandb_init.py:init():953] updated telemetry +2025-09-07 00:16:03,544 INFO MainThread:6396 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-09-07 00:16:03,890 INFO MainThread:6396 [wandb_init.py:init():1029] starting run threads in backend +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_console_start():2458] atexit reg +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_redirect():2375] Wrapping output streams. +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_redirect():2398] Redirects installed. +2025-09-07 00:16:03,974 INFO MainThread:6396 [wandb_init.py:init():1075] run started, returning control to user process +2025-09-07 04:10:26,472 INFO MsgRouterThr:6396 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/wandb/latest-run/files/config.yaml b/wandb/latest-run/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb75e6935bf53a3b951f26b8d75154d4747b848 --- /dev/null +++ b/wandb/latest-run/files/config.yaml @@ -0,0 +1,251 @@ +_wandb: + value: + cli_version: 0.21.0 + e: + fzxncgo02qx7l3nog6m82ys6oh1pwriz: + args: + - --config_path=src/training_cfg/pusht/act_abs.json + - --dataset.repo_id=pusht_60_v5 + - --job_name=2025-09-07_00-15-58_act_abs_v5 + - --output_dir=outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5 + codePath: lerobot/scripts/train.py + codePathLocal: lerobot/scripts/train.py + cpu_count: 4 + cpu_count_logical: 8 + cudaVersion: "12.8" + disk: + /: + total: "1063956480000" + used: "162176774144" + executable: /home/ubuntu/miniconda3/envs/lerobot/bin/python + git: + commit: e3e730f94ee641237d6e067d143a1794c6c9e5aa + remote: git@github.com:SceniX-AI/policy.git + gpu: NVIDIA L40S + gpu_count: 1 + gpu_nvidia: + - architecture: Ada + cudaCores: 18176 + memoryTotal: "48305799168" + name: NVIDIA L40S + uuid: GPU-aa974e28-ca07-9688-5ab1-68deb2b19654 + host: ip-172-31-0-58 + memory: + total: "66507816960" + os: Linux-6.8.0-1031-aws-x86_64-with-glibc2.39 + program: /home/ubuntu/projects/policy/lerobot/scripts/train.py + python: CPython 3.10.13 + root: outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5 + startedAt: "2025-09-07T00:16:03.329573Z" + writerId: fzxncgo02qx7l3nog6m82ys6oh1pwriz + m: [] + python_version: 3.10.13 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + "3": + - 13 + - 15 + - 16 + - 61 + "4": 3.10.13 + "5": 0.21.0 + "6": 4.53.1 + "10": + - 21 + "12": 0.21.0 + "13": linux-x86_64 +batch_size: + value: 512 +dataset: + value: + episodes: null + image_transforms: + downsample: + resize: + kwargs: + size: + - 240 + - 240 + type: Resize + weight: 1 + enable: true + max_num_transforms: 3 + random_order: true + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + crop: + kwargs: + size: + - 200 + - 200 + type: RandomCrop + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + perspective: + kwargs: + distortion_scale: 0.025 + p: 1 + type: RandomPerspective + weight: 1 + rotation: + kwargs: + degrees: + - -5 + - 5 + type: RandomRotation + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: pusht_60_v5 + revision: null + root: null + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 2000 +job_name: + value: 2025-09-07_00-15-58_act_abs_v5 +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 1e-05 + type: adamw + weight_decay: 0.0001 +output_dir: + value: outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5 +policy: + value: + chunk_size: 50 + device: cuda + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + input_features: + observation.image.side: + shape: + - 3 + - 240 + - 240 + type: VISUAL + observation.image.wrist: + shape: + - 3 + - 240 + - 240 + type: VISUAL + observation.state: + shape: + - 2 + type: STATE + kl_weight: 10 + latent_dim: 32 + n_action_steps: 50 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: MEAN_STD + optimizer_lr: 1e-05 + optimizer_lr_backbone: 1e-05 + optimizer_weight_decay: 0.0001 + output_features: + action: + shape: + - 2 + type: ACTION + pre_norm: false + pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1 + relative_actions: false + replace_final_stride_with_dilation: 0 + temporal_ensemble_coeff: null + type: act + use_amp: false + use_vae: true + vision_backbone: resnet18 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 1000 +scheduler: + value: null +seed: + value: 100000 +steps: + value: 20000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: true + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/wandb/latest-run/files/output.log b/wandb/latest-run/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7f7858f8db52fc34943d826258a53d5973a3e7b8 --- /dev/null +++ b/wandb/latest-run/files/output.log @@ -0,0 +1,101 @@ +Logs will be synced with wandb. +INFO 2025-09-07 00:16:03 ndb_utils.py:96 Track this run --> https://wandb.ai/ss7050-columbia/lerobot/runs/7fd56jqi +INFO 2025-09-07 00:16:03 ts/train.py:260 Creating dataset +Resolving data files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:00<00:00, 308783.12it/s] +INFO 2025-09-07 00:16:04 ts/train.py:272 Creating policy +INFO 2025-09-07 00:16:04 ts/train.py:275 Using relative actions: False +INFO 2025-09-07 00:16:05 ts/train.py:301 Creating optimizer and scheduler +INFO 2025-09-07 00:16:05 ts/train.py:313 Output dir: outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5 +INFO 2025-09-07 00:16:05 ts/train.py:316 cfg.steps=20000 (20K) +INFO 2025-09-07 00:16:05 ts/train.py:317 dataset.num_frames=47582 (48K) +INFO 2025-09-07 00:16:05 ts/train.py:318 dataset.num_episodes=60 +INFO 2025-09-07 00:16:05 ts/train.py:319 num_learnable_params=51563394 (52M) +INFO 2025-09-07 00:16:05 ts/train.py:320 num_total_params=51563418 (52M) +INFO 2025-09-07 00:16:05 ts/train.py:359 Start offline training on a fixed dataset +INFO 2025-09-07 00:22:39 ts/train.py:392 step:200 smpl:102K ep:129 epch:2.15 loss:5.003 grdn:75.835 lr:1.0e-05 updt_s:0.900 data_s:0.813 +INFO 2025-09-07 00:28:54 ts/train.py:392 step:400 smpl:205K ep:258 epch:4.30 loss:1.574 grdn:26.507 lr:1.0e-05 updt_s:0.872 data_s:0.747 +INFO 2025-09-07 00:35:18 ts/train.py:392 step:600 smpl:307K ep:387 epch:6.46 loss:1.032 grdn:23.980 lr:1.0e-05 updt_s:0.872 data_s:0.791 +INFO 2025-09-07 00:41:37 ts/train.py:392 step:800 smpl:410K ep:516 epch:8.61 loss:0.701 grdn:19.744 lr:1.0e-05 updt_s:0.872 data_s:0.766 +INFO 2025-09-07 00:47:54 ts/train.py:392 step:1K smpl:512K ep:646 epch:10.76 loss:0.482 grdn:17.509 lr:1.0e-05 updt_s:0.872 data_s:0.754 +INFO 2025-09-07 00:47:54 ts/train.py:401 Checkpoint policy after step 1000 +INFO 2025-09-07 00:54:10 ts/train.py:392 step:1K smpl:614K ep:775 epch:12.91 loss:0.343 grdn:14.252 lr:1.0e-05 updt_s:0.872 data_s:0.742 +INFO 2025-09-07 01:00:37 ts/train.py:392 step:1K smpl:717K ep:904 epch:15.06 loss:0.263 grdn:13.375 lr:1.0e-05 updt_s:0.872 data_s:0.809 +INFO 2025-09-07 01:06:58 ts/train.py:392 step:2K smpl:819K ep:1K epch:17.22 loss:0.218 grdn:13.069 lr:1.0e-05 updt_s:0.872 data_s:0.780 +INFO 2025-09-07 01:13:18 ts/train.py:392 step:2K smpl:922K ep:1K epch:19.37 loss:0.187 grdn:9.993 lr:1.0e-05 updt_s:0.872 data_s:0.771 +INFO 2025-09-07 01:19:41 ts/train.py:392 step:2K smpl:1M ep:1K epch:21.52 loss:0.167 grdn:9.161 lr:1.0e-05 updt_s:0.872 data_s:0.790 +INFO 2025-09-07 01:19:41 ts/train.py:401 Checkpoint policy after step 2000 +INFO 2025-09-07 01:26:03 ts/train.py:392 step:2K smpl:1M ep:1K epch:23.67 loss:0.152 grdn:8.440 lr:1.0e-05 updt_s:0.872 data_s:0.781 +INFO 2025-09-07 01:32:18 ts/train.py:392 step:2K smpl:1M ep:2K epch:25.82 loss:0.141 grdn:8.228 lr:1.0e-05 updt_s:0.872 data_s:0.747 +INFO 2025-09-07 01:38:40 ts/train.py:392 step:3K smpl:1M ep:2K epch:27.98 loss:0.133 grdn:7.579 lr:1.0e-05 updt_s:0.872 data_s:0.777 +INFO 2025-09-07 01:45:01 ts/train.py:392 step:3K smpl:1M ep:2K epch:30.13 loss:0.125 grdn:7.218 lr:1.0e-05 updt_s:0.872 data_s:0.777 +INFO 2025-09-07 01:51:22 ts/train.py:392 step:3K smpl:2M ep:2K epch:32.28 loss:0.119 grdn:6.565 lr:1.0e-05 updt_s:0.872 data_s:0.779 +INFO 2025-09-07 01:51:22 ts/train.py:401 Checkpoint policy after step 3000 +INFO 2025-09-07 01:57:43 ts/train.py:392 step:3K smpl:2M ep:2K epch:34.43 loss:0.114 grdn:5.693 lr:1.0e-05 updt_s:0.872 data_s:0.769 +INFO 2025-09-07 02:04:01 ts/train.py:392 step:3K smpl:2M ep:2K epch:36.59 loss:0.109 grdn:6.519 lr:1.0e-05 updt_s:0.872 data_s:0.760 +INFO 2025-09-07 02:10:20 ts/train.py:392 step:4K smpl:2M ep:2K epch:38.74 loss:0.105 grdn:5.599 lr:1.0e-05 updt_s:0.871 data_s:0.763 +INFO 2025-09-07 02:16:40 ts/train.py:392 step:4K smpl:2M ep:2K epch:40.89 loss:0.102 grdn:6.007 lr:1.0e-05 updt_s:0.872 data_s:0.776 +INFO 2025-09-07 02:23:08 ts/train.py:392 step:4K smpl:2M ep:3K epch:43.04 loss:0.098 grdn:5.610 lr:1.0e-05 updt_s:0.871 data_s:0.814 +INFO 2025-09-07 02:23:08 ts/train.py:401 Checkpoint policy after step 4000 +INFO 2025-09-07 02:29:23 ts/train.py:392 step:4K smpl:2M ep:3K epch:45.19 loss:0.095 grdn:5.365 lr:1.0e-05 updt_s:0.872 data_s:0.742 +INFO 2025-09-07 02:35:45 ts/train.py:392 step:4K smpl:2M ep:3K epch:47.35 loss:0.093 grdn:5.133 lr:1.0e-05 updt_s:0.872 data_s:0.781 +INFO 2025-09-07 02:42:07 ts/train.py:392 step:5K smpl:2M ep:3K epch:49.50 loss:0.090 grdn:5.767 lr:1.0e-05 updt_s:0.871 data_s:0.782 +INFO 2025-09-07 02:48:26 ts/train.py:392 step:5K smpl:2M ep:3K epch:51.65 loss:0.088 grdn:5.940 lr:1.0e-05 updt_s:0.871 data_s:0.766 +INFO 2025-09-07 02:54:46 ts/train.py:392 step:5K smpl:3M ep:3K epch:53.80 loss:0.086 grdn:4.788 lr:1.0e-05 updt_s:0.871 data_s:0.774 +INFO 2025-09-07 02:54:46 ts/train.py:401 Checkpoint policy after step 5000 +INFO 2025-09-07 03:01:04 ts/train.py:392 step:5K smpl:3M ep:3K epch:55.95 loss:0.084 grdn:4.971 lr:1.0e-05 updt_s:0.871 data_s:0.754 +INFO 2025-09-07 03:07:30 ts/train.py:392 step:5K smpl:3M ep:3K epch:58.11 loss:0.082 grdn:4.674 lr:1.0e-05 updt_s:0.871 data_s:0.803 +INFO 2025-09-07 03:13:56 ts/train.py:392 step:6K smpl:3M ep:4K epch:60.26 loss:0.080 grdn:4.616 lr:1.0e-05 updt_s:0.871 data_s:0.811 +INFO 2025-09-07 03:20:13 ts/train.py:392 step:6K smpl:3M ep:4K epch:62.41 loss:0.079 grdn:4.765 lr:1.0e-05 updt_s:0.871 data_s:0.755 +INFO 2025-09-07 03:26:34 ts/train.py:392 step:6K smpl:3M ep:4K epch:64.56 loss:0.077 grdn:5.062 lr:1.0e-05 updt_s:0.871 data_s:0.772 +INFO 2025-09-07 03:26:34 ts/train.py:401 Checkpoint policy after step 6000 +INFO 2025-09-07 03:32:56 ts/train.py:392 step:6K smpl:3M ep:4K epch:66.71 loss:0.075 grdn:4.582 lr:1.0e-05 updt_s:0.871 data_s:0.776 +INFO 2025-09-07 03:39:19 ts/train.py:392 step:6K smpl:3M ep:4K epch:68.87 loss:0.074 grdn:4.802 lr:1.0e-05 updt_s:0.871 data_s:0.794 +INFO 2025-09-07 03:45:36 ts/train.py:392 step:7K smpl:3M ep:4K epch:71.02 loss:0.073 grdn:4.404 lr:1.0e-05 updt_s:0.871 data_s:0.761 +INFO 2025-09-07 03:52:03 ts/train.py:392 step:7K smpl:3M ep:4K epch:73.17 loss:0.071 grdn:4.001 lr:1.0e-05 updt_s:0.870 data_s:0.807 +INFO 2025-09-07 03:58:25 ts/train.py:392 step:7K smpl:4M ep:5K epch:75.32 loss:0.070 grdn:4.808 lr:1.0e-05 updt_s:0.871 data_s:0.784 +INFO 2025-09-07 03:58:25 ts/train.py:401 Checkpoint policy after step 7000 +INFO 2025-09-07 04:04:42 ts/train.py:392 step:7K smpl:4M ep:5K epch:77.47 loss:0.069 grdn:4.324 lr:1.0e-05 updt_s:0.871 data_s:0.749 +Traceback (most recent call last): + File "/home/ubuntu/projects/policy/lerobot/scripts/train.py", line 448, in + train() + File "/home/ubuntu/projects/policy/lerobot/configs/parser.py", line 226, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/ubuntu/projects/policy/lerobot/scripts/train.py", line 362, in train + batch = next(dl_iter) + File "/home/ubuntu/projects/policy/lerobot/common/datasets/utils.py", line 641, in cycle + yield next(iterator) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 733, in __next__ + data = self._next_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1491, in _next_data + idx, data = self._get_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1443, in _get_data + success, data = self._try_get_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1284, in _try_get_data + data = self._data_queue.get(timeout=timeout) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/queue.py", line 180, in get + self.not_empty.wait(remaining) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/threading.py", line 324, in wait + gotit = waiter.acquire(True, timeout) +KeyboardInterrupt +Traceback (most recent call last): + File "/home/ubuntu/projects/policy/lerobot/scripts/train.py", line 448, in + train() + File "/home/ubuntu/projects/policy/lerobot/configs/parser.py", line 226, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/ubuntu/projects/policy/lerobot/scripts/train.py", line 362, in train + batch = next(dl_iter) + File "/home/ubuntu/projects/policy/lerobot/common/datasets/utils.py", line 641, in cycle + yield next(iterator) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 733, in __next__ + data = self._next_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1491, in _next_data + idx, data = self._get_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1443, in _get_data + success, data = self._try_get_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1284, in _try_get_data + data = self._data_queue.get(timeout=timeout) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/queue.py", line 180, in get + self.not_empty.wait(remaining) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/threading.py", line 324, in wait + gotit = waiter.acquire(True, timeout) +KeyboardInterrupt diff --git a/wandb/latest-run/files/requirements.txt b/wandb/latest-run/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b8400bab743cd976cfe0eee113a15b016584209a --- /dev/null +++ b/wandb/latest-run/files/requirements.txt @@ -0,0 +1,140 @@ +orderly-set==5.4.1 +evdev==1.9.2 +sympy==1.14.0 +mergedeep==1.3.4 +platformdirs==4.3.8 +numba==0.61.2 +sentry-sdk==2.32.0 +PySocks==1.7.1 +frozenlist==1.7.0 +nvidia-nccl-cu12==2.26.2 +nvidia-cuda-cupti-cu12==12.6.80 +python-xlib==0.33 +transformers==4.53.1 +protobuf==6.31.1 +certifi==2025.7.9 +pfzy==0.3.4 +torch==2.7.1 +huggingface-hub==0.33.2 +itsdangerous==2.2.0 +exceptiongroup==1.3.0 +beautifulsoup4==4.13.4 +PyYAML==6.0.2 +inquirerpy==0.3.4 +datasets==3.6.0 +filelock==3.18.0 +cffi==1.17.1 +multidict==6.6.3 +pluggy==1.6.0 +fasteners==0.19 +lerobot==0.1.0 +pydantic_core==2.33.2 +av==15.0.0 +aiohappyeyeballs==2.6.1 +Farama-Notifications==0.0.4 +pyarrow==20.0.0 +imageio-ffmpeg==0.6.0 +aiosignal==1.4.0 +pyserial==3.5 +num2words==0.5.14 +draccus==0.10.0 +numpy==2.2.6 +asciitree==0.3.3 +gdown==5.2.0 +pycparser==2.22 +six==1.17.0 +pydantic==2.11.7 +accelerate==1.8.1 +pip==25.1 +multiprocess==0.70.16 +numcodecs==0.13.1 +aiohttp==3.12.14 +nvidia-cublas-cu12==12.6.4.1 +torchcodec==0.4.0 +pandas==2.3.1 +tokenizers==0.21.2 +GitPython==3.1.44 +termcolor==3.1.0 +typing_extensions==4.14.1 +click==8.2.1 +deepdiff==8.5.0 +diffusers==0.34.0 +hf-xet==1.1.5 +typing-inspect==0.9.0 +wcwidth==0.2.13 +Werkzeug==3.1.3 +iniconfig==2.1.0 +Pygments==2.19.2 +antlr4-python3-runtime==4.9.3 +gymnasium==0.29.1 +blinker==1.9.0 +cloudpickle==3.1.1 +fsspec==2025.3.0 +yarl==1.20.1 +setuptools==78.1.1 +tzdata==2025.2 +safetensors==0.5.3 +nvidia-cusparse-cu12==12.5.4.2 +pyyaml-include==1.4.1 +soupsieve==2.7 +idna==3.10 +nvidia-cudnn-cu12==9.5.1.17 +pyzmq==27.0.0 +llvmlite==0.44.0 +jsonlines==4.0.0 +MarkupSafe==3.0.2 +pymunk==6.11.1 +pillow==11.3.0 +einops==0.8.1 +nvidia-cusolver-cu12==11.7.1.2 +nvidia-cuda-nvrtc-cu12==12.6.77 +torchvision==0.22.1 +wheel==0.45.1 +wandb==0.21.0 +zipp==3.23.0 +nvidia-cufile-cu12==1.11.1.6 +imageio==2.37.0 +opencv-python-headless==4.12.0.88 +toml==0.10.2 +hf_transfer==0.1.9 +nvidia-curand-cu12==10.3.7.77 +gitdb==4.0.12 +importlib_metadata==8.7.0 +nvidia-cufft-cu12==11.3.0.4 +zarr==2.18.3 +prompt_toolkit==3.0.51 +psutil==7.0.0 +smmap==5.0.2 +packaging==25.0 +xxhash==3.5.0 +attrs==25.3.0 +nvidia-nvjitlink-cu12==12.6.85 +pytest==8.4.1 +nvidia-cuda-runtime-cu12==12.6.77 +omegaconf==2.3.0 +annotated-types==0.7.0 +pytz==2025.2 +mypy_extensions==1.1.0 +networkx==3.4.2 +h5py==3.14.0 +Flask==3.1.1 +nvidia-cusparselt-cu12==0.6.3 +tomli==2.2.1 +nvidia-nvtx-cu12==12.6.77 +docopt==0.6.2 +urllib3==2.5.0 +async-timeout==5.0.1 +rerun-sdk==0.23.4 +cmake==4.0.3 +regex==2024.11.6 +python-dateutil==2.9.0.post0 +Jinja2==3.1.6 +pynput==1.8.1 +charset-normalizer==3.4.2 +requests==2.32.4 +typing-inspection==0.4.1 +triton==3.3.1 +tqdm==4.67.1 +propcache==0.3.2 +dill==0.3.8 +mpmath==1.3.0 diff --git a/wandb/latest-run/files/wandb-metadata.json b/wandb/latest-run/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..dae5f4d38c724fb6601e83e36ecebf8db1a4a4e0 --- /dev/null +++ b/wandb/latest-run/files/wandb-metadata.json @@ -0,0 +1,45 @@ +{ + "os": "Linux-6.8.0-1031-aws-x86_64-with-glibc2.39", + "python": "CPython 3.10.13", + "startedAt": "2025-09-07T00:16:03.329573Z", + "args": [ + "--config_path=src/training_cfg/pusht/act_abs.json", + "--dataset.repo_id=pusht_60_v5", + "--job_name=2025-09-07_00-15-58_act_abs_v5", + "--output_dir=outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5" + ], + "program": "/home/ubuntu/projects/policy/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "codePathLocal": "lerobot/scripts/train.py", + "git": { + "remote": "git@github.com:SceniX-AI/policy.git", + "commit": "e3e730f94ee641237d6e067d143a1794c6c9e5aa" + }, + "root": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "host": "ip-172-31-0-58", + "executable": "/home/ubuntu/miniconda3/envs/lerobot/bin/python", + "cpu_count": 4, + "cpu_count_logical": 8, + "gpu": "NVIDIA L40S", + "gpu_count": 1, + "disk": { + "/": { + "total": "1063956480000", + "used": "162176774144" + } + }, + "memory": { + "total": "66507816960" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA L40S", + "memoryTotal": "48305799168", + "cudaCores": 18176, + "architecture": "Ada", + "uuid": "GPU-aa974e28-ca07-9688-5ab1-68deb2b19654" + } + ], + "cudaVersion": "12.8", + "writerId": "fzxncgo02qx7l3nog6m82ys6oh1pwriz" +} \ No newline at end of file diff --git a/wandb/latest-run/files/wandb-summary.json b/wandb/latest-run/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..38d0d123ad82c472bb078b0bd2e0293d4bc0adce --- /dev/null +++ b/wandb/latest-run/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/samples":3686400,"train/update_s":0.8710163691250545,"_runtime":14062.581932241,"train/loss":0.06915990173816682,"train/dataloading_s":0.7490912805700282,"_timestamp":1.7572178821978567e+09,"train/l1_loss":0.060983188450336456,"_step":7200,"train/steps":7200,"train/epochs":77.47467529738137,"train/kld_loss":0.0006338062230497599,"train/grad_norm":4.323919188976288,"_wandb":{"runtime":14062},"train/episodes":4648.480517842882,"train/lr":1.0000000000000021e-05} \ No newline at end of file diff --git a/wandb/latest-run/logs/debug-core.log b/wandb/latest-run/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..fd2ae415af41d416de19c7b002c7da2c83590084 --- /dev/null +++ b/wandb/latest-run/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-09-07T00:16:03.350057123Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp3ckjnf_l/port-6396.txt","pid":6396,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-07T00:16:03.350480069Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":6396} +{"time":"2025-09-07T00:16:03.350465459Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-6396-6575-2544364374/socket","Net":"unix"}} +{"time":"2025-09-07T00:16:03.535885052Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-07T00:16:03.541491234Z","level":"INFO","msg":"handleInformInit: received","streamId":"7fd56jqi","id":"1(@)"} +{"time":"2025-09-07T00:16:03.717837844Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"7fd56jqi","id":"1(@)"} +{"time":"2025-09-07T04:10:26.473492126Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-07T04:10:26.473566899Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-07T04:10:26.473587189Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-07T04:10:26.473616721Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-07T04:10:26.473645411Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-6396-6575-2544364374/socket","Net":"unix"}} +{"time":"2025-09-07T04:10:26.873687645Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-07T04:10:26.873708085Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-07T04:10:26.873716246Z","level":"INFO","msg":"server is closed"} diff --git a/wandb/latest-run/logs/debug-internal.log b/wandb/latest-run/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ebc2d2a58835f0787067a9bb4095819f798697ce --- /dev/null +++ b/wandb/latest-run/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2025-09-07T00:16:03.541599108Z","level":"INFO","msg":"stream: starting","core version":"0.21.0"} +{"time":"2025-09-07T00:16:03.717802463Z","level":"INFO","msg":"stream: created new stream","id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717830474Z","level":"INFO","msg":"stream: started","id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717843784Z","level":"INFO","msg":"handler: started","stream_id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717862125Z","level":"INFO","msg":"sender: started","stream_id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717879775Z","level":"INFO","msg":"writer: Do: started","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.47360162Z","level":"INFO","msg":"stream: closing","id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.766374646Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-07T04:10:26.87005929Z","level":"INFO","msg":"handler: closed","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.870105072Z","level":"INFO","msg":"sender: closed","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.870094282Z","level":"INFO","msg":"writer: Close: closed","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.870214416Z","level":"INFO","msg":"stream: closed","id":"7fd56jqi"} diff --git a/wandb/latest-run/logs/debug.log b/wandb/latest-run/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b184fb9bfad3a85f5e7ee81fdd7f00e5059cd921 --- /dev/null +++ b/wandb/latest-run/logs/debug.log @@ -0,0 +1,22 @@ +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0 +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Configure stats pid to 6396 +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/projects/policy/wandb/settings +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5/wandb/run-20250907_001603-7fd56jqi/logs/debug.log +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5/wandb/run-20250907_001603-7fd56jqi/logs/debug-internal.log +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:init():830] calling init triggers +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'pusht_60_v5', 'root': None, 'episodes': None, 'image_transforms': {'enable': True, 'max_num_transforms': 3, 'random_order': True, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'perspective': {'weight': 1.0, 'type': 'RandomPerspective', 'kwargs': {'distortion_scale': 0.025, 'p': 1.0}}, 'rotation': {'weight': 1.0, 'type': 'RandomRotation', 'kwargs': {'degrees': [-5, 5]}}, 'crop': {'weight': 1.0, 'type': 'RandomCrop', 'kwargs': {'size': [200, 200]}}}, 'downsample': {'resize': {'weight': 1.0, 'type': 'Resize', 'kwargs': {'size': [240, 240]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'relative_actions': False, 'input_features': {'observation.image.side': {'type': , 'shape': [3, 240, 240]}, 'observation.image.wrist': {'type': , 'shape': [3, 240, 240]}, 'observation.state': {'type': , 'shape': [2]}}, 'output_features': {'action': {'type': , 'shape': [2]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 50, 'n_action_steps': 50, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': 0, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5', 'job_name': '2025-09-07_00-15-58_act_abs_v5', 'resume': False, 'seed': 100000, 'num_workers': 4, 'batch_size': 512, 'steps': 20000, 'eval_freq': 2000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 1000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:init():871] starting backend +2025-09-07 00:16:03,536 INFO MainThread:6396 [wandb_init.py:init():874] sending inform_init request +2025-09-07 00:16:03,539 INFO MainThread:6396 [wandb_init.py:init():882] backend started and connected +2025-09-07 00:16:03,541 INFO MainThread:6396 [wandb_init.py:init():953] updated telemetry +2025-09-07 00:16:03,544 INFO MainThread:6396 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-09-07 00:16:03,890 INFO MainThread:6396 [wandb_init.py:init():1029] starting run threads in backend +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_console_start():2458] atexit reg +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_redirect():2375] Wrapping output streams. +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_redirect():2398] Redirects installed. +2025-09-07 00:16:03,974 INFO MainThread:6396 [wandb_init.py:init():1075] run started, returning control to user process +2025-09-07 04:10:26,472 INFO MsgRouterThr:6396 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/wandb/latest-run/run-7fd56jqi.wandb b/wandb/latest-run/run-7fd56jqi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..91e18cca03611cd5725b1c5cc7d6ade0a57c70b8 --- /dev/null +++ b/wandb/latest-run/run-7fd56jqi.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c2f871c15fce12d0c8c0e38e749458c0ddb55bbc14aaf0d1c2a0f7e71257752 +size 881243 diff --git a/wandb/run-20250907_001603-7fd56jqi/files/config.yaml b/wandb/run-20250907_001603-7fd56jqi/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb75e6935bf53a3b951f26b8d75154d4747b848 --- /dev/null +++ b/wandb/run-20250907_001603-7fd56jqi/files/config.yaml @@ -0,0 +1,251 @@ +_wandb: + value: + cli_version: 0.21.0 + e: + fzxncgo02qx7l3nog6m82ys6oh1pwriz: + args: + - --config_path=src/training_cfg/pusht/act_abs.json + - --dataset.repo_id=pusht_60_v5 + - --job_name=2025-09-07_00-15-58_act_abs_v5 + - --output_dir=outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5 + codePath: lerobot/scripts/train.py + codePathLocal: lerobot/scripts/train.py + cpu_count: 4 + cpu_count_logical: 8 + cudaVersion: "12.8" + disk: + /: + total: "1063956480000" + used: "162176774144" + executable: /home/ubuntu/miniconda3/envs/lerobot/bin/python + git: + commit: e3e730f94ee641237d6e067d143a1794c6c9e5aa + remote: git@github.com:SceniX-AI/policy.git + gpu: NVIDIA L40S + gpu_count: 1 + gpu_nvidia: + - architecture: Ada + cudaCores: 18176 + memoryTotal: "48305799168" + name: NVIDIA L40S + uuid: GPU-aa974e28-ca07-9688-5ab1-68deb2b19654 + host: ip-172-31-0-58 + memory: + total: "66507816960" + os: Linux-6.8.0-1031-aws-x86_64-with-glibc2.39 + program: /home/ubuntu/projects/policy/lerobot/scripts/train.py + python: CPython 3.10.13 + root: outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5 + startedAt: "2025-09-07T00:16:03.329573Z" + writerId: fzxncgo02qx7l3nog6m82ys6oh1pwriz + m: [] + python_version: 3.10.13 + t: + "1": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + "2": + - 1 + - 11 + - 41 + - 49 + - 51 + - 71 + "3": + - 13 + - 15 + - 16 + - 61 + "4": 3.10.13 + "5": 0.21.0 + "6": 4.53.1 + "10": + - 21 + "12": 0.21.0 + "13": linux-x86_64 +batch_size: + value: 512 +dataset: + value: + episodes: null + image_transforms: + downsample: + resize: + kwargs: + size: + - 240 + - 240 + type: Resize + weight: 1 + enable: true + max_num_transforms: 3 + random_order: true + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + crop: + kwargs: + size: + - 200 + - 200 + type: RandomCrop + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + perspective: + kwargs: + distortion_scale: 0.025 + p: 1 + type: RandomPerspective + weight: 1 + rotation: + kwargs: + degrees: + - -5 + - 5 + type: RandomRotation + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: pusht_60_v5 + revision: null + root: null + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 2000 +job_name: + value: 2025-09-07_00-15-58_act_abs_v5 +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.9 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 1e-05 + type: adamw + weight_decay: 0.0001 +output_dir: + value: outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5 +policy: + value: + chunk_size: 50 + device: cuda + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + input_features: + observation.image.side: + shape: + - 3 + - 240 + - 240 + type: VISUAL + observation.image.wrist: + shape: + - 3 + - 240 + - 240 + type: VISUAL + observation.state: + shape: + - 2 + type: STATE + kl_weight: 10 + latent_dim: 32 + n_action_steps: 50 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + normalization_mapping: + ACTION: MEAN_STD + STATE: MEAN_STD + VISUAL: MEAN_STD + optimizer_lr: 1e-05 + optimizer_lr_backbone: 1e-05 + optimizer_weight_decay: 0.0001 + output_features: + action: + shape: + - 2 + type: ACTION + pre_norm: false + pretrained_backbone_weights: ResNet18_Weights.IMAGENET1K_V1 + relative_actions: false + replace_final_stride_with_dilation: 0 + temporal_ensemble_coeff: null + type: act + use_amp: false + use_vae: true + vision_backbone: resnet18 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 1000 +scheduler: + value: null +seed: + value: 100000 +steps: + value: 20000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: true + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/wandb/run-20250907_001603-7fd56jqi/files/output.log b/wandb/run-20250907_001603-7fd56jqi/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7f7858f8db52fc34943d826258a53d5973a3e7b8 --- /dev/null +++ b/wandb/run-20250907_001603-7fd56jqi/files/output.log @@ -0,0 +1,101 @@ +Logs will be synced with wandb. +INFO 2025-09-07 00:16:03 ndb_utils.py:96 Track this run --> https://wandb.ai/ss7050-columbia/lerobot/runs/7fd56jqi +INFO 2025-09-07 00:16:03 ts/train.py:260 Creating dataset +Resolving data files: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 60/60 [00:00<00:00, 308783.12it/s] +INFO 2025-09-07 00:16:04 ts/train.py:272 Creating policy +INFO 2025-09-07 00:16:04 ts/train.py:275 Using relative actions: False +INFO 2025-09-07 00:16:05 ts/train.py:301 Creating optimizer and scheduler +INFO 2025-09-07 00:16:05 ts/train.py:313 Output dir: outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5 +INFO 2025-09-07 00:16:05 ts/train.py:316 cfg.steps=20000 (20K) +INFO 2025-09-07 00:16:05 ts/train.py:317 dataset.num_frames=47582 (48K) +INFO 2025-09-07 00:16:05 ts/train.py:318 dataset.num_episodes=60 +INFO 2025-09-07 00:16:05 ts/train.py:319 num_learnable_params=51563394 (52M) +INFO 2025-09-07 00:16:05 ts/train.py:320 num_total_params=51563418 (52M) +INFO 2025-09-07 00:16:05 ts/train.py:359 Start offline training on a fixed dataset +INFO 2025-09-07 00:22:39 ts/train.py:392 step:200 smpl:102K ep:129 epch:2.15 loss:5.003 grdn:75.835 lr:1.0e-05 updt_s:0.900 data_s:0.813 +INFO 2025-09-07 00:28:54 ts/train.py:392 step:400 smpl:205K ep:258 epch:4.30 loss:1.574 grdn:26.507 lr:1.0e-05 updt_s:0.872 data_s:0.747 +INFO 2025-09-07 00:35:18 ts/train.py:392 step:600 smpl:307K ep:387 epch:6.46 loss:1.032 grdn:23.980 lr:1.0e-05 updt_s:0.872 data_s:0.791 +INFO 2025-09-07 00:41:37 ts/train.py:392 step:800 smpl:410K ep:516 epch:8.61 loss:0.701 grdn:19.744 lr:1.0e-05 updt_s:0.872 data_s:0.766 +INFO 2025-09-07 00:47:54 ts/train.py:392 step:1K smpl:512K ep:646 epch:10.76 loss:0.482 grdn:17.509 lr:1.0e-05 updt_s:0.872 data_s:0.754 +INFO 2025-09-07 00:47:54 ts/train.py:401 Checkpoint policy after step 1000 +INFO 2025-09-07 00:54:10 ts/train.py:392 step:1K smpl:614K ep:775 epch:12.91 loss:0.343 grdn:14.252 lr:1.0e-05 updt_s:0.872 data_s:0.742 +INFO 2025-09-07 01:00:37 ts/train.py:392 step:1K smpl:717K ep:904 epch:15.06 loss:0.263 grdn:13.375 lr:1.0e-05 updt_s:0.872 data_s:0.809 +INFO 2025-09-07 01:06:58 ts/train.py:392 step:2K smpl:819K ep:1K epch:17.22 loss:0.218 grdn:13.069 lr:1.0e-05 updt_s:0.872 data_s:0.780 +INFO 2025-09-07 01:13:18 ts/train.py:392 step:2K smpl:922K ep:1K epch:19.37 loss:0.187 grdn:9.993 lr:1.0e-05 updt_s:0.872 data_s:0.771 +INFO 2025-09-07 01:19:41 ts/train.py:392 step:2K smpl:1M ep:1K epch:21.52 loss:0.167 grdn:9.161 lr:1.0e-05 updt_s:0.872 data_s:0.790 +INFO 2025-09-07 01:19:41 ts/train.py:401 Checkpoint policy after step 2000 +INFO 2025-09-07 01:26:03 ts/train.py:392 step:2K smpl:1M ep:1K epch:23.67 loss:0.152 grdn:8.440 lr:1.0e-05 updt_s:0.872 data_s:0.781 +INFO 2025-09-07 01:32:18 ts/train.py:392 step:2K smpl:1M ep:2K epch:25.82 loss:0.141 grdn:8.228 lr:1.0e-05 updt_s:0.872 data_s:0.747 +INFO 2025-09-07 01:38:40 ts/train.py:392 step:3K smpl:1M ep:2K epch:27.98 loss:0.133 grdn:7.579 lr:1.0e-05 updt_s:0.872 data_s:0.777 +INFO 2025-09-07 01:45:01 ts/train.py:392 step:3K smpl:1M ep:2K epch:30.13 loss:0.125 grdn:7.218 lr:1.0e-05 updt_s:0.872 data_s:0.777 +INFO 2025-09-07 01:51:22 ts/train.py:392 step:3K smpl:2M ep:2K epch:32.28 loss:0.119 grdn:6.565 lr:1.0e-05 updt_s:0.872 data_s:0.779 +INFO 2025-09-07 01:51:22 ts/train.py:401 Checkpoint policy after step 3000 +INFO 2025-09-07 01:57:43 ts/train.py:392 step:3K smpl:2M ep:2K epch:34.43 loss:0.114 grdn:5.693 lr:1.0e-05 updt_s:0.872 data_s:0.769 +INFO 2025-09-07 02:04:01 ts/train.py:392 step:3K smpl:2M ep:2K epch:36.59 loss:0.109 grdn:6.519 lr:1.0e-05 updt_s:0.872 data_s:0.760 +INFO 2025-09-07 02:10:20 ts/train.py:392 step:4K smpl:2M ep:2K epch:38.74 loss:0.105 grdn:5.599 lr:1.0e-05 updt_s:0.871 data_s:0.763 +INFO 2025-09-07 02:16:40 ts/train.py:392 step:4K smpl:2M ep:2K epch:40.89 loss:0.102 grdn:6.007 lr:1.0e-05 updt_s:0.872 data_s:0.776 +INFO 2025-09-07 02:23:08 ts/train.py:392 step:4K smpl:2M ep:3K epch:43.04 loss:0.098 grdn:5.610 lr:1.0e-05 updt_s:0.871 data_s:0.814 +INFO 2025-09-07 02:23:08 ts/train.py:401 Checkpoint policy after step 4000 +INFO 2025-09-07 02:29:23 ts/train.py:392 step:4K smpl:2M ep:3K epch:45.19 loss:0.095 grdn:5.365 lr:1.0e-05 updt_s:0.872 data_s:0.742 +INFO 2025-09-07 02:35:45 ts/train.py:392 step:4K smpl:2M ep:3K epch:47.35 loss:0.093 grdn:5.133 lr:1.0e-05 updt_s:0.872 data_s:0.781 +INFO 2025-09-07 02:42:07 ts/train.py:392 step:5K smpl:2M ep:3K epch:49.50 loss:0.090 grdn:5.767 lr:1.0e-05 updt_s:0.871 data_s:0.782 +INFO 2025-09-07 02:48:26 ts/train.py:392 step:5K smpl:2M ep:3K epch:51.65 loss:0.088 grdn:5.940 lr:1.0e-05 updt_s:0.871 data_s:0.766 +INFO 2025-09-07 02:54:46 ts/train.py:392 step:5K smpl:3M ep:3K epch:53.80 loss:0.086 grdn:4.788 lr:1.0e-05 updt_s:0.871 data_s:0.774 +INFO 2025-09-07 02:54:46 ts/train.py:401 Checkpoint policy after step 5000 +INFO 2025-09-07 03:01:04 ts/train.py:392 step:5K smpl:3M ep:3K epch:55.95 loss:0.084 grdn:4.971 lr:1.0e-05 updt_s:0.871 data_s:0.754 +INFO 2025-09-07 03:07:30 ts/train.py:392 step:5K smpl:3M ep:3K epch:58.11 loss:0.082 grdn:4.674 lr:1.0e-05 updt_s:0.871 data_s:0.803 +INFO 2025-09-07 03:13:56 ts/train.py:392 step:6K smpl:3M ep:4K epch:60.26 loss:0.080 grdn:4.616 lr:1.0e-05 updt_s:0.871 data_s:0.811 +INFO 2025-09-07 03:20:13 ts/train.py:392 step:6K smpl:3M ep:4K epch:62.41 loss:0.079 grdn:4.765 lr:1.0e-05 updt_s:0.871 data_s:0.755 +INFO 2025-09-07 03:26:34 ts/train.py:392 step:6K smpl:3M ep:4K epch:64.56 loss:0.077 grdn:5.062 lr:1.0e-05 updt_s:0.871 data_s:0.772 +INFO 2025-09-07 03:26:34 ts/train.py:401 Checkpoint policy after step 6000 +INFO 2025-09-07 03:32:56 ts/train.py:392 step:6K smpl:3M ep:4K epch:66.71 loss:0.075 grdn:4.582 lr:1.0e-05 updt_s:0.871 data_s:0.776 +INFO 2025-09-07 03:39:19 ts/train.py:392 step:6K smpl:3M ep:4K epch:68.87 loss:0.074 grdn:4.802 lr:1.0e-05 updt_s:0.871 data_s:0.794 +INFO 2025-09-07 03:45:36 ts/train.py:392 step:7K smpl:3M ep:4K epch:71.02 loss:0.073 grdn:4.404 lr:1.0e-05 updt_s:0.871 data_s:0.761 +INFO 2025-09-07 03:52:03 ts/train.py:392 step:7K smpl:3M ep:4K epch:73.17 loss:0.071 grdn:4.001 lr:1.0e-05 updt_s:0.870 data_s:0.807 +INFO 2025-09-07 03:58:25 ts/train.py:392 step:7K smpl:4M ep:5K epch:75.32 loss:0.070 grdn:4.808 lr:1.0e-05 updt_s:0.871 data_s:0.784 +INFO 2025-09-07 03:58:25 ts/train.py:401 Checkpoint policy after step 7000 +INFO 2025-09-07 04:04:42 ts/train.py:392 step:7K smpl:4M ep:5K epch:77.47 loss:0.069 grdn:4.324 lr:1.0e-05 updt_s:0.871 data_s:0.749 +Traceback (most recent call last): + File "/home/ubuntu/projects/policy/lerobot/scripts/train.py", line 448, in + train() + File "/home/ubuntu/projects/policy/lerobot/configs/parser.py", line 226, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/ubuntu/projects/policy/lerobot/scripts/train.py", line 362, in train + batch = next(dl_iter) + File "/home/ubuntu/projects/policy/lerobot/common/datasets/utils.py", line 641, in cycle + yield next(iterator) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 733, in __next__ + data = self._next_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1491, in _next_data + idx, data = self._get_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1443, in _get_data + success, data = self._try_get_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1284, in _try_get_data + data = self._data_queue.get(timeout=timeout) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/queue.py", line 180, in get + self.not_empty.wait(remaining) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/threading.py", line 324, in wait + gotit = waiter.acquire(True, timeout) +KeyboardInterrupt +Traceback (most recent call last): + File "/home/ubuntu/projects/policy/lerobot/scripts/train.py", line 448, in + train() + File "/home/ubuntu/projects/policy/lerobot/configs/parser.py", line 226, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/home/ubuntu/projects/policy/lerobot/scripts/train.py", line 362, in train + batch = next(dl_iter) + File "/home/ubuntu/projects/policy/lerobot/common/datasets/utils.py", line 641, in cycle + yield next(iterator) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 733, in __next__ + data = self._next_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1491, in _next_data + idx, data = self._get_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1443, in _get_data + success, data = self._try_get_data() + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1284, in _try_get_data + data = self._data_queue.get(timeout=timeout) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/queue.py", line 180, in get + self.not_empty.wait(remaining) + File "/home/ubuntu/miniconda3/envs/lerobot/lib/python3.10/threading.py", line 324, in wait + gotit = waiter.acquire(True, timeout) +KeyboardInterrupt diff --git a/wandb/run-20250907_001603-7fd56jqi/files/requirements.txt b/wandb/run-20250907_001603-7fd56jqi/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b8400bab743cd976cfe0eee113a15b016584209a --- /dev/null +++ b/wandb/run-20250907_001603-7fd56jqi/files/requirements.txt @@ -0,0 +1,140 @@ +orderly-set==5.4.1 +evdev==1.9.2 +sympy==1.14.0 +mergedeep==1.3.4 +platformdirs==4.3.8 +numba==0.61.2 +sentry-sdk==2.32.0 +PySocks==1.7.1 +frozenlist==1.7.0 +nvidia-nccl-cu12==2.26.2 +nvidia-cuda-cupti-cu12==12.6.80 +python-xlib==0.33 +transformers==4.53.1 +protobuf==6.31.1 +certifi==2025.7.9 +pfzy==0.3.4 +torch==2.7.1 +huggingface-hub==0.33.2 +itsdangerous==2.2.0 +exceptiongroup==1.3.0 +beautifulsoup4==4.13.4 +PyYAML==6.0.2 +inquirerpy==0.3.4 +datasets==3.6.0 +filelock==3.18.0 +cffi==1.17.1 +multidict==6.6.3 +pluggy==1.6.0 +fasteners==0.19 +lerobot==0.1.0 +pydantic_core==2.33.2 +av==15.0.0 +aiohappyeyeballs==2.6.1 +Farama-Notifications==0.0.4 +pyarrow==20.0.0 +imageio-ffmpeg==0.6.0 +aiosignal==1.4.0 +pyserial==3.5 +num2words==0.5.14 +draccus==0.10.0 +numpy==2.2.6 +asciitree==0.3.3 +gdown==5.2.0 +pycparser==2.22 +six==1.17.0 +pydantic==2.11.7 +accelerate==1.8.1 +pip==25.1 +multiprocess==0.70.16 +numcodecs==0.13.1 +aiohttp==3.12.14 +nvidia-cublas-cu12==12.6.4.1 +torchcodec==0.4.0 +pandas==2.3.1 +tokenizers==0.21.2 +GitPython==3.1.44 +termcolor==3.1.0 +typing_extensions==4.14.1 +click==8.2.1 +deepdiff==8.5.0 +diffusers==0.34.0 +hf-xet==1.1.5 +typing-inspect==0.9.0 +wcwidth==0.2.13 +Werkzeug==3.1.3 +iniconfig==2.1.0 +Pygments==2.19.2 +antlr4-python3-runtime==4.9.3 +gymnasium==0.29.1 +blinker==1.9.0 +cloudpickle==3.1.1 +fsspec==2025.3.0 +yarl==1.20.1 +setuptools==78.1.1 +tzdata==2025.2 +safetensors==0.5.3 +nvidia-cusparse-cu12==12.5.4.2 +pyyaml-include==1.4.1 +soupsieve==2.7 +idna==3.10 +nvidia-cudnn-cu12==9.5.1.17 +pyzmq==27.0.0 +llvmlite==0.44.0 +jsonlines==4.0.0 +MarkupSafe==3.0.2 +pymunk==6.11.1 +pillow==11.3.0 +einops==0.8.1 +nvidia-cusolver-cu12==11.7.1.2 +nvidia-cuda-nvrtc-cu12==12.6.77 +torchvision==0.22.1 +wheel==0.45.1 +wandb==0.21.0 +zipp==3.23.0 +nvidia-cufile-cu12==1.11.1.6 +imageio==2.37.0 +opencv-python-headless==4.12.0.88 +toml==0.10.2 +hf_transfer==0.1.9 +nvidia-curand-cu12==10.3.7.77 +gitdb==4.0.12 +importlib_metadata==8.7.0 +nvidia-cufft-cu12==11.3.0.4 +zarr==2.18.3 +prompt_toolkit==3.0.51 +psutil==7.0.0 +smmap==5.0.2 +packaging==25.0 +xxhash==3.5.0 +attrs==25.3.0 +nvidia-nvjitlink-cu12==12.6.85 +pytest==8.4.1 +nvidia-cuda-runtime-cu12==12.6.77 +omegaconf==2.3.0 +annotated-types==0.7.0 +pytz==2025.2 +mypy_extensions==1.1.0 +networkx==3.4.2 +h5py==3.14.0 +Flask==3.1.1 +nvidia-cusparselt-cu12==0.6.3 +tomli==2.2.1 +nvidia-nvtx-cu12==12.6.77 +docopt==0.6.2 +urllib3==2.5.0 +async-timeout==5.0.1 +rerun-sdk==0.23.4 +cmake==4.0.3 +regex==2024.11.6 +python-dateutil==2.9.0.post0 +Jinja2==3.1.6 +pynput==1.8.1 +charset-normalizer==3.4.2 +requests==2.32.4 +typing-inspection==0.4.1 +triton==3.3.1 +tqdm==4.67.1 +propcache==0.3.2 +dill==0.3.8 +mpmath==1.3.0 diff --git a/wandb/run-20250907_001603-7fd56jqi/files/wandb-metadata.json b/wandb/run-20250907_001603-7fd56jqi/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..dae5f4d38c724fb6601e83e36ecebf8db1a4a4e0 --- /dev/null +++ b/wandb/run-20250907_001603-7fd56jqi/files/wandb-metadata.json @@ -0,0 +1,45 @@ +{ + "os": "Linux-6.8.0-1031-aws-x86_64-with-glibc2.39", + "python": "CPython 3.10.13", + "startedAt": "2025-09-07T00:16:03.329573Z", + "args": [ + "--config_path=src/training_cfg/pusht/act_abs.json", + "--dataset.repo_id=pusht_60_v5", + "--job_name=2025-09-07_00-15-58_act_abs_v5", + "--output_dir=outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5" + ], + "program": "/home/ubuntu/projects/policy/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "codePathLocal": "lerobot/scripts/train.py", + "git": { + "remote": "git@github.com:SceniX-AI/policy.git", + "commit": "e3e730f94ee641237d6e067d143a1794c6c9e5aa" + }, + "root": "outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5", + "host": "ip-172-31-0-58", + "executable": "/home/ubuntu/miniconda3/envs/lerobot/bin/python", + "cpu_count": 4, + "cpu_count_logical": 8, + "gpu": "NVIDIA L40S", + "gpu_count": 1, + "disk": { + "/": { + "total": "1063956480000", + "used": "162176774144" + } + }, + "memory": { + "total": "66507816960" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA L40S", + "memoryTotal": "48305799168", + "cudaCores": 18176, + "architecture": "Ada", + "uuid": "GPU-aa974e28-ca07-9688-5ab1-68deb2b19654" + } + ], + "cudaVersion": "12.8", + "writerId": "fzxncgo02qx7l3nog6m82ys6oh1pwriz" +} \ No newline at end of file diff --git a/wandb/run-20250907_001603-7fd56jqi/files/wandb-summary.json b/wandb/run-20250907_001603-7fd56jqi/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..38d0d123ad82c472bb078b0bd2e0293d4bc0adce --- /dev/null +++ b/wandb/run-20250907_001603-7fd56jqi/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/samples":3686400,"train/update_s":0.8710163691250545,"_runtime":14062.581932241,"train/loss":0.06915990173816682,"train/dataloading_s":0.7490912805700282,"_timestamp":1.7572178821978567e+09,"train/l1_loss":0.060983188450336456,"_step":7200,"train/steps":7200,"train/epochs":77.47467529738137,"train/kld_loss":0.0006338062230497599,"train/grad_norm":4.323919188976288,"_wandb":{"runtime":14062},"train/episodes":4648.480517842882,"train/lr":1.0000000000000021e-05} \ No newline at end of file diff --git a/wandb/run-20250907_001603-7fd56jqi/logs/debug-core.log b/wandb/run-20250907_001603-7fd56jqi/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..fd2ae415af41d416de19c7b002c7da2c83590084 --- /dev/null +++ b/wandb/run-20250907_001603-7fd56jqi/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-09-07T00:16:03.350057123Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp3ckjnf_l/port-6396.txt","pid":6396,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-07T00:16:03.350480069Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":6396} +{"time":"2025-09-07T00:16:03.350465459Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-6396-6575-2544364374/socket","Net":"unix"}} +{"time":"2025-09-07T00:16:03.535885052Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-07T00:16:03.541491234Z","level":"INFO","msg":"handleInformInit: received","streamId":"7fd56jqi","id":"1(@)"} +{"time":"2025-09-07T00:16:03.717837844Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"7fd56jqi","id":"1(@)"} +{"time":"2025-09-07T04:10:26.473492126Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-07T04:10:26.473566899Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-07T04:10:26.473587189Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-07T04:10:26.473616721Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-07T04:10:26.473645411Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-6396-6575-2544364374/socket","Net":"unix"}} +{"time":"2025-09-07T04:10:26.873687645Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-07T04:10:26.873708085Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-07T04:10:26.873716246Z","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20250907_001603-7fd56jqi/logs/debug-internal.log b/wandb/run-20250907_001603-7fd56jqi/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ebc2d2a58835f0787067a9bb4095819f798697ce --- /dev/null +++ b/wandb/run-20250907_001603-7fd56jqi/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2025-09-07T00:16:03.541599108Z","level":"INFO","msg":"stream: starting","core version":"0.21.0"} +{"time":"2025-09-07T00:16:03.717802463Z","level":"INFO","msg":"stream: created new stream","id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717830474Z","level":"INFO","msg":"stream: started","id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717843784Z","level":"INFO","msg":"handler: started","stream_id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717862125Z","level":"INFO","msg":"sender: started","stream_id":"7fd56jqi"} +{"time":"2025-09-07T00:16:03.717879775Z","level":"INFO","msg":"writer: Do: started","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.47360162Z","level":"INFO","msg":"stream: closing","id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.766374646Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-07T04:10:26.87005929Z","level":"INFO","msg":"handler: closed","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.870105072Z","level":"INFO","msg":"sender: closed","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.870094282Z","level":"INFO","msg":"writer: Close: closed","stream_id":"7fd56jqi"} +{"time":"2025-09-07T04:10:26.870214416Z","level":"INFO","msg":"stream: closed","id":"7fd56jqi"} diff --git a/wandb/run-20250907_001603-7fd56jqi/logs/debug.log b/wandb/run-20250907_001603-7fd56jqi/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b184fb9bfad3a85f5e7ee81fdd7f00e5059cd921 --- /dev/null +++ b/wandb/run-20250907_001603-7fd56jqi/logs/debug.log @@ -0,0 +1,22 @@ +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Current SDK version is 0.21.0 +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Configure stats pid to 6396 +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/.config/wandb/settings +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Loading settings from /home/ubuntu/projects/policy/wandb/settings +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_setup.py:_flush():80] Loading settings from environment variables +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:setup_run_log_directory():703] Logging user logs to outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5/wandb/run-20250907_001603-7fd56jqi/logs/debug.log +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5/wandb/run-20250907_001603-7fd56jqi/logs/debug-internal.log +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:init():830] calling init triggers +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:init():835] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'pusht_60_v5', 'root': None, 'episodes': None, 'image_transforms': {'enable': True, 'max_num_transforms': 3, 'random_order': True, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}, 'perspective': {'weight': 1.0, 'type': 'RandomPerspective', 'kwargs': {'distortion_scale': 0.025, 'p': 1.0}}, 'rotation': {'weight': 1.0, 'type': 'RandomRotation', 'kwargs': {'degrees': [-5, 5]}}, 'crop': {'weight': 1.0, 'type': 'RandomCrop', 'kwargs': {'size': [200, 200]}}}, 'downsample': {'resize': {'weight': 1.0, 'type': 'Resize', 'kwargs': {'size': [240, 240]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'relative_actions': False, 'input_features': {'observation.image.side': {'type': , 'shape': [3, 240, 240]}, 'observation.image.wrist': {'type': , 'shape': [3, 240, 240]}, 'observation.state': {'type': , 'shape': [2]}}, 'output_features': {'action': {'type': , 'shape': [2]}}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 50, 'n_action_steps': 50, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': 0, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/pusht_60_v5/2025-09-07_00-15-58_act_abs_v5', 'job_name': '2025-09-07_00-15-58_act_abs_v5', 'resume': False, 'seed': 100000, 'num_workers': 4, 'batch_size': 512, 'steps': 20000, 'eval_freq': 2000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 1000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': True, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-09-07 00:16:03,330 INFO MainThread:6396 [wandb_init.py:init():871] starting backend +2025-09-07 00:16:03,536 INFO MainThread:6396 [wandb_init.py:init():874] sending inform_init request +2025-09-07 00:16:03,539 INFO MainThread:6396 [wandb_init.py:init():882] backend started and connected +2025-09-07 00:16:03,541 INFO MainThread:6396 [wandb_init.py:init():953] updated telemetry +2025-09-07 00:16:03,544 INFO MainThread:6396 [wandb_init.py:init():977] communicating run to backend with 90.0 second timeout +2025-09-07 00:16:03,890 INFO MainThread:6396 [wandb_init.py:init():1029] starting run threads in backend +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_console_start():2458] atexit reg +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_redirect():2375] Wrapping output streams. +2025-09-07 00:16:03,972 INFO MainThread:6396 [wandb_run.py:_redirect():2398] Redirects installed. +2025-09-07 00:16:03,974 INFO MainThread:6396 [wandb_init.py:init():1075] run started, returning control to user process +2025-09-07 04:10:26,472 INFO MsgRouterThr:6396 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/wandb/run-20250907_001603-7fd56jqi/run-7fd56jqi.wandb b/wandb/run-20250907_001603-7fd56jqi/run-7fd56jqi.wandb new file mode 100644 index 0000000000000000000000000000000000000000..91e18cca03611cd5725b1c5cc7d6ade0a57c70b8 --- /dev/null +++ b/wandb/run-20250907_001603-7fd56jqi/run-7fd56jqi.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c2f871c15fce12d0c8c0e38e749458c0ddb55bbc14aaf0d1c2a0f7e71257752 +size 881243