diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..08696a93603ee591dfa9a9537a7b57ee8b8df3e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a2a87afcfb29ad2a1bca085769cf2f6c54e99200ae6879e9e1610b5ff90e1fa +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..85d863715dc241fd41b24c5316e09855e12972da --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.0631933206103992e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d69e5cb2e8ea6e5130ec7c364307cab4ad55ae1f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07bf50ee8deed24c55cd63800e73370262b36cd8873739baa9c46faae7ac79df +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..189f42c0b6ce44b88cc14393cd14f2b9eccc337e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ba23041b6cc06e4967906e6f4b29b0f0a6a8fc9e52859fcb6bc6420115735cb +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a7d20ad7dd005e2ca9bb3b3b833686c191654091 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 630000, + "verbose": false, + "_step_count": 630001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.0631933206103992e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..71be4f013a155caed9df1f4d0513d9218f407ac3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 630000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2fb8a6ba50a1d6347b15a0f1c2c32a44f30a8014 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeb78780689b6359d0c6bf7b8fe9e205eb220527c6453775d736e22d83b8ccf4 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..5e685a39dc53152fbe671658b7b94efc36625ab5 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 1.6560776687438e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7888b46d54c28778f7b926797f318652d52b6765 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89b5584eb2a4f1e077d7794b4a6353760eb9c07ce8860ff8842a9b21179e1e03 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..391fae3fc71281908f66980a5d95c3ed01ce824b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f4a153e1f2508caa235c5521905027545b38d0860fbfaffa49ff466f7749d26 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b278d58035ccdcfa8cd70128a420b21e10878aa9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 660000, + "verbose": false, + "_step_count": 660001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.6560776687438e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..1946a814a39401d13bc968ec845f3081adf9bc13 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 660000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6fa2ba3c286aca66e6ff933c80c5d9bf60da7969 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4095bbe866da11104c1e0605a5ea0c4ecbac13a0e5df9a10f7cea68730da7242 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..4001dc6024934e2ec0029ed10ab9c1a4358ee373 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 1.2856394338260691e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e9a7e521a3e22d4b3ab96b4707244cc4f17f7159 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:558cf536590cc486110782a0cf506117c16091308c848df64f80d5f5c1aef273 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9aa56bd2292584c0b8a730e0c792b3de2a950d59 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:553e00de9ff2e041b8b90bcab643b61965d3e788d662e67a1e4ef31a3aafb7f7 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bfe642da76c15eb5d08bf30c1a297ef1f5d4799c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 690000, + "verbose": false, + "_step_count": 690001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.2856394338260691e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8b5daa23c20b59c1eb7b88487eaeb111856e1464 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/690000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 690000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..23896cc1db78bc01066514b15c1334e92135b15f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59cb50b6186932fb6554032a8c5bdb7c1e71be6162c8c8a74d82dcd90079e136 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..6926324a229b5f3a741c4acd6c9f9055873ff93e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.55941723863299e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7e1cb2271028fcb557ab02602b5a9e1100fe6b8c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1af74fa541b7978de7a9fd16e8bb8c94a088d917c6e5e13316c92af2d867b1a5 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e87891dd333a2bd8035068a2d95533c6b80917b7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e554b4b3155a2bedbb9d7ec870733b3a9f28de316f71f74b33a6cbec6133890 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..57f7f32291879d54dd6884b139a3353e9208cf67 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 720000, + "verbose": false, + "_step_count": 720001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.55941723863299e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..721e07e355e18f869a5dd8aa6f83c651ec7c3772 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/720000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 720000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..810cd0c7b4f042411ec98bbd96dd4719dae32752 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc4a2db295f656d981e74cf95b980971b39962a9e83c5f94a980f9758482d89a +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..910528e41cdfbb139e48f5656e7a9c98cbc08d9e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.706007892278027e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bbb4c958185575582e204d49c907632b45fc70e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d00729d94f5aa7bc4379fedf228c94cfc6d9921ebca2dc941073fd68c443a161 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3a9fa2759d582cf91e35f3d18b818b16a6f396b1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b617848aba881da02a8a8ab0bbb22b567feef45165eedd21a973d93686c96ff +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..531a163e77edd5cec217b3d8dfdf81d78aac5b48 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 750000, + "verbose": false, + "_step_count": 750001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.706007892278027e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..362e20eb49f9138dd347449af4f6da372b3c74ac --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/750000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 750000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b284749fa831f22374ec0e95d8b00357c4d1a2bf --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28aadbd36749b757293970543dadf8d73173cff9e6a46a68500d1e1b10a223ce +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9c890340d82de989b405601d8eb6f70d68373 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 4.327463582432278e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..da466c82820a73a39fd3962af8edf9715671c9a4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c0a2bcf6552bc34f8cef0773c6a2c9cb405ffb94320dfb54760caa0fc586c1 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c9444168701eefce4efc8c9c71902b464d57ac7d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55b853f170fd27addf7df90ffe016e1ae81c121e3e4e5c795f43705999b336a2 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1fed77bf3648d162e332234925925207b7372ef2 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 780000, + "verbose": false, + "_step_count": 780001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 4.327463582432278e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..e908e4d92cd192332ca7f02b2314283190a92940 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/780000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 780000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..466592ca4604eacc252409a016977bb43542b695 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1805df631a10be79ceb85b14e35c1fb18df0890d9ae0798c7b3250b8bca4e96 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..24fd12b096a8b3cc8374a56fa3f1444ddd1e79a7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.449873091292648e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9bfd96332b8efa48af75865ec3179323caba6184 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9610a7bb97c137359bebfadea84eabfc20a4a0084345505bfabb94af8a7bc66 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..254af3637ff96390f6cd85f8af51b7a19d59448b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd509ba3fa364c99bae1713e1123aaaf643dc474eefef46249f12bd1da854cee +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ccfa1dee81e2d12eb7db6aab02f6d04e953e7d64 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 810000, + "verbose": false, + "_step_count": 810001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.449873091292648e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dd21436a578c3f7692acb15fcf45406552aea847 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/810000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 810000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a7b243d416268dc926b074d20100812ed749e43f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4de954eeabcf30641f42b7a9134d6ec3b267146d681cd802f9f4987540b10eb +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..d7d9b557f0d16e9ca21004cc468242db56e55f03 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 1.0938305482918298e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bdfe0798022cc7490209ed74f39e2fec10601aa8 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25c01f786e5e49b5ce0e91007be83212b98cd1b9b685dfee6edbad8e35698db2 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..95c6b6ae065ce7900087956b479988f6b566afe0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:746df261164d2886cb721e37198f6163bc3d4dabc27d783880fdf90b1668a4ee +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..34b4a3103a1dd0587363f83d1665fd3745f5768f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 840000, + "verbose": false, + "_step_count": 840001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.0938305482918298e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dcc98c000f570a21f87e055135c99c45d8095ae6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/840000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 840000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log index 0f739f5d73b02ccb5464c78f556d225706e7483b..2879aef5947db11c83a230b579e9bed0d3395884 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log @@ -14,3 +14,12 @@ {"time":"2025-05-06T22:47:41.804454606Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} {"time":"2025-05-06T23:06:22.73637838Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} {"time":"2025-05-07T10:20:45.433607767Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:33216->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-07T16:28:15.433106661Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49715->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-08T02:14:18.772849593Z","level":"INFO","msg":"stream: closing","id":"or9d7oy0"} +{"time":"2025-05-08T02:14:18.772873613Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-05-08T02:14:18.772915274Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-05-08T02:14:20.058837021Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-05-08T02:14:20.336944052Z","level":"INFO","msg":"handler: closed","stream_id":"or9d7oy0"} +{"time":"2025-05-08T02:14:20.336978492Z","level":"INFO","msg":"writer: Close: closed","stream_id":"or9d7oy0"} +{"time":"2025-05-08T02:14:20.336987063Z","level":"INFO","msg":"sender: closed","stream_id":"or9d7oy0"} +{"time":"2025-05-08T02:14:20.337076564Z","level":"INFO","msg":"stream: closed","id":"or9d7oy0"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug.log index c9380a323f5aecefbc6b4434c56996a5a5a9d062..01a06dc9569f9f115e41881f835af94d98f74df9 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug.log @@ -20,3 +20,4 @@ config: {'dataset': {'repo_id': 'shylee/so100_pengripA', 'root': '/SSD/LSY/lerob 2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2371] Wrapping output streams. 2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2394] Redirects installed. 2025-05-05 13:46:34,902 INFO MainThread:102711 [wandb_init.py:init():1056] run started, returning control to user process +2025-05-08 02:14:18,771 INFO MsgRouterThr:102711 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/config.yaml b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5def68a59fcd85be58521b1d2c402cabe956d9cd --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/config.yaml @@ -0,0 +1,189 @@ +_wandb: + value: + cli_version: 0.19.9 + m: [] + python_version: 3.10.17 + t: + "1": + - 1 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 41 + - 49 + - 51 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.10.17 + "5": 0.19.9 + "8": + - 5 + "12": 0.19.9 + "13": linux-x86_64 +batch_size: + value: 16 +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: shylee/so100_pengripA + revision: null + root: /SSD/LSY/lerobot_dataset/pengripA + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4 +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.95 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 0.0001 + type: adam + weight_decay: 1e-06 +output_dir: + value: /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4 +policy: + value: + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + clip_sample_range: 1 + crop_is_random: false + crop_shape: + - 224 + - 224 + device: cuda + diffusion_step_embed_dim: 128 + do_mask_loss_for_padding: false + down_dims: + - 256 + - 512 + - 1024 + drop_n_last_frames: 7 + ema_lambda: 0.5 + freeze_vision_backbone: false + horizon: 32 + kernel_size: 5 + n_action_steps: 32 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler_type: DDIM + normalization_mapping: + ACTION: MIN_MAX + STATE: MIN_MAX + VISUAL: MEAN_STD + num_inference_steps: 10 + num_train_timesteps: 100 + optimizer_betas: + - 0.95 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-06 + pre_resize_shape: + - 224 + - 224 + prediction_type: epsilon + pretrained_backbone_weights: ResNet50_Weights.IMAGENET1K_V1 + scheduler_name: cosine + scheduler_warmup_steps: 500 + spatial_softmax_num_keypoints: 32 + type: diffusion + use_amp: false + use_ema: true + use_film_scale_modulation: true + use_group_norm: false + use_separate_rgb_encoder_per_camera: false + vision_backbone: resnet50 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 30000 +scheduler: + value: + name: cosine + num_warmup_steps: 500 + type: diffuser +seed: + value: 1000 +steps: + value: 900000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/output.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/output.log index ef7d0600ecdd6c0790b8d8c6fce2f300670e7989..c00cb78a4c39a591c076c9209a363997505d1074 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/output.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/output.log @@ -3155,3 +3155,1108 @@ INFO 2025-05-07 11:23:46 ts/train.py:232 step:624K smpl:10M ep:52K epch:171.93 l INFO 2025-05-07 11:24:43 ts/train.py:232 step:624K smpl:10M ep:52K epch:171.98 loss:0.002 grdn:0.135 lr:2.1e-05 updt_s:0.159 data_s:0.124 INFO 2025-05-07 11:25:41 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.04 loss:0.002 grdn:0.128 lr:2.1e-05 updt_s:0.160 data_s:0.129 INFO 2025-05-07 11:26:38 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.09 loss:0.002 grdn:0.133 lr:2.1e-05 updt_s:0.159 data_s:0.127 +INFO 2025-05-07 11:27:36 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.15 loss:0.003 grdn:0.145 lr:2.1e-05 updt_s:0.159 data_s:0.129 +INFO 2025-05-07 11:28:32 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.20 loss:0.002 grdn:0.135 lr:2.1e-05 updt_s:0.159 data_s:0.123 +INFO 2025-05-07 11:29:28 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.26 loss:0.002 grdn:0.131 lr:2.1e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-07 11:30:25 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.31 loss:0.002 grdn:0.128 lr:2.1e-05 updt_s:0.159 data_s:0.123 +INFO 2025-05-07 11:31:21 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.37 loss:0.002 grdn:0.142 lr:2.1e-05 updt_s:0.159 data_s:0.123 +INFO 2025-05-07 11:32:18 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.42 loss:0.002 grdn:0.127 lr:2.1e-05 updt_s:0.159 data_s:0.125 +INFO 2025-05-07 11:33:14 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.48 loss:0.002 grdn:0.143 lr:2.1e-05 updt_s:0.159 data_s:0.119 +INFO 2025-05-07 11:34:10 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.53 loss:0.002 grdn:0.125 lr:2.1e-05 updt_s:0.159 data_s:0.119 +INFO 2025-05-07 11:35:06 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.59 loss:0.002 grdn:0.138 lr:2.1e-05 updt_s:0.158 data_s:0.122 +INFO 2025-05-07 11:35:59 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.64 loss:0.002 grdn:0.142 lr:2.1e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 11:36:53 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.70 loss:0.002 grdn:0.123 lr:2.1e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 11:37:46 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.75 loss:0.002 grdn:0.135 lr:2.1e-05 updt_s:0.159 data_s:0.108 +INFO 2025-05-07 11:38:41 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.81 loss:0.002 grdn:0.123 lr:2.1e-05 updt_s:0.159 data_s:0.112 +INFO 2025-05-07 11:39:35 ts/train.py:232 step:628K smpl:10M ep:52K epch:172.86 loss:0.002 grdn:0.130 lr:2.1e-05 updt_s:0.159 data_s:0.111 +INFO 2025-05-07 11:40:30 ts/train.py:232 step:628K smpl:10M ep:52K epch:172.92 loss:0.003 grdn:0.143 lr:2.1e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-07 11:41:28 ts/train.py:232 step:628K smpl:10M ep:52K epch:172.97 loss:0.002 grdn:0.139 lr:2.1e-05 updt_s:0.159 data_s:0.131 +INFO 2025-05-07 11:42:27 ts/train.py:232 step:628K smpl:10M ep:52K epch:173.03 loss:0.002 grdn:0.134 lr:2.1e-05 updt_s:0.159 data_s:0.132 +INFO 2025-05-07 11:43:25 ts/train.py:232 step:628K smpl:10M ep:52K epch:173.08 loss:0.002 grdn:0.138 lr:2.1e-05 updt_s:0.160 data_s:0.128 +INFO 2025-05-07 11:44:24 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.14 loss:0.003 grdn:0.141 lr:2.1e-05 updt_s:0.159 data_s:0.136 +INFO 2025-05-07 11:45:22 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.19 loss:0.002 grdn:0.137 lr:2.1e-05 updt_s:0.159 data_s:0.130 +INFO 2025-05-07 11:46:20 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.25 loss:0.002 grdn:0.131 lr:2.1e-05 updt_s:0.159 data_s:0.131 +INFO 2025-05-07 11:47:17 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.30 loss:0.002 grdn:0.130 lr:2.1e-05 updt_s:0.159 data_s:0.125 +INFO 2025-05-07 11:48:15 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.36 loss:0.002 grdn:0.139 lr:2.1e-05 updt_s:0.159 data_s:0.129 +INFO 2025-05-07 11:49:12 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.41 loss:0.003 grdn:0.147 lr:2.1e-05 updt_s:0.159 data_s:0.127 +INFO 2025-05-07 11:50:11 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.47 loss:0.003 grdn:0.144 lr:2.1e-05 updt_s:0.159 data_s:0.133 +INFO 2025-05-07 11:51:07 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.52 loss:0.002 grdn:0.135 lr:2.1e-05 updt_s:0.159 data_s:0.123 +INFO 2025-05-07 11:51:07 ts/train.py:241 Checkpoint policy after step 630000 +INFO 2025-05-07 11:52:07 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.58 loss:0.002 grdn:0.140 lr:2.1e-05 updt_s:0.160 data_s:0.120 +INFO 2025-05-07 11:53:00 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.63 loss:0.002 grdn:0.131 lr:2.1e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 11:53:54 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.69 loss:0.002 grdn:0.137 lr:2.1e-05 updt_s:0.160 data_s:0.108 +INFO 2025-05-07 11:54:47 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.74 loss:0.002 grdn:0.120 lr:2.1e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-07 11:55:40 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.80 loss:0.002 grdn:0.135 lr:2.1e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-07 11:56:34 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.85 loss:0.002 grdn:0.136 lr:2.0e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-07 11:57:28 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.91 loss:0.002 grdn:0.125 lr:2.0e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-07 11:58:21 ts/train.py:232 step:632K smpl:10M ep:52K epch:173.96 loss:0.002 grdn:0.133 lr:2.0e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-07 11:59:15 ts/train.py:232 step:632K smpl:10M ep:52K epch:174.02 loss:0.002 grdn:0.131 lr:2.0e-05 updt_s:0.159 data_s:0.109 +INFO 2025-05-07 12:00:09 ts/train.py:232 step:632K smpl:10M ep:52K epch:174.07 loss:0.002 grdn:0.126 lr:2.0e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-07 12:01:02 ts/train.py:232 step:632K smpl:10M ep:52K epch:174.13 loss:0.002 grdn:0.142 lr:2.0e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 12:01:56 ts/train.py:232 step:632K smpl:10M ep:52K epch:174.18 loss:0.002 grdn:0.135 lr:2.0e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-07 12:02:49 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.24 loss:0.002 grdn:0.133 lr:2.0e-05 updt_s:0.159 data_s:0.108 +INFO 2025-05-07 12:03:44 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.30 loss:0.002 grdn:0.127 lr:2.0e-05 updt_s:0.159 data_s:0.112 +INFO 2025-05-07 12:04:38 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.35 loss:0.002 grdn:0.137 lr:2.0e-05 updt_s:0.159 data_s:0.111 +INFO 2025-05-07 12:05:32 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.41 loss:0.002 grdn:0.142 lr:2.0e-05 updt_s:0.159 data_s:0.111 +INFO 2025-05-07 12:06:25 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.46 loss:0.002 grdn:0.137 lr:2.0e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 12:07:19 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.52 loss:0.002 grdn:0.133 lr:2.0e-05 updt_s:0.159 data_s:0.113 +INFO 2025-05-07 12:08:11 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.57 loss:0.002 grdn:0.133 lr:2.0e-05 updt_s:0.160 data_s:0.096 +INFO 2025-05-07 12:09:02 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.63 loss:0.002 grdn:0.135 lr:2.0e-05 updt_s:0.159 data_s:0.094 +INFO 2025-05-07 12:09:53 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.68 loss:0.002 grdn:0.139 lr:2.0e-05 updt_s:0.160 data_s:0.095 +INFO 2025-05-07 12:10:43 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.74 loss:0.002 grdn:0.137 lr:2.0e-05 updt_s:0.160 data_s:0.093 +INFO 2025-05-07 12:11:34 ts/train.py:232 step:635K smpl:10M ep:52K epch:174.79 loss:0.002 grdn:0.134 lr:2.0e-05 updt_s:0.159 data_s:0.095 +INFO 2025-05-07 12:12:28 ts/train.py:232 step:635K smpl:10M ep:52K epch:174.85 loss:0.002 grdn:0.129 lr:2.0e-05 updt_s:0.159 data_s:0.109 +INFO 2025-05-07 12:13:25 ts/train.py:232 step:635K smpl:10M ep:52K epch:174.90 loss:0.002 grdn:0.127 lr:2.0e-05 updt_s:0.159 data_s:0.122 +INFO 2025-05-07 12:14:22 ts/train.py:232 step:635K smpl:10M ep:52K epch:174.96 loss:0.002 grdn:0.136 lr:2.0e-05 updt_s:0.159 data_s:0.125 +INFO 2025-05-07 12:15:19 ts/train.py:232 step:635K smpl:10M ep:53K epch:175.01 loss:0.002 grdn:0.115 lr:2.0e-05 updt_s:0.159 data_s:0.126 +INFO 2025-05-07 12:16:16 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.07 loss:0.002 grdn:0.139 lr:2.0e-05 updt_s:0.159 data_s:0.126 +INFO 2025-05-07 12:17:13 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.12 loss:0.002 grdn:0.134 lr:2.0e-05 updt_s:0.160 data_s:0.123 +INFO 2025-05-07 12:18:09 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.18 loss:0.002 grdn:0.125 lr:2.0e-05 updt_s:0.159 data_s:0.120 +INFO 2025-05-07 12:19:06 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.23 loss:0.002 grdn:0.138 lr:2.0e-05 updt_s:0.160 data_s:0.126 +INFO 2025-05-07 12:20:03 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.29 loss:0.002 grdn:0.135 lr:2.0e-05 updt_s:0.159 data_s:0.122 +INFO 2025-05-07 12:20:59 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.34 loss:0.002 grdn:0.131 lr:2.0e-05 updt_s:0.160 data_s:0.122 +INFO 2025-05-07 12:21:56 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.40 loss:0.002 grdn:0.138 lr:2.0e-05 updt_s:0.160 data_s:0.125 +INFO 2025-05-07 12:22:55 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.45 loss:0.002 grdn:0.124 lr:2.0e-05 updt_s:0.159 data_s:0.134 +INFO 2025-05-07 12:23:45 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.51 loss:0.002 grdn:0.120 lr:2.0e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-07 12:24:32 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.56 loss:0.002 grdn:0.144 lr:2.0e-05 updt_s:0.160 data_s:0.074 +INFO 2025-05-07 12:25:18 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.62 loss:0.002 grdn:0.138 lr:2.0e-05 updt_s:0.160 data_s:0.070 +INFO 2025-05-07 12:26:06 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.67 loss:0.002 grdn:0.137 lr:2.0e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-07 12:26:52 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.73 loss:0.002 grdn:0.140 lr:2.0e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 12:27:39 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.78 loss:0.003 grdn:0.148 lr:1.9e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 12:28:26 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.84 loss:0.002 grdn:0.131 lr:1.9e-05 updt_s:0.160 data_s:0.073 +INFO 2025-05-07 12:29:14 ts/train.py:232 step:639K smpl:10M ep:53K epch:175.89 loss:0.002 grdn:0.125 lr:1.9e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-07 12:30:02 ts/train.py:232 step:639K smpl:10M ep:53K epch:175.95 loss:0.002 grdn:0.137 lr:1.9e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-07 12:30:51 ts/train.py:232 step:639K smpl:10M ep:53K epch:176.00 loss:0.002 grdn:0.133 lr:1.9e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-07 12:31:39 ts/train.py:232 step:639K smpl:10M ep:53K epch:176.06 loss:0.002 grdn:0.125 lr:1.9e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 12:32:28 ts/train.py:232 step:639K smpl:10M ep:53K epch:176.11 loss:0.002 grdn:0.134 lr:1.9e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-07 12:33:16 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.17 loss:0.002 grdn:0.120 lr:1.9e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-07 12:34:04 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.22 loss:0.002 grdn:0.125 lr:1.9e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-07 12:34:53 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.28 loss:0.003 grdn:0.147 lr:1.9e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-07 12:35:41 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.33 loss:0.002 grdn:0.131 lr:1.9e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-07 12:36:29 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.39 loss:0.002 grdn:0.137 lr:1.9e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-07 12:37:22 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.44 loss:0.002 grdn:0.128 lr:1.9e-05 updt_s:0.158 data_s:0.107 +INFO 2025-05-07 12:38:15 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.50 loss:0.002 grdn:0.134 lr:1.9e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 12:39:09 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.55 loss:0.002 grdn:0.130 lr:1.9e-05 updt_s:0.159 data_s:0.110 +INFO 2025-05-07 12:40:02 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.61 loss:0.002 grdn:0.140 lr:1.9e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 12:40:56 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.66 loss:0.002 grdn:0.149 lr:1.9e-05 updt_s:0.159 data_s:0.108 +INFO 2025-05-07 12:41:50 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.72 loss:0.002 grdn:0.131 lr:1.9e-05 updt_s:0.159 data_s:0.109 +INFO 2025-05-07 12:42:46 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.77 loss:0.002 grdn:0.119 lr:1.9e-05 updt_s:0.160 data_s:0.120 +INFO 2025-05-07 12:43:44 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.83 loss:0.002 grdn:0.122 lr:1.9e-05 updt_s:0.160 data_s:0.129 +INFO 2025-05-07 12:44:42 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.88 loss:0.002 grdn:0.145 lr:1.9e-05 updt_s:0.159 data_s:0.130 +INFO 2025-05-07 12:45:40 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.94 loss:0.002 grdn:0.133 lr:1.9e-05 updt_s:0.159 data_s:0.131 +INFO 2025-05-07 12:46:37 ts/train.py:232 step:643K smpl:10M ep:53K epch:176.99 loss:0.002 grdn:0.139 lr:1.9e-05 updt_s:0.159 data_s:0.126 +INFO 2025-05-07 12:47:34 ts/train.py:232 step:643K smpl:10M ep:53K epch:177.05 loss:0.002 grdn:0.138 lr:1.9e-05 updt_s:0.159 data_s:0.126 +INFO 2025-05-07 12:48:33 ts/train.py:232 step:643K smpl:10M ep:53K epch:177.10 loss:0.002 grdn:0.133 lr:1.9e-05 updt_s:0.159 data_s:0.133 +INFO 2025-05-07 12:49:30 ts/train.py:232 step:643K smpl:10M ep:53K epch:177.16 loss:0.002 grdn:0.122 lr:1.9e-05 updt_s:0.159 data_s:0.125 +INFO 2025-05-07 12:50:29 ts/train.py:232 step:643K smpl:10M ep:53K epch:177.21 loss:0.002 grdn:0.132 lr:1.9e-05 updt_s:0.159 data_s:0.134 +INFO 2025-05-07 12:51:26 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.27 loss:0.002 grdn:0.133 lr:1.9e-05 updt_s:0.160 data_s:0.128 +INFO 2025-05-07 12:52:24 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.32 loss:0.002 grdn:0.117 lr:1.9e-05 updt_s:0.159 data_s:0.129 +INFO 2025-05-07 12:53:22 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.38 loss:0.002 grdn:0.140 lr:1.9e-05 updt_s:0.159 data_s:0.127 +INFO 2025-05-07 12:54:20 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.44 loss:0.002 grdn:0.141 lr:1.9e-05 updt_s:0.160 data_s:0.130 +INFO 2025-05-07 12:55:14 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.49 loss:0.002 grdn:0.143 lr:1.9e-05 updt_s:0.159 data_s:0.112 +INFO 2025-05-07 12:56:08 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.55 loss:0.002 grdn:0.134 lr:1.9e-05 updt_s:0.159 data_s:0.112 +INFO 2025-05-07 12:57:03 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.60 loss:0.002 grdn:0.135 lr:1.9e-05 updt_s:0.159 data_s:0.113 +INFO 2025-05-07 12:57:58 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.66 loss:0.002 grdn:0.131 lr:1.9e-05 updt_s:0.159 data_s:0.114 +INFO 2025-05-07 12:58:54 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.71 loss:0.002 grdn:0.127 lr:1.9e-05 updt_s:0.159 data_s:0.119 +INFO 2025-05-07 12:59:51 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.77 loss:0.002 grdn:0.149 lr:1.9e-05 updt_s:0.159 data_s:0.129 +INFO 2025-05-07 13:00:52 ts/train.py:232 step:646K smpl:10M ep:53K epch:177.82 loss:0.002 grdn:0.126 lr:1.8e-05 updt_s:0.159 data_s:0.142 +INFO 2025-05-07 13:01:50 ts/train.py:232 step:646K smpl:10M ep:53K epch:177.88 loss:0.002 grdn:0.131 lr:1.8e-05 updt_s:0.160 data_s:0.131 +INFO 2025-05-07 13:02:48 ts/train.py:232 step:646K smpl:10M ep:53K epch:177.93 loss:0.002 grdn:0.128 lr:1.8e-05 updt_s:0.159 data_s:0.131 +INFO 2025-05-07 13:03:47 ts/train.py:232 step:646K smpl:10M ep:53K epch:177.99 loss:0.002 grdn:0.145 lr:1.8e-05 updt_s:0.159 data_s:0.136 +INFO 2025-05-07 13:04:46 ts/train.py:232 step:646K smpl:10M ep:53K epch:178.04 loss:0.002 grdn:0.135 lr:1.8e-05 updt_s:0.160 data_s:0.133 +INFO 2025-05-07 13:05:45 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.10 loss:0.002 grdn:0.130 lr:1.8e-05 updt_s:0.160 data_s:0.132 +INFO 2025-05-07 13:06:44 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.15 loss:0.002 grdn:0.129 lr:1.8e-05 updt_s:0.159 data_s:0.136 +INFO 2025-05-07 13:07:44 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.21 loss:0.002 grdn:0.135 lr:1.8e-05 updt_s:0.159 data_s:0.138 +INFO 2025-05-07 13:08:41 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.26 loss:0.002 grdn:0.126 lr:1.8e-05 updt_s:0.160 data_s:0.127 +INFO 2025-05-07 13:09:40 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.32 loss:0.002 grdn:0.138 lr:1.8e-05 updt_s:0.160 data_s:0.134 +INFO 2025-05-07 13:10:38 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.37 loss:0.002 grdn:0.126 lr:1.8e-05 updt_s:0.159 data_s:0.131 +INFO 2025-05-07 13:11:34 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.43 loss:0.002 grdn:0.142 lr:1.8e-05 updt_s:0.159 data_s:0.117 +INFO 2025-05-07 13:12:29 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.48 loss:0.002 grdn:0.133 lr:1.8e-05 updt_s:0.159 data_s:0.116 +INFO 2025-05-07 13:13:24 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.54 loss:0.002 grdn:0.132 lr:1.8e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-07 13:14:19 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.59 loss:0.002 grdn:0.134 lr:1.8e-05 updt_s:0.159 data_s:0.117 +INFO 2025-05-07 13:15:15 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.65 loss:0.002 grdn:0.135 lr:1.8e-05 updt_s:0.159 data_s:0.116 +INFO 2025-05-07 13:16:11 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.70 loss:0.002 grdn:0.127 lr:1.8e-05 updt_s:0.159 data_s:0.121 +INFO 2025-05-07 13:17:10 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.76 loss:0.002 grdn:0.133 lr:1.8e-05 updt_s:0.161 data_s:0.135 +INFO 2025-05-07 13:18:10 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.81 loss:0.002 grdn:0.139 lr:1.8e-05 updt_s:0.159 data_s:0.139 +INFO 2025-05-07 13:19:09 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.87 loss:0.002 grdn:0.124 lr:1.8e-05 updt_s:0.159 data_s:0.138 +INFO 2025-05-07 13:20:10 ts/train.py:232 step:650K smpl:10M ep:54K epch:178.92 loss:0.002 grdn:0.133 lr:1.8e-05 updt_s:0.159 data_s:0.144 +INFO 2025-05-07 13:21:10 ts/train.py:232 step:650K smpl:10M ep:54K epch:178.98 loss:0.002 grdn:0.143 lr:1.8e-05 updt_s:0.159 data_s:0.141 +INFO 2025-05-07 13:22:10 ts/train.py:232 step:650K smpl:10M ep:54K epch:179.03 loss:0.002 grdn:0.126 lr:1.8e-05 updt_s:0.159 data_s:0.138 +INFO 2025-05-07 13:23:08 ts/train.py:232 step:650K smpl:10M ep:54K epch:179.09 loss:0.002 grdn:0.127 lr:1.8e-05 updt_s:0.159 data_s:0.131 +INFO 2025-05-07 13:24:07 ts/train.py:232 step:650K smpl:10M ep:54K epch:179.14 loss:0.002 grdn:0.134 lr:1.8e-05 updt_s:0.160 data_s:0.132 +INFO 2025-05-07 13:25:06 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.20 loss:0.002 grdn:0.133 lr:1.8e-05 updt_s:0.159 data_s:0.136 +INFO 2025-05-07 13:26:04 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.25 loss:0.002 grdn:0.126 lr:1.8e-05 updt_s:0.159 data_s:0.132 +INFO 2025-05-07 13:27:02 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.31 loss:0.002 grdn:0.138 lr:1.8e-05 updt_s:0.159 data_s:0.130 +INFO 2025-05-07 13:27:57 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.36 loss:0.002 grdn:0.130 lr:1.8e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-07 13:28:50 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.42 loss:0.002 grdn:0.136 lr:1.8e-05 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 13:29:43 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.47 loss:0.002 grdn:0.131 lr:1.8e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 13:30:37 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.53 loss:0.002 grdn:0.139 lr:1.8e-05 updt_s:0.159 data_s:0.107 +INFO 2025-05-07 13:31:30 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.58 loss:0.002 grdn:0.134 lr:1.8e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 13:32:23 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.64 loss:0.002 grdn:0.140 lr:1.8e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-07 13:33:16 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.69 loss:0.002 grdn:0.143 lr:1.8e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 13:34:09 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.75 loss:0.002 grdn:0.151 lr:1.8e-05 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 13:35:02 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.80 loss:0.002 grdn:0.130 lr:1.8e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 13:35:54 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.86 loss:0.002 grdn:0.139 lr:1.7e-05 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 13:36:47 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.91 loss:0.002 grdn:0.144 lr:1.7e-05 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 13:37:40 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.97 loss:0.002 grdn:0.126 lr:1.7e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 13:38:33 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.02 loss:0.002 grdn:0.123 lr:1.7e-05 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 13:39:25 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.08 loss:0.002 grdn:0.127 lr:1.7e-05 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 13:40:18 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.13 loss:0.002 grdn:0.144 lr:1.7e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 13:41:11 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.19 loss:0.002 grdn:0.127 lr:1.7e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 13:42:04 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.24 loss:0.002 grdn:0.116 lr:1.7e-05 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 13:42:59 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.30 loss:0.002 grdn:0.135 lr:1.7e-05 updt_s:0.158 data_s:0.115 +INFO 2025-05-07 13:43:52 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.35 loss:0.002 grdn:0.131 lr:1.7e-05 updt_s:0.159 data_s:0.107 +INFO 2025-05-07 13:44:46 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.41 loss:0.002 grdn:0.128 lr:1.7e-05 updt_s:0.159 data_s:0.109 +INFO 2025-05-07 13:45:39 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.46 loss:0.002 grdn:0.127 lr:1.7e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 13:46:32 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.52 loss:0.002 grdn:0.138 lr:1.7e-05 updt_s:0.159 data_s:0.107 +INFO 2025-05-07 13:47:26 ts/train.py:232 step:656K smpl:10M ep:54K epch:180.57 loss:0.002 grdn:0.140 lr:1.7e-05 updt_s:0.159 data_s:0.108 +INFO 2025-05-07 13:48:21 ts/train.py:232 step:656K smpl:10M ep:54K epch:180.63 loss:0.002 grdn:0.128 lr:1.7e-05 updt_s:0.159 data_s:0.115 +INFO 2025-05-07 13:49:18 ts/train.py:232 step:656K smpl:10M ep:54K epch:180.69 loss:0.002 grdn:0.130 lr:1.7e-05 updt_s:0.159 data_s:0.126 +INFO 2025-05-07 13:50:15 ts/train.py:232 step:656K smpl:10M ep:54K epch:180.74 loss:0.002 grdn:0.131 lr:1.7e-05 updt_s:0.159 data_s:0.124 +INFO 2025-05-07 13:51:13 ts/train.py:232 step:656K smpl:11M ep:54K epch:180.80 loss:0.002 grdn:0.132 lr:1.7e-05 updt_s:0.160 data_s:0.129 +INFO 2025-05-07 13:52:09 ts/train.py:232 step:657K smpl:11M ep:54K epch:180.85 loss:0.002 grdn:0.142 lr:1.7e-05 updt_s:0.159 data_s:0.121 +INFO 2025-05-07 13:53:06 ts/train.py:232 step:657K smpl:11M ep:54K epch:180.91 loss:0.002 grdn:0.128 lr:1.7e-05 updt_s:0.159 data_s:0.126 +INFO 2025-05-07 13:54:03 ts/train.py:232 step:657K smpl:11M ep:54K epch:180.96 loss:0.002 grdn:0.133 lr:1.7e-05 updt_s:0.159 data_s:0.123 +INFO 2025-05-07 13:55:01 ts/train.py:232 step:657K smpl:11M ep:54K epch:181.02 loss:0.002 grdn:0.136 lr:1.7e-05 updt_s:0.159 data_s:0.130 +INFO 2025-05-07 13:55:58 ts/train.py:232 step:657K smpl:11M ep:54K epch:181.07 loss:0.002 grdn:0.136 lr:1.7e-05 updt_s:0.159 data_s:0.124 +INFO 2025-05-07 13:56:53 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.13 loss:0.002 grdn:0.132 lr:1.7e-05 updt_s:0.160 data_s:0.119 +INFO 2025-05-07 13:57:49 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.18 loss:0.002 grdn:0.123 lr:1.7e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-07 13:58:45 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.24 loss:0.002 grdn:0.128 lr:1.7e-05 updt_s:0.159 data_s:0.122 +INFO 2025-05-07 13:59:36 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.29 loss:0.002 grdn:0.136 lr:1.7e-05 updt_s:0.159 data_s:0.095 +INFO 2025-05-07 14:00:23 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.35 loss:0.002 grdn:0.138 lr:1.7e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 14:01:10 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.40 loss:0.002 grdn:0.142 lr:1.7e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 14:01:57 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.46 loss:0.002 grdn:0.141 lr:1.7e-05 updt_s:0.160 data_s:0.076 +INFO 2025-05-07 14:02:43 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.51 loss:0.002 grdn:0.141 lr:1.7e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 14:03:31 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.57 loss:0.002 grdn:0.129 lr:1.7e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-07 14:04:21 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.62 loss:0.002 grdn:0.135 lr:1.7e-05 updt_s:0.159 data_s:0.090 +INFO 2025-05-07 14:05:11 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.68 loss:0.002 grdn:0.134 lr:1.7e-05 updt_s:0.159 data_s:0.090 +INFO 2025-05-07 14:06:01 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.73 loss:0.002 grdn:0.134 lr:1.7e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-07 14:06:49 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.79 loss:0.002 grdn:0.124 lr:1.7e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-07 14:06:49 ts/train.py:241 Checkpoint policy after step 660000 +INFO 2025-05-07 14:07:43 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.84 loss:0.002 grdn:0.140 lr:1.7e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-07 14:08:32 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.90 loss:0.002 grdn:0.130 lr:1.7e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-07 14:09:20 ts/train.py:232 step:661K smpl:11M ep:55K epch:181.95 loss:0.002 grdn:0.141 lr:1.6e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-07 14:10:09 ts/train.py:232 step:661K smpl:11M ep:55K epch:182.01 loss:0.002 grdn:0.146 lr:1.6e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-07 14:10:59 ts/train.py:232 step:661K smpl:11M ep:55K epch:182.06 loss:0.002 grdn:0.136 lr:1.6e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-07 14:11:47 ts/train.py:232 step:661K smpl:11M ep:55K epch:182.12 loss:0.002 grdn:0.124 lr:1.6e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-07 14:12:37 ts/train.py:232 step:661K smpl:11M ep:55K epch:182.17 loss:0.002 grdn:0.130 lr:1.6e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-07 14:13:29 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.23 loss:0.002 grdn:0.134 lr:1.6e-05 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 14:14:22 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.28 loss:0.002 grdn:0.132 lr:1.6e-05 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 14:15:15 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.34 loss:0.002 grdn:0.144 lr:1.6e-05 updt_s:0.159 data_s:0.102 +INFO 2025-05-07 14:16:08 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.39 loss:0.002 grdn:0.132 lr:1.6e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 14:17:01 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.45 loss:0.002 grdn:0.135 lr:1.6e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 14:17:53 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.50 loss:0.002 grdn:0.150 lr:1.6e-05 updt_s:0.159 data_s:0.101 +INFO 2025-05-07 14:18:44 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.56 loss:0.002 grdn:0.136 lr:1.6e-05 updt_s:0.160 data_s:0.092 +INFO 2025-05-07 14:19:32 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.61 loss:0.002 grdn:0.129 lr:1.6e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-07 14:20:18 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.67 loss:0.002 grdn:0.145 lr:1.6e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 14:21:06 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.72 loss:0.002 grdn:0.136 lr:1.6e-05 updt_s:0.160 data_s:0.077 +INFO 2025-05-07 14:21:52 ts/train.py:232 step:664K smpl:11M ep:55K epch:182.78 loss:0.002 grdn:0.128 lr:1.6e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 14:22:40 ts/train.py:232 step:664K smpl:11M ep:55K epch:182.83 loss:0.002 grdn:0.134 lr:1.6e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 14:23:27 ts/train.py:232 step:664K smpl:11M ep:55K epch:182.89 loss:0.002 grdn:0.126 lr:1.6e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 14:24:13 ts/train.py:232 step:664K smpl:11M ep:55K epch:182.94 loss:0.002 grdn:0.128 lr:1.6e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 14:25:00 ts/train.py:232 step:664K smpl:11M ep:55K epch:183.00 loss:0.002 grdn:0.140 lr:1.6e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 14:25:50 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.05 loss:0.002 grdn:0.132 lr:1.6e-05 updt_s:0.160 data_s:0.089 +INFO 2025-05-07 14:26:39 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.11 loss:0.002 grdn:0.132 lr:1.6e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-07 14:27:27 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.16 loss:0.002 grdn:0.133 lr:1.6e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-07 14:28:16 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.22 loss:0.002 grdn:0.138 lr:1.6e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-07 14:29:04 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.27 loss:0.002 grdn:0.147 lr:1.6e-05 updt_s:0.160 data_s:0.076 +INFO 2025-05-07 14:29:50 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.33 loss:0.002 grdn:0.140 lr:1.6e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 14:30:36 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.38 loss:0.002 grdn:0.124 lr:1.6e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 14:31:23 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.44 loss:0.002 grdn:0.122 lr:1.6e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 14:32:11 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.49 loss:0.002 grdn:0.137 lr:1.6e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-07 14:32:55 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.55 loss:0.002 grdn:0.128 lr:1.6e-05 updt_s:0.159 data_s:0.063 +INFO 2025-05-07 14:33:40 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.60 loss:0.002 grdn:0.135 lr:1.6e-05 updt_s:0.159 data_s:0.063 +INFO 2025-05-07 14:34:25 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.66 loss:0.002 grdn:0.124 lr:1.6e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 14:35:12 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.71 loss:0.002 grdn:0.128 lr:1.6e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 14:35:57 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.77 loss:0.002 grdn:0.133 lr:1.6e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 14:36:43 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.83 loss:0.002 grdn:0.136 lr:1.6e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 14:37:28 ts/train.py:232 step:668K smpl:11M ep:55K epch:183.88 loss:0.002 grdn:0.135 lr:1.6e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 14:38:14 ts/train.py:232 step:668K smpl:11M ep:55K epch:183.94 loss:0.002 grdn:0.139 lr:1.6e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 14:39:00 ts/train.py:232 step:668K smpl:11M ep:55K epch:183.99 loss:0.002 grdn:0.139 lr:1.6e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 14:39:46 ts/train.py:232 step:668K smpl:11M ep:55K epch:184.05 loss:0.002 grdn:0.136 lr:1.6e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 14:40:32 ts/train.py:232 step:668K smpl:11M ep:55K epch:184.10 loss:0.002 grdn:0.136 lr:1.5e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 14:41:23 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.16 loss:0.002 grdn:0.132 lr:1.5e-05 updt_s:0.158 data_s:0.100 +INFO 2025-05-07 14:42:18 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.21 loss:0.002 grdn:0.133 lr:1.5e-05 updt_s:0.159 data_s:0.114 +INFO 2025-05-07 14:43:13 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.27 loss:0.002 grdn:0.126 lr:1.5e-05 updt_s:0.159 data_s:0.112 +INFO 2025-05-07 14:44:06 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.32 loss:0.002 grdn:0.127 lr:1.5e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 14:44:57 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.38 loss:0.002 grdn:0.125 lr:1.5e-05 updt_s:0.159 data_s:0.098 +INFO 2025-05-07 14:45:51 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.43 loss:0.002 grdn:0.142 lr:1.5e-05 updt_s:0.159 data_s:0.110 +INFO 2025-05-07 14:46:42 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.49 loss:0.002 grdn:0.136 lr:1.5e-05 updt_s:0.159 data_s:0.093 +INFO 2025-05-07 14:47:27 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.54 loss:0.002 grdn:0.133 lr:1.5e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 14:48:13 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.60 loss:0.002 grdn:0.138 lr:1.5e-05 updt_s:0.160 data_s:0.070 +INFO 2025-05-07 14:48:58 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.65 loss:0.002 grdn:0.148 lr:1.5e-05 updt_s:0.160 data_s:0.066 +INFO 2025-05-07 14:49:43 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.71 loss:0.002 grdn:0.130 lr:1.5e-05 updt_s:0.160 data_s:0.066 +INFO 2025-05-07 14:50:29 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.76 loss:0.002 grdn:0.138 lr:1.5e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 14:51:14 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.82 loss:0.002 grdn:0.122 lr:1.5e-05 updt_s:0.160 data_s:0.066 +INFO 2025-05-07 14:51:59 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.87 loss:0.002 grdn:0.135 lr:1.5e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 14:52:45 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.93 loss:0.002 grdn:0.127 lr:1.5e-05 updt_s:0.160 data_s:0.067 +INFO 2025-05-07 14:53:30 ts/train.py:232 step:672K smpl:11M ep:55K epch:184.98 loss:0.002 grdn:0.135 lr:1.5e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 14:54:15 ts/train.py:232 step:672K smpl:11M ep:56K epch:185.04 loss:0.002 grdn:0.141 lr:1.5e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 14:55:00 ts/train.py:232 step:672K smpl:11M ep:56K epch:185.09 loss:0.002 grdn:0.139 lr:1.5e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 14:55:55 ts/train.py:232 step:672K smpl:11M ep:56K epch:185.15 loss:0.002 grdn:0.126 lr:1.5e-05 updt_s:0.159 data_s:0.114 +INFO 2025-05-07 14:56:47 ts/train.py:232 step:672K smpl:11M ep:56K epch:185.20 loss:0.002 grdn:0.130 lr:1.5e-05 updt_s:0.159 data_s:0.102 +INFO 2025-05-07 14:57:41 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.26 loss:0.002 grdn:0.133 lr:1.5e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-07 14:58:34 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.31 loss:0.002 grdn:0.130 lr:1.5e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-07 14:59:28 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.37 loss:0.002 grdn:0.116 lr:1.5e-05 updt_s:0.159 data_s:0.108 +INFO 2025-05-07 15:00:21 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.42 loss:0.002 grdn:0.135 lr:1.5e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 15:01:07 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.48 loss:0.002 grdn:0.131 lr:1.5e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 15:01:51 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.53 loss:0.002 grdn:0.141 lr:1.5e-05 updt_s:0.158 data_s:0.064 +INFO 2025-05-07 15:02:36 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.59 loss:0.002 grdn:0.131 lr:1.5e-05 updt_s:0.159 data_s:0.062 +INFO 2025-05-07 15:03:20 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.64 loss:0.002 grdn:0.133 lr:1.5e-05 updt_s:0.159 data_s:0.061 +INFO 2025-05-07 15:04:05 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.70 loss:0.002 grdn:0.141 lr:1.5e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 15:04:48 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.75 loss:0.002 grdn:0.142 lr:1.5e-05 updt_s:0.160 data_s:0.059 +INFO 2025-05-07 15:05:33 ts/train.py:232 step:675K smpl:11M ep:56K epch:185.81 loss:0.002 grdn:0.134 lr:1.5e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 15:06:18 ts/train.py:232 step:675K smpl:11M ep:56K epch:185.86 loss:0.002 grdn:0.135 lr:1.5e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 15:07:03 ts/train.py:232 step:675K smpl:11M ep:56K epch:185.92 loss:0.002 grdn:0.130 lr:1.5e-05 updt_s:0.159 data_s:0.062 +INFO 2025-05-07 15:07:47 ts/train.py:232 step:675K smpl:11M ep:56K epch:185.97 loss:0.002 grdn:0.125 lr:1.5e-05 updt_s:0.160 data_s:0.063 +INFO 2025-05-07 15:08:35 ts/train.py:232 step:675K smpl:11M ep:56K epch:186.03 loss:0.002 grdn:0.140 lr:1.5e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-07 15:09:23 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.08 loss:0.002 grdn:0.120 lr:1.5e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-07 15:10:12 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.14 loss:0.002 grdn:0.135 lr:1.5e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-07 15:10:59 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.19 loss:0.002 grdn:0.134 lr:1.5e-05 updt_s:0.160 data_s:0.072 +INFO 2025-05-07 15:11:47 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.25 loss:0.002 grdn:0.122 lr:1.5e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-07 15:12:34 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.30 loss:0.002 grdn:0.130 lr:1.4e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-07 15:13:23 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.36 loss:0.002 grdn:0.142 lr:1.4e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-07 15:14:09 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.41 loss:0.002 grdn:0.135 lr:1.4e-05 updt_s:0.160 data_s:0.070 +INFO 2025-05-07 15:14:55 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.47 loss:0.002 grdn:0.135 lr:1.4e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 15:15:42 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.52 loss:0.002 grdn:0.127 lr:1.4e-05 updt_s:0.160 data_s:0.074 +INFO 2025-05-07 15:16:28 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.58 loss:0.002 grdn:0.140 lr:1.4e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 15:17:14 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.63 loss:0.002 grdn:0.131 lr:1.4e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 15:18:00 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.69 loss:0.002 grdn:0.140 lr:1.4e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 15:18:46 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.74 loss:0.002 grdn:0.125 lr:1.4e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 15:19:32 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.80 loss:0.002 grdn:0.135 lr:1.4e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 15:20:18 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.85 loss:0.002 grdn:0.125 lr:1.4e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 15:21:05 ts/train.py:232 step:679K smpl:11M ep:56K epch:186.91 loss:0.002 grdn:0.129 lr:1.4e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-07 15:21:51 ts/train.py:232 step:679K smpl:11M ep:56K epch:186.97 loss:0.002 grdn:0.137 lr:1.4e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 15:22:38 ts/train.py:232 step:679K smpl:11M ep:56K epch:187.02 loss:0.001 grdn:0.110 lr:1.4e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 15:23:32 ts/train.py:232 step:679K smpl:11M ep:56K epch:187.08 loss:0.002 grdn:0.129 lr:1.4e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-07 15:24:25 ts/train.py:232 step:679K smpl:11M ep:56K epch:187.13 loss:0.002 grdn:0.140 lr:1.4e-05 updt_s:0.158 data_s:0.104 +INFO 2025-05-07 15:25:18 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.19 loss:0.002 grdn:0.134 lr:1.4e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 15:26:11 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.24 loss:0.002 grdn:0.131 lr:1.4e-05 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 15:27:03 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.30 loss:0.002 grdn:0.135 lr:1.4e-05 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 15:27:56 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.35 loss:0.002 grdn:0.129 lr:1.4e-05 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 15:28:42 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.41 loss:0.002 grdn:0.129 lr:1.4e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 15:29:27 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.46 loss:0.002 grdn:0.147 lr:1.4e-05 updt_s:0.158 data_s:0.064 +INFO 2025-05-07 15:30:12 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.52 loss:0.002 grdn:0.128 lr:1.4e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 15:30:58 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.57 loss:0.002 grdn:0.132 lr:1.4e-05 updt_s:0.160 data_s:0.067 +INFO 2025-05-07 15:31:43 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.63 loss:0.002 grdn:0.138 lr:1.4e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 15:32:28 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.68 loss:0.002 grdn:0.123 lr:1.4e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 15:33:13 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.74 loss:0.002 grdn:0.126 lr:1.4e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 15:33:58 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.79 loss:0.002 grdn:0.140 lr:1.4e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 15:34:43 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.85 loss:0.002 grdn:0.128 lr:1.4e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 15:35:28 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.90 loss:0.002 grdn:0.136 lr:1.4e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 15:36:12 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.96 loss:0.002 grdn:0.125 lr:1.4e-05 updt_s:0.159 data_s:0.063 +INFO 2025-05-07 15:37:00 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.01 loss:0.002 grdn:0.137 lr:1.4e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 15:37:46 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.07 loss:0.002 grdn:0.133 lr:1.4e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 15:38:31 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.12 loss:0.002 grdn:0.139 lr:1.4e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 15:39:19 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.18 loss:0.002 grdn:0.135 lr:1.4e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-07 15:40:07 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.23 loss:0.002 grdn:0.131 lr:1.4e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-07 15:40:53 ts/train.py:232 step:684K smpl:11M ep:56K epch:188.29 loss:0.002 grdn:0.129 lr:1.4e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 15:41:40 ts/train.py:232 step:684K smpl:11M ep:57K epch:188.34 loss:0.002 grdn:0.148 lr:1.4e-05 updt_s:0.160 data_s:0.073 +INFO 2025-05-07 15:42:25 ts/train.py:232 step:684K smpl:11M ep:57K epch:188.40 loss:0.002 grdn:0.137 lr:1.4e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 15:43:11 ts/train.py:232 step:684K smpl:11M ep:57K epch:188.45 loss:0.002 grdn:0.134 lr:1.4e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 15:43:57 ts/train.py:232 step:684K smpl:11M ep:57K epch:188.51 loss:0.002 grdn:0.129 lr:1.4e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 15:44:44 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.56 loss:0.002 grdn:0.131 lr:1.4e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 15:45:30 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.62 loss:0.002 grdn:0.129 lr:1.3e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 15:46:17 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.67 loss:0.002 grdn:0.123 lr:1.3e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 15:47:04 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.73 loss:0.002 grdn:0.130 lr:1.3e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 15:47:49 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.78 loss:0.002 grdn:0.121 lr:1.3e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 15:48:35 ts/train.py:232 step:686K smpl:11M ep:57K epch:188.84 loss:0.002 grdn:0.124 lr:1.3e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 15:49:20 ts/train.py:232 step:686K smpl:11M ep:57K epch:188.89 loss:0.002 grdn:0.132 lr:1.3e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 15:50:05 ts/train.py:232 step:686K smpl:11M ep:57K epch:188.95 loss:0.002 grdn:0.141 lr:1.3e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 15:51:01 ts/train.py:232 step:686K smpl:11M ep:57K epch:189.00 loss:0.002 grdn:0.130 lr:1.3e-05 updt_s:0.159 data_s:0.117 +INFO 2025-05-07 15:51:54 ts/train.py:232 step:686K smpl:11M ep:57K epch:189.06 loss:0.002 grdn:0.128 lr:1.3e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 15:52:48 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.11 loss:0.002 grdn:0.137 lr:1.3e-05 updt_s:0.159 data_s:0.110 +INFO 2025-05-07 15:53:41 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.17 loss:0.002 grdn:0.137 lr:1.3e-05 updt_s:0.158 data_s:0.107 +INFO 2025-05-07 15:54:34 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.22 loss:0.002 grdn:0.138 lr:1.3e-05 updt_s:0.158 data_s:0.108 +INFO 2025-05-07 15:55:27 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.28 loss:0.002 grdn:0.131 lr:1.3e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 15:56:14 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.33 loss:0.002 grdn:0.142 lr:1.3e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 15:57:00 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.39 loss:0.002 grdn:0.134 lr:1.3e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 15:57:45 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.44 loss:0.002 grdn:0.119 lr:1.3e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 15:58:30 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.50 loss:0.002 grdn:0.126 lr:1.3e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 15:59:17 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.55 loss:0.002 grdn:0.132 lr:1.3e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 16:00:02 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.61 loss:0.002 grdn:0.119 lr:1.3e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 16:00:48 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.66 loss:0.002 grdn:0.127 lr:1.3e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 16:01:33 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.72 loss:0.002 grdn:0.130 lr:1.3e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 16:02:18 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.77 loss:0.002 grdn:0.145 lr:1.3e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 16:03:04 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.83 loss:0.002 grdn:0.123 lr:1.3e-05 updt_s:0.160 data_s:0.071 +INFO 2025-05-07 16:03:50 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.88 loss:0.002 grdn:0.132 lr:1.3e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 16:04:38 ts/train.py:232 step:690K smpl:11M ep:57K epch:189.94 loss:0.002 grdn:0.131 lr:1.3e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 16:05:23 ts/train.py:232 step:690K smpl:11M ep:57K epch:189.99 loss:0.002 grdn:0.143 lr:1.3e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 16:06:08 ts/train.py:232 step:690K smpl:11M ep:57K epch:190.05 loss:0.002 grdn:0.126 lr:1.3e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 16:06:08 ts/train.py:241 Checkpoint policy after step 690000 +INFO 2025-05-07 16:06:56 ts/train.py:232 step:690K smpl:11M ep:57K epch:190.11 loss:0.002 grdn:0.151 lr:1.3e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 16:07:41 ts/train.py:232 step:690K smpl:11M ep:57K epch:190.16 loss:0.002 grdn:0.137 lr:1.3e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 16:08:27 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.22 loss:0.002 grdn:0.116 lr:1.3e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 16:09:12 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.27 loss:0.002 grdn:0.139 lr:1.3e-05 updt_s:0.160 data_s:0.068 +INFO 2025-05-07 16:09:57 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.33 loss:0.002 grdn:0.143 lr:1.3e-05 updt_s:0.160 data_s:0.063 +INFO 2025-05-07 16:10:42 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.38 loss:0.002 grdn:0.148 lr:1.3e-05 updt_s:0.160 data_s:0.061 +INFO 2025-05-07 16:11:26 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.44 loss:0.002 grdn:0.139 lr:1.3e-05 updt_s:0.159 data_s:0.063 +INFO 2025-05-07 16:12:11 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.49 loss:0.002 grdn:0.122 lr:1.3e-05 updt_s:0.160 data_s:0.060 +INFO 2025-05-07 16:12:57 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.55 loss:0.002 grdn:0.126 lr:1.3e-05 updt_s:0.161 data_s:0.069 +INFO 2025-05-07 16:13:43 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.60 loss:0.002 grdn:0.137 lr:1.3e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 16:14:27 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.66 loss:0.002 grdn:0.134 lr:1.3e-05 updt_s:0.159 data_s:0.062 +INFO 2025-05-07 16:15:12 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.71 loss:0.002 grdn:0.135 lr:1.3e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 16:15:57 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.77 loss:0.002 grdn:0.127 lr:1.3e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 16:16:42 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.82 loss:0.002 grdn:0.136 lr:1.3e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 16:17:26 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.88 loss:0.002 grdn:0.128 lr:1.3e-05 updt_s:0.160 data_s:0.061 +INFO 2025-05-07 16:18:16 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.93 loss:0.002 grdn:0.132 lr:1.2e-05 updt_s:0.160 data_s:0.091 +INFO 2025-05-07 16:19:06 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.99 loss:0.002 grdn:0.131 lr:1.2e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-07 16:19:54 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.04 loss:0.002 grdn:0.132 lr:1.2e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-07 16:20:42 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.10 loss:0.002 grdn:0.143 lr:1.2e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-07 16:21:28 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.15 loss:0.002 grdn:0.133 lr:1.2e-05 updt_s:0.160 data_s:0.070 +INFO 2025-05-07 16:22:14 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.21 loss:0.002 grdn:0.127 lr:1.2e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 16:23:01 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.26 loss:0.002 grdn:0.134 lr:1.2e-05 updt_s:0.161 data_s:0.075 +INFO 2025-05-07 16:23:45 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.32 loss:0.002 grdn:0.136 lr:1.2e-05 updt_s:0.159 data_s:0.062 +INFO 2025-05-07 16:24:31 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.37 loss:0.002 grdn:0.134 lr:1.2e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 16:25:16 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.43 loss:0.002 grdn:0.138 lr:1.2e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 16:26:02 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.48 loss:0.002 grdn:0.125 lr:1.2e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 16:26:47 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.54 loss:0.002 grdn:0.131 lr:1.2e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 16:27:32 ts/train.py:232 step:696K smpl:11M ep:57K epch:191.59 loss:0.002 grdn:0.129 lr:1.2e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 16:28:17 ts/train.py:232 step:696K smpl:11M ep:57K epch:191.65 loss:0.002 grdn:0.133 lr:1.2e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 16:28:59 ts/train.py:232 step:696K smpl:11M ep:58K epch:191.70 loss:0.002 grdn:0.131 lr:1.2e-05 updt_s:0.160 data_s:0.052 +INFO 2025-05-07 16:29:43 ts/train.py:232 step:696K smpl:11M ep:58K epch:191.76 loss:0.002 grdn:0.134 lr:1.2e-05 updt_s:0.160 data_s:0.058 +INFO 2025-05-07 16:30:28 ts/train.py:232 step:696K smpl:11M ep:58K epch:191.81 loss:0.002 grdn:0.137 lr:1.2e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 16:31:15 ts/train.py:232 step:697K smpl:11M ep:58K epch:191.87 loss:0.001 grdn:0.122 lr:1.2e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 16:32:01 ts/train.py:232 step:697K smpl:11M ep:58K epch:191.92 loss:0.002 grdn:0.133 lr:1.2e-05 updt_s:0.160 data_s:0.066 +INFO 2025-05-07 16:32:46 ts/train.py:232 step:697K smpl:11M ep:58K epch:191.98 loss:0.002 grdn:0.135 lr:1.2e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 16:33:31 ts/train.py:232 step:697K smpl:11M ep:58K epch:192.03 loss:0.002 grdn:0.138 lr:1.2e-05 updt_s:0.160 data_s:0.068 +INFO 2025-05-07 16:34:17 ts/train.py:232 step:697K smpl:11M ep:58K epch:192.09 loss:0.002 grdn:0.145 lr:1.2e-05 updt_s:0.160 data_s:0.068 +INFO 2025-05-07 16:35:02 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.14 loss:0.002 grdn:0.132 lr:1.2e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 16:35:48 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.20 loss:0.002 grdn:0.124 lr:1.2e-05 updt_s:0.160 data_s:0.067 +INFO 2025-05-07 16:36:32 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.25 loss:0.002 grdn:0.135 lr:1.2e-05 updt_s:0.159 data_s:0.063 +INFO 2025-05-07 16:37:17 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.31 loss:0.002 grdn:0.121 lr:1.2e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 16:38:02 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.36 loss:0.002 grdn:0.124 lr:1.2e-05 updt_s:0.159 data_s:0.062 +INFO 2025-05-07 16:38:46 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.42 loss:0.002 grdn:0.129 lr:1.2e-05 updt_s:0.159 data_s:0.060 +INFO 2025-05-07 16:39:32 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.47 loss:0.002 grdn:0.138 lr:1.2e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 16:40:16 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.53 loss:0.002 grdn:0.126 lr:1.2e-05 updt_s:0.160 data_s:0.063 +INFO 2025-05-07 16:41:01 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.58 loss:0.002 grdn:0.118 lr:1.2e-05 updt_s:0.160 data_s:0.062 +INFO 2025-05-07 16:41:45 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.64 loss:0.002 grdn:0.128 lr:1.2e-05 updt_s:0.159 data_s:0.060 +INFO 2025-05-07 16:42:30 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.69 loss:0.002 grdn:0.126 lr:1.2e-05 updt_s:0.160 data_s:0.063 +INFO 2025-05-07 16:43:13 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.75 loss:0.002 grdn:0.139 lr:1.2e-05 updt_s:0.160 data_s:0.058 +INFO 2025-05-07 16:43:57 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.80 loss:0.002 grdn:0.128 lr:1.2e-05 updt_s:0.159 data_s:0.060 +INFO 2025-05-07 16:44:52 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.86 loss:0.002 grdn:0.139 lr:1.2e-05 updt_s:0.159 data_s:0.116 +INFO 2025-05-07 16:45:45 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.91 loss:0.002 grdn:0.140 lr:1.2e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-07 16:46:39 ts/train.py:232 step:701K smpl:11M ep:58K epch:192.97 loss:0.002 grdn:0.135 lr:1.2e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-07 16:47:33 ts/train.py:232 step:701K smpl:11M ep:58K epch:193.02 loss:0.002 grdn:0.134 lr:1.2e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 16:48:26 ts/train.py:232 step:701K smpl:11M ep:58K epch:193.08 loss:0.002 grdn:0.129 lr:1.2e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 16:49:18 ts/train.py:232 step:701K smpl:11M ep:58K epch:193.13 loss:0.002 grdn:0.134 lr:1.2e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-07 16:50:04 ts/train.py:232 step:701K smpl:11M ep:58K epch:193.19 loss:0.002 grdn:0.135 lr:1.2e-05 updt_s:0.160 data_s:0.069 +INFO 2025-05-07 16:50:48 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.24 loss:0.002 grdn:0.133 lr:1.2e-05 updt_s:0.160 data_s:0.061 +INFO 2025-05-07 16:51:32 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.30 loss:0.002 grdn:0.137 lr:1.2e-05 updt_s:0.160 data_s:0.059 +INFO 2025-05-07 16:52:17 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.36 loss:0.002 grdn:0.137 lr:1.1e-05 updt_s:0.160 data_s:0.061 +INFO 2025-05-07 16:53:02 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.41 loss:0.002 grdn:0.143 lr:1.1e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 16:53:47 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.47 loss:0.001 grdn:0.124 lr:1.1e-05 updt_s:0.160 data_s:0.065 +INFO 2025-05-07 16:54:31 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.52 loss:0.002 grdn:0.143 lr:1.1e-05 updt_s:0.160 data_s:0.062 +INFO 2025-05-07 16:55:16 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.58 loss:0.002 grdn:0.138 lr:1.1e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 16:56:01 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.63 loss:0.002 grdn:0.130 lr:1.1e-05 updt_s:0.160 data_s:0.062 +INFO 2025-05-07 16:56:45 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.69 loss:0.002 grdn:0.126 lr:1.1e-05 updt_s:0.160 data_s:0.060 +INFO 2025-05-07 16:57:30 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.74 loss:0.002 grdn:0.119 lr:1.1e-05 updt_s:0.160 data_s:0.062 +INFO 2025-05-07 16:58:19 ts/train.py:232 step:704K smpl:11M ep:58K epch:193.80 loss:0.002 grdn:0.134 lr:1.1e-05 updt_s:0.160 data_s:0.088 +INFO 2025-05-07 16:59:10 ts/train.py:232 step:704K smpl:11M ep:58K epch:193.85 loss:0.002 grdn:0.140 lr:1.1e-05 updt_s:0.160 data_s:0.093 +INFO 2025-05-07 17:00:01 ts/train.py:232 step:704K smpl:11M ep:58K epch:193.91 loss:0.002 grdn:0.140 lr:1.1e-05 updt_s:0.160 data_s:0.093 +INFO 2025-05-07 17:00:50 ts/train.py:232 step:704K smpl:11M ep:58K epch:193.96 loss:0.002 grdn:0.124 lr:1.1e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-07 17:01:41 ts/train.py:232 step:704K smpl:11M ep:58K epch:194.02 loss:0.002 grdn:0.130 lr:1.1e-05 updt_s:0.159 data_s:0.092 +INFO 2025-05-07 17:02:31 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.07 loss:0.002 grdn:0.131 lr:1.1e-05 updt_s:0.159 data_s:0.093 +INFO 2025-05-07 17:03:20 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.13 loss:0.002 grdn:0.137 lr:1.1e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-07 17:04:05 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.18 loss:0.002 grdn:0.130 lr:1.1e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 17:04:50 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.24 loss:0.002 grdn:0.143 lr:1.1e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 17:05:35 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.29 loss:0.002 grdn:0.134 lr:1.1e-05 updt_s:0.160 data_s:0.063 +INFO 2025-05-07 17:06:19 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.35 loss:0.002 grdn:0.129 lr:1.1e-05 updt_s:0.160 data_s:0.058 +INFO 2025-05-07 17:07:04 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.40 loss:0.002 grdn:0.139 lr:1.1e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 17:07:48 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.46 loss:0.002 grdn:0.137 lr:1.1e-05 updt_s:0.159 data_s:0.060 +INFO 2025-05-07 17:08:33 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.51 loss:0.002 grdn:0.128 lr:1.1e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 17:09:18 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.57 loss:0.002 grdn:0.133 lr:1.1e-05 updt_s:0.160 data_s:0.067 +INFO 2025-05-07 17:10:03 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.62 loss:0.002 grdn:0.132 lr:1.1e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 17:10:48 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.68 loss:0.002 grdn:0.126 lr:1.1e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 17:11:34 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.73 loss:0.002 grdn:0.134 lr:1.1e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 17:12:25 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.79 loss:0.001 grdn:0.118 lr:1.1e-05 updt_s:0.159 data_s:0.096 +INFO 2025-05-07 17:13:16 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.84 loss:0.002 grdn:0.127 lr:1.1e-05 updt_s:0.159 data_s:0.096 +INFO 2025-05-07 17:14:07 ts/train.py:232 step:708K smpl:11M ep:58K epch:194.90 loss:0.002 grdn:0.134 lr:1.1e-05 updt_s:0.159 data_s:0.095 +INFO 2025-05-07 17:14:57 ts/train.py:232 step:708K smpl:11M ep:58K epch:194.95 loss:0.002 grdn:0.132 lr:1.1e-05 updt_s:0.159 data_s:0.092 +INFO 2025-05-07 17:15:46 ts/train.py:232 step:708K smpl:11M ep:59K epch:195.01 loss:0.002 grdn:0.143 lr:1.1e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-07 17:16:37 ts/train.py:232 step:708K smpl:11M ep:59K epch:195.06 loss:0.002 grdn:0.129 lr:1.1e-05 updt_s:0.159 data_s:0.092 +INFO 2025-05-07 17:17:21 ts/train.py:232 step:708K smpl:11M ep:59K epch:195.12 loss:0.002 grdn:0.143 lr:1.1e-05 updt_s:0.160 data_s:0.061 +INFO 2025-05-07 17:18:06 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.17 loss:0.002 grdn:0.140 lr:1.1e-05 updt_s:0.159 data_s:0.062 +INFO 2025-05-07 17:18:49 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.23 loss:0.002 grdn:0.130 lr:1.1e-05 updt_s:0.159 data_s:0.059 +INFO 2025-05-07 17:19:35 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.28 loss:0.002 grdn:0.127 lr:1.1e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 17:20:19 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.34 loss:0.002 grdn:0.127 lr:1.1e-05 updt_s:0.160 data_s:0.062 +INFO 2025-05-07 17:21:03 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.39 loss:0.002 grdn:0.131 lr:1.1e-05 updt_s:0.159 data_s:0.061 +INFO 2025-05-07 17:21:48 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.45 loss:0.002 grdn:0.130 lr:1.1e-05 updt_s:0.159 data_s:0.063 +INFO 2025-05-07 17:22:32 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.50 loss:0.002 grdn:0.138 lr:1.1e-05 updt_s:0.159 data_s:0.061 +INFO 2025-05-07 17:23:16 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.56 loss:0.002 grdn:0.126 lr:1.1e-05 updt_s:0.159 data_s:0.060 +INFO 2025-05-07 17:24:00 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.61 loss:0.002 grdn:0.130 lr:1.1e-05 updt_s:0.159 data_s:0.061 +INFO 2025-05-07 17:24:45 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.67 loss:0.002 grdn:0.136 lr:1.1e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 17:25:32 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.72 loss:0.001 grdn:0.120 lr:1.1e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 17:26:21 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.78 loss:0.001 grdn:0.124 lr:1.1e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-07 17:27:08 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.83 loss:0.002 grdn:0.142 lr:1.1e-05 updt_s:0.160 data_s:0.076 +INFO 2025-05-07 17:27:55 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.89 loss:0.002 grdn:0.151 lr:1.0e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 17:28:41 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.94 loss:0.002 grdn:0.130 lr:1.0e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 17:29:27 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.00 loss:0.002 grdn:0.138 lr:1.0e-05 updt_s:0.160 data_s:0.071 +INFO 2025-05-07 17:30:13 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.05 loss:0.002 grdn:0.138 lr:1.0e-05 updt_s:0.161 data_s:0.067 +INFO 2025-05-07 17:30:58 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.11 loss:0.002 grdn:0.142 lr:1.0e-05 updt_s:0.160 data_s:0.065 +INFO 2025-05-07 17:31:43 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.16 loss:0.002 grdn:0.141 lr:1.0e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 17:32:28 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.22 loss:0.002 grdn:0.144 lr:1.0e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 17:33:13 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.27 loss:0.002 grdn:0.125 lr:1.0e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 17:33:58 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.33 loss:0.001 grdn:0.126 lr:1.0e-05 updt_s:0.160 data_s:0.065 +INFO 2025-05-07 17:34:43 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.38 loss:0.002 grdn:0.140 lr:1.0e-05 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 17:35:29 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.44 loss:0.002 grdn:0.127 lr:1.0e-05 updt_s:0.160 data_s:0.070 +INFO 2025-05-07 17:36:15 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.50 loss:0.002 grdn:0.148 lr:1.0e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 17:37:00 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.55 loss:0.002 grdn:0.128 lr:1.0e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 17:37:46 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.61 loss:0.002 grdn:0.135 lr:1.0e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 17:38:32 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.66 loss:0.002 grdn:0.140 lr:1.0e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 17:39:28 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.72 loss:0.001 grdn:0.124 lr:1.0e-05 updt_s:0.160 data_s:0.120 +INFO 2025-05-07 17:40:23 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.77 loss:0.002 grdn:0.133 lr:1.0e-05 updt_s:0.159 data_s:0.113 +INFO 2025-05-07 17:41:17 ts/train.py:232 step:715K smpl:11M ep:59K epch:196.83 loss:0.002 grdn:0.143 lr:1.0e-05 updt_s:0.159 data_s:0.111 +INFO 2025-05-07 17:42:10 ts/train.py:232 step:715K smpl:11M ep:59K epch:196.88 loss:0.002 grdn:0.130 lr:1.0e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 17:43:02 ts/train.py:232 step:715K smpl:11M ep:59K epch:196.94 loss:0.002 grdn:0.137 lr:1.0e-05 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 17:43:56 ts/train.py:232 step:715K smpl:11M ep:59K epch:196.99 loss:0.001 grdn:0.127 lr:1.0e-05 updt_s:0.159 data_s:0.107 +INFO 2025-05-07 17:44:43 ts/train.py:232 step:715K smpl:11M ep:59K epch:197.05 loss:0.002 grdn:0.137 lr:1.0e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-07 17:45:30 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.10 loss:0.002 grdn:0.128 lr:1.0e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 17:46:15 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.16 loss:0.002 grdn:0.133 lr:1.0e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 17:47:01 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.21 loss:0.001 grdn:0.125 lr:1.0e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 17:47:46 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.27 loss:0.002 grdn:0.141 lr:1.0e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 17:48:33 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.32 loss:0.002 grdn:0.144 lr:9.9e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 17:49:19 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.38 loss:0.002 grdn:0.137 lr:9.9e-06 updt_s:0.160 data_s:0.070 +INFO 2025-05-07 17:50:06 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.43 loss:0.001 grdn:0.127 lr:9.9e-06 updt_s:0.160 data_s:0.072 +INFO 2025-05-07 17:50:52 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.49 loss:0.001 grdn:0.132 lr:9.9e-06 updt_s:0.160 data_s:0.069 +INFO 2025-05-07 17:51:38 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.54 loss:0.002 grdn:0.136 lr:9.9e-06 updt_s:0.160 data_s:0.071 +INFO 2025-05-07 17:52:24 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.60 loss:0.002 grdn:0.136 lr:9.8e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 17:53:16 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.65 loss:0.002 grdn:0.140 lr:9.8e-06 updt_s:0.159 data_s:0.098 +INFO 2025-05-07 17:54:08 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.71 loss:0.001 grdn:0.121 lr:9.8e-06 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 17:55:01 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.76 loss:0.001 grdn:0.119 lr:9.8e-06 updt_s:0.160 data_s:0.100 +INFO 2025-05-07 17:55:53 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.82 loss:0.002 grdn:0.133 lr:9.8e-06 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 17:56:45 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.87 loss:0.001 grdn:0.123 lr:9.7e-06 updt_s:0.160 data_s:0.100 +INFO 2025-05-07 17:57:37 ts/train.py:232 step:719K smpl:11M ep:59K epch:197.93 loss:0.002 grdn:0.145 lr:9.7e-06 updt_s:0.159 data_s:0.099 +INFO 2025-05-07 17:58:27 ts/train.py:232 step:719K smpl:12M ep:59K epch:197.98 loss:0.002 grdn:0.138 lr:9.7e-06 updt_s:0.159 data_s:0.092 +INFO 2025-05-07 17:59:14 ts/train.py:232 step:719K smpl:12M ep:59K epch:198.04 loss:0.002 grdn:0.129 lr:9.7e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 18:00:01 ts/train.py:232 step:719K smpl:12M ep:59K epch:198.09 loss:0.002 grdn:0.139 lr:9.7e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 18:00:47 ts/train.py:232 step:719K smpl:12M ep:59K epch:198.15 loss:0.002 grdn:0.136 lr:9.6e-06 updt_s:0.160 data_s:0.069 +INFO 2025-05-07 18:01:31 ts/train.py:232 step:720K smpl:12M ep:59K epch:198.20 loss:0.002 grdn:0.159 lr:9.6e-06 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 18:02:19 ts/train.py:232 step:720K smpl:12M ep:59K epch:198.26 loss:0.002 grdn:0.129 lr:9.6e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 18:03:09 ts/train.py:232 step:720K smpl:12M ep:59K epch:198.31 loss:0.002 grdn:0.139 lr:9.6e-06 updt_s:0.159 data_s:0.090 +INFO 2025-05-07 18:03:09 ts/train.py:241 Checkpoint policy after step 720000 +INFO 2025-05-07 18:04:00 ts/train.py:232 step:720K smpl:12M ep:60K epch:198.37 loss:0.002 grdn:0.141 lr:9.5e-06 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 18:04:47 ts/train.py:232 step:720K smpl:12M ep:60K epch:198.42 loss:0.002 grdn:0.133 lr:9.5e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 18:05:34 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.48 loss:0.001 grdn:0.124 lr:9.5e-06 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 18:06:20 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.53 loss:0.001 grdn:0.121 lr:9.5e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 18:07:07 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.59 loss:0.001 grdn:0.123 lr:9.5e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 18:08:01 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.64 loss:0.001 grdn:0.133 lr:9.4e-06 updt_s:0.158 data_s:0.111 +INFO 2025-05-07 18:08:52 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.70 loss:0.001 grdn:0.122 lr:9.4e-06 updt_s:0.158 data_s:0.099 +INFO 2025-05-07 18:09:42 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.75 loss:0.002 grdn:0.134 lr:9.4e-06 updt_s:0.158 data_s:0.090 +INFO 2025-05-07 18:10:33 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.81 loss:0.001 grdn:0.135 lr:9.4e-06 updt_s:0.158 data_s:0.096 +INFO 2025-05-07 18:11:25 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.86 loss:0.001 grdn:0.124 lr:9.4e-06 updt_s:0.158 data_s:0.102 +INFO 2025-05-07 18:12:16 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.92 loss:0.002 grdn:0.136 lr:9.3e-06 updt_s:0.158 data_s:0.096 +INFO 2025-05-07 18:13:03 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.97 loss:0.002 grdn:0.156 lr:9.3e-06 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 18:13:52 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.03 loss:0.002 grdn:0.138 lr:9.3e-06 updt_s:0.158 data_s:0.081 +INFO 2025-05-07 18:14:39 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.08 loss:0.002 grdn:0.140 lr:9.3e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 18:15:26 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.14 loss:0.002 grdn:0.128 lr:9.3e-06 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 18:16:13 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.19 loss:0.001 grdn:0.124 lr:9.2e-06 updt_s:0.159 data_s:0.077 +INFO 2025-05-07 18:16:59 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.25 loss:0.002 grdn:0.132 lr:9.2e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 18:17:46 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.30 loss:0.002 grdn:0.129 lr:9.2e-06 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 18:18:33 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.36 loss:0.001 grdn:0.118 lr:9.2e-06 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 18:19:20 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.41 loss:0.001 grdn:0.129 lr:9.2e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 18:20:05 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.47 loss:0.002 grdn:0.137 lr:9.1e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 18:20:52 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.52 loss:0.002 grdn:0.143 lr:9.1e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 18:21:42 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.58 loss:0.002 grdn:0.134 lr:9.1e-06 updt_s:0.158 data_s:0.092 +INFO 2025-05-07 18:22:30 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.64 loss:0.002 grdn:0.131 lr:9.1e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 18:23:19 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.69 loss:0.001 grdn:0.123 lr:9.1e-06 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 18:24:07 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.75 loss:0.002 grdn:0.149 lr:9.0e-06 updt_s:0.158 data_s:0.083 +INFO 2025-05-07 18:24:55 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.80 loss:0.001 grdn:0.124 lr:9.0e-06 updt_s:0.158 data_s:0.081 +INFO 2025-05-07 18:25:43 ts/train.py:232 step:726K smpl:12M ep:60K epch:199.86 loss:0.002 grdn:0.129 lr:9.0e-06 updt_s:0.158 data_s:0.079 +INFO 2025-05-07 18:26:30 ts/train.py:232 step:726K smpl:12M ep:60K epch:199.91 loss:0.002 grdn:0.133 lr:9.0e-06 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 18:27:15 ts/train.py:232 step:726K smpl:12M ep:60K epch:199.97 loss:0.002 grdn:0.139 lr:9.0e-06 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 18:28:00 ts/train.py:232 step:726K smpl:12M ep:60K epch:200.02 loss:0.001 grdn:0.126 lr:8.9e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 18:28:46 ts/train.py:232 step:726K smpl:12M ep:60K epch:200.08 loss:0.001 grdn:0.117 lr:8.9e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 18:29:32 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.13 loss:0.001 grdn:0.126 lr:8.9e-06 updt_s:0.158 data_s:0.069 +INFO 2025-05-07 18:30:17 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.19 loss:0.002 grdn:0.140 lr:8.9e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 18:31:02 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.24 loss:0.002 grdn:0.134 lr:8.9e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 18:31:47 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.30 loss:0.001 grdn:0.127 lr:8.8e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 18:32:33 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.35 loss:0.001 grdn:0.123 lr:8.8e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 18:33:18 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.41 loss:0.002 grdn:0.145 lr:8.8e-06 updt_s:0.158 data_s:0.068 +INFO 2025-05-07 18:34:04 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.46 loss:0.001 grdn:0.128 lr:8.8e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 18:34:50 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.52 loss:0.002 grdn:0.137 lr:8.8e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 18:35:46 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.57 loss:0.001 grdn:0.128 lr:8.7e-06 updt_s:0.158 data_s:0.122 +INFO 2025-05-07 18:36:40 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.63 loss:0.001 grdn:0.129 lr:8.7e-06 updt_s:0.158 data_s:0.113 +INFO 2025-05-07 18:37:33 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.68 loss:0.001 grdn:0.128 lr:8.7e-06 updt_s:0.158 data_s:0.105 +INFO 2025-05-07 18:38:28 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.74 loss:0.002 grdn:0.140 lr:8.7e-06 updt_s:0.159 data_s:0.116 +INFO 2025-05-07 18:39:23 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.79 loss:0.001 grdn:0.118 lr:8.7e-06 updt_s:0.158 data_s:0.116 +INFO 2025-05-07 18:40:16 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.85 loss:0.001 grdn:0.132 lr:8.6e-06 updt_s:0.158 data_s:0.108 +INFO 2025-05-07 18:41:05 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.90 loss:0.001 grdn:0.127 lr:8.6e-06 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 18:41:49 ts/train.py:232 step:730K smpl:12M ep:60K epch:200.96 loss:0.001 grdn:0.126 lr:8.6e-06 updt_s:0.158 data_s:0.065 +INFO 2025-05-07 18:42:34 ts/train.py:232 step:730K smpl:12M ep:60K epch:201.01 loss:0.001 grdn:0.128 lr:8.6e-06 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 18:43:19 ts/train.py:232 step:730K smpl:12M ep:60K epch:201.07 loss:0.001 grdn:0.129 lr:8.6e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 18:44:05 ts/train.py:232 step:730K smpl:12M ep:60K epch:201.12 loss:0.002 grdn:0.145 lr:8.5e-06 updt_s:0.158 data_s:0.069 +INFO 2025-05-07 18:44:50 ts/train.py:232 step:730K smpl:12M ep:60K epch:201.18 loss:0.002 grdn:0.135 lr:8.5e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 18:45:36 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.23 loss:0.002 grdn:0.144 lr:8.5e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 18:46:22 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.29 loss:0.001 grdn:0.127 lr:8.5e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 18:47:09 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.34 loss:0.001 grdn:0.127 lr:8.5e-06 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 18:47:55 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.40 loss:0.002 grdn:0.136 lr:8.5e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 18:48:42 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.45 loss:0.002 grdn:0.133 lr:8.4e-06 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 18:49:33 ts/train.py:232 step:732K smpl:12M ep:60K epch:201.51 loss:0.002 grdn:0.138 lr:8.4e-06 updt_s:0.157 data_s:0.097 +INFO 2025-05-07 18:50:24 ts/train.py:232 step:732K smpl:12M ep:60K epch:201.56 loss:0.001 grdn:0.128 lr:8.4e-06 updt_s:0.159 data_s:0.095 +INFO 2025-05-07 18:51:14 ts/train.py:232 step:732K smpl:12M ep:60K epch:201.62 loss:0.001 grdn:0.131 lr:8.4e-06 updt_s:0.159 data_s:0.091 +INFO 2025-05-07 18:52:05 ts/train.py:232 step:732K smpl:12M ep:61K epch:201.67 loss:0.001 grdn:0.133 lr:8.4e-06 updt_s:0.159 data_s:0.093 +INFO 2025-05-07 18:52:55 ts/train.py:232 step:732K smpl:12M ep:61K epch:201.73 loss:0.002 grdn:0.133 lr:8.3e-06 updt_s:0.159 data_s:0.092 +INFO 2025-05-07 18:53:45 ts/train.py:232 step:733K smpl:12M ep:61K epch:201.78 loss:0.002 grdn:0.140 lr:8.3e-06 updt_s:0.159 data_s:0.092 +INFO 2025-05-07 18:54:33 ts/train.py:232 step:733K smpl:12M ep:61K epch:201.84 loss:0.001 grdn:0.130 lr:8.3e-06 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 18:55:19 ts/train.py:232 step:733K smpl:12M ep:61K epch:201.89 loss:0.002 grdn:0.137 lr:8.3e-06 updt_s:0.158 data_s:0.068 +INFO 2025-05-07 18:56:08 ts/train.py:232 step:733K smpl:12M ep:61K epch:201.95 loss:0.002 grdn:0.137 lr:8.3e-06 updt_s:0.158 data_s:0.090 +INFO 2025-05-07 18:56:55 ts/train.py:232 step:733K smpl:12M ep:61K epch:202.00 loss:0.001 grdn:0.132 lr:8.2e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 18:57:42 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.06 loss:0.001 grdn:0.133 lr:8.2e-06 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 18:58:28 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.11 loss:0.002 grdn:0.152 lr:8.2e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 18:59:15 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.17 loss:0.001 grdn:0.133 lr:8.2e-06 updt_s:0.159 data_s:0.076 +INFO 2025-05-07 19:00:01 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.22 loss:0.002 grdn:0.143 lr:8.2e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 19:00:48 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.28 loss:0.002 grdn:0.137 lr:8.1e-06 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 19:01:34 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.33 loss:0.002 grdn:0.132 lr:8.1e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 19:02:20 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.39 loss:0.002 grdn:0.134 lr:8.1e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 19:03:07 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.44 loss:0.001 grdn:0.127 lr:8.1e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 19:04:02 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.50 loss:0.001 grdn:0.133 lr:8.1e-06 updt_s:0.158 data_s:0.118 +INFO 2025-05-07 19:04:56 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.55 loss:0.001 grdn:0.129 lr:8.0e-06 updt_s:0.158 data_s:0.110 +INFO 2025-05-07 19:05:50 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.61 loss:0.001 grdn:0.128 lr:8.0e-06 updt_s:0.158 data_s:0.112 +INFO 2025-05-07 19:06:43 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.66 loss:0.001 grdn:0.122 lr:8.0e-06 updt_s:0.158 data_s:0.104 +INFO 2025-05-07 19:07:37 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.72 loss:0.002 grdn:0.136 lr:8.0e-06 updt_s:0.158 data_s:0.112 +INFO 2025-05-07 19:08:30 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.78 loss:0.001 grdn:0.134 lr:8.0e-06 updt_s:0.158 data_s:0.110 +INFO 2025-05-07 19:09:17 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.83 loss:0.002 grdn:0.144 lr:8.0e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 19:10:03 ts/train.py:232 step:737K smpl:12M ep:61K epch:202.89 loss:0.001 grdn:0.133 lr:7.9e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 19:10:50 ts/train.py:232 step:737K smpl:12M ep:61K epch:202.94 loss:0.002 grdn:0.137 lr:7.9e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 19:11:36 ts/train.py:232 step:737K smpl:12M ep:61K epch:203.00 loss:0.001 grdn:0.130 lr:7.9e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 19:12:23 ts/train.py:232 step:737K smpl:12M ep:61K epch:203.05 loss:0.001 grdn:0.125 lr:7.9e-06 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 19:13:09 ts/train.py:232 step:737K smpl:12M ep:61K epch:203.11 loss:0.001 grdn:0.124 lr:7.9e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 19:13:55 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.16 loss:0.001 grdn:0.135 lr:7.8e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 19:14:41 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.22 loss:0.001 grdn:0.124 lr:7.8e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 19:15:27 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.27 loss:0.001 grdn:0.123 lr:7.8e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 19:16:12 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.33 loss:0.002 grdn:0.142 lr:7.8e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 19:16:58 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.38 loss:0.002 grdn:0.141 lr:7.8e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 19:17:50 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.44 loss:0.001 grdn:0.132 lr:7.7e-06 updt_s:0.158 data_s:0.099 +INFO 2025-05-07 19:18:43 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.49 loss:0.002 grdn:0.140 lr:7.7e-06 updt_s:0.158 data_s:0.110 +INFO 2025-05-07 19:19:37 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.55 loss:0.002 grdn:0.136 lr:7.7e-06 updt_s:0.158 data_s:0.108 +INFO 2025-05-07 19:20:30 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.60 loss:0.001 grdn:0.130 lr:7.7e-06 updt_s:0.159 data_s:0.107 +INFO 2025-05-07 19:21:24 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.66 loss:0.001 grdn:0.127 lr:7.7e-06 updt_s:0.158 data_s:0.111 +INFO 2025-05-07 19:22:18 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.71 loss:0.001 grdn:0.130 lr:7.7e-06 updt_s:0.158 data_s:0.110 +INFO 2025-05-07 19:23:11 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.77 loss:0.002 grdn:0.145 lr:7.6e-06 updt_s:0.158 data_s:0.106 +INFO 2025-05-07 19:23:57 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.82 loss:0.001 grdn:0.125 lr:7.6e-06 updt_s:0.160 data_s:0.070 +INFO 2025-05-07 19:24:43 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.88 loss:0.002 grdn:0.134 lr:7.6e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 19:25:29 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.93 loss:0.002 grdn:0.143 lr:7.6e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 19:26:14 ts/train.py:232 step:741K smpl:12M ep:61K epch:203.99 loss:0.001 grdn:0.131 lr:7.6e-06 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 19:27:00 ts/train.py:232 step:741K smpl:12M ep:61K epch:204.04 loss:0.001 grdn:0.133 lr:7.5e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 19:27:45 ts/train.py:232 step:741K smpl:12M ep:61K epch:204.10 loss:0.001 grdn:0.118 lr:7.5e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 19:28:31 ts/train.py:232 step:741K smpl:12M ep:61K epch:204.15 loss:0.001 grdn:0.135 lr:7.5e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 19:29:16 ts/train.py:232 step:741K smpl:12M ep:61K epch:204.21 loss:0.001 grdn:0.130 lr:7.5e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 19:30:02 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.26 loss:0.001 grdn:0.136 lr:7.5e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 19:30:47 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.32 loss:0.001 grdn:0.127 lr:7.4e-06 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 19:31:32 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.37 loss:0.001 grdn:0.115 lr:7.4e-06 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 19:32:27 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.43 loss:0.001 grdn:0.125 lr:7.4e-06 updt_s:0.158 data_s:0.115 +INFO 2025-05-07 19:33:20 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.48 loss:0.002 grdn:0.142 lr:7.4e-06 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 19:34:12 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.54 loss:0.001 grdn:0.126 lr:7.4e-06 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 19:35:05 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.59 loss:0.001 grdn:0.133 lr:7.4e-06 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 19:35:58 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.65 loss:0.001 grdn:0.129 lr:7.3e-06 updt_s:0.160 data_s:0.103 +INFO 2025-05-07 19:36:51 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.70 loss:0.001 grdn:0.136 lr:7.3e-06 updt_s:0.159 data_s:0.107 +INFO 2025-05-07 19:37:37 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.76 loss:0.001 grdn:0.128 lr:7.3e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 19:38:22 ts/train.py:232 step:744K smpl:12M ep:61K epch:204.81 loss:0.001 grdn:0.131 lr:7.3e-06 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 19:39:07 ts/train.py:232 step:744K smpl:12M ep:61K epch:204.87 loss:0.001 grdn:0.132 lr:7.3e-06 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 19:39:52 ts/train.py:232 step:744K smpl:12M ep:61K epch:204.92 loss:0.001 grdn:0.131 lr:7.2e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 19:40:37 ts/train.py:232 step:744K smpl:12M ep:61K epch:204.98 loss:0.001 grdn:0.124 lr:7.2e-06 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 19:41:22 ts/train.py:232 step:744K smpl:12M ep:62K epch:205.03 loss:0.001 grdn:0.118 lr:7.2e-06 updt_s:0.158 data_s:0.065 +INFO 2025-05-07 19:42:07 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.09 loss:0.001 grdn:0.132 lr:7.2e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 19:42:52 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.14 loss:0.001 grdn:0.123 lr:7.2e-06 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 19:43:38 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.20 loss:0.001 grdn:0.133 lr:7.2e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 19:44:26 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.25 loss:0.001 grdn:0.119 lr:7.1e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 19:45:12 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.31 loss:0.001 grdn:0.134 lr:7.1e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 19:46:01 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.36 loss:0.002 grdn:0.141 lr:7.1e-06 updt_s:0.159 data_s:0.088 +INFO 2025-05-07 19:46:48 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.42 loss:0.001 grdn:0.124 lr:7.1e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 19:47:36 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.47 loss:0.002 grdn:0.144 lr:7.1e-06 updt_s:0.158 data_s:0.081 +INFO 2025-05-07 19:48:26 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.53 loss:0.001 grdn:0.138 lr:7.1e-06 updt_s:0.159 data_s:0.090 +INFO 2025-05-07 19:49:15 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.58 loss:0.001 grdn:0.120 lr:7.0e-06 updt_s:0.159 data_s:0.086 +INFO 2025-05-07 19:50:05 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.64 loss:0.001 grdn:0.116 lr:7.0e-06 updt_s:0.159 data_s:0.088 +INFO 2025-05-07 19:50:54 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.69 loss:0.001 grdn:0.130 lr:7.0e-06 updt_s:0.159 data_s:0.085 +INFO 2025-05-07 19:51:40 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.75 loss:0.001 grdn:0.126 lr:7.0e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 19:52:28 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.80 loss:0.001 grdn:0.119 lr:7.0e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 19:53:16 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.86 loss:0.001 grdn:0.141 lr:6.9e-06 updt_s:0.158 data_s:0.083 +INFO 2025-05-07 19:54:03 ts/train.py:232 step:748K smpl:12M ep:62K epch:205.91 loss:0.001 grdn:0.130 lr:6.9e-06 updt_s:0.159 data_s:0.077 +INFO 2025-05-07 19:54:50 ts/train.py:232 step:748K smpl:12M ep:62K epch:205.97 loss:0.001 grdn:0.110 lr:6.9e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 19:55:39 ts/train.py:232 step:748K smpl:12M ep:62K epch:206.03 loss:0.001 grdn:0.115 lr:6.9e-06 updt_s:0.159 data_s:0.085 +INFO 2025-05-07 19:56:28 ts/train.py:232 step:748K smpl:12M ep:62K epch:206.08 loss:0.001 grdn:0.127 lr:6.9e-06 updt_s:0.159 data_s:0.086 +INFO 2025-05-07 19:57:14 ts/train.py:232 step:748K smpl:12M ep:62K epch:206.14 loss:0.002 grdn:0.134 lr:6.9e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 19:58:02 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.19 loss:0.001 grdn:0.125 lr:6.8e-06 updt_s:0.159 data_s:0.080 +INFO 2025-05-07 19:58:48 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.25 loss:0.001 grdn:0.134 lr:6.8e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 19:59:36 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.30 loss:0.001 grdn:0.128 lr:6.8e-06 updt_s:0.158 data_s:0.079 +INFO 2025-05-07 20:00:26 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.36 loss:0.001 grdn:0.118 lr:6.8e-06 updt_s:0.159 data_s:0.092 +INFO 2025-05-07 20:01:14 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.41 loss:0.001 grdn:0.121 lr:6.8e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 20:02:05 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.47 loss:0.001 grdn:0.128 lr:6.7e-06 updt_s:0.159 data_s:0.092 +INFO 2025-05-07 20:02:55 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.52 loss:0.001 grdn:0.126 lr:6.7e-06 updt_s:0.159 data_s:0.091 +INFO 2025-05-07 20:03:43 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.58 loss:0.001 grdn:0.126 lr:6.7e-06 updt_s:0.159 data_s:0.082 +INFO 2025-05-07 20:03:43 ts/train.py:241 Checkpoint policy after step 750000 +INFO 2025-05-07 20:04:36 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.63 loss:0.002 grdn:0.141 lr:6.7e-06 updt_s:0.159 data_s:0.087 +INFO 2025-05-07 20:05:21 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.69 loss:0.001 grdn:0.132 lr:6.7e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 20:06:08 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.74 loss:0.001 grdn:0.128 lr:6.7e-06 updt_s:0.160 data_s:0.073 +INFO 2025-05-07 20:06:54 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.80 loss:0.002 grdn:0.140 lr:6.6e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 20:07:40 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.85 loss:0.001 grdn:0.118 lr:6.6e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 20:08:26 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.91 loss:0.001 grdn:0.122 lr:6.6e-06 updt_s:0.160 data_s:0.069 +INFO 2025-05-07 20:09:13 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.96 loss:0.001 grdn:0.120 lr:6.6e-06 updt_s:0.160 data_s:0.072 +INFO 2025-05-07 20:09:59 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.02 loss:0.001 grdn:0.118 lr:6.6e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 20:10:47 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.07 loss:0.001 grdn:0.126 lr:6.6e-06 updt_s:0.159 data_s:0.078 +INFO 2025-05-07 20:11:35 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.13 loss:0.001 grdn:0.118 lr:6.5e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 20:12:23 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.18 loss:0.001 grdn:0.129 lr:6.5e-06 updt_s:0.159 data_s:0.080 +INFO 2025-05-07 20:13:10 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.24 loss:0.001 grdn:0.132 lr:6.5e-06 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 20:13:58 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.29 loss:0.001 grdn:0.112 lr:6.5e-06 updt_s:0.159 data_s:0.081 +INFO 2025-05-07 20:14:44 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.35 loss:0.001 grdn:0.133 lr:6.5e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 20:15:31 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.40 loss:0.001 grdn:0.127 lr:6.5e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 20:16:17 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.46 loss:0.002 grdn:0.135 lr:6.4e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 20:17:04 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.51 loss:0.001 grdn:0.128 lr:6.4e-06 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 20:17:51 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.57 loss:0.001 grdn:0.120 lr:6.4e-06 updt_s:0.160 data_s:0.075 +INFO 2025-05-07 20:18:38 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.62 loss:0.001 grdn:0.119 lr:6.4e-06 updt_s:0.159 data_s:0.077 +INFO 2025-05-07 20:19:24 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.68 loss:0.001 grdn:0.127 lr:6.4e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 20:20:11 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.73 loss:0.001 grdn:0.129 lr:6.4e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 20:20:57 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.79 loss:0.001 grdn:0.121 lr:6.3e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 20:21:44 ts/train.py:232 step:755K smpl:12M ep:62K epch:207.84 loss:0.001 grdn:0.130 lr:6.3e-06 updt_s:0.159 data_s:0.076 +INFO 2025-05-07 20:22:30 ts/train.py:232 step:755K smpl:12M ep:62K epch:207.90 loss:0.001 grdn:0.110 lr:6.3e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 20:23:18 ts/train.py:232 step:755K smpl:12M ep:62K epch:207.95 loss:0.001 grdn:0.128 lr:6.3e-06 updt_s:0.159 data_s:0.080 +INFO 2025-05-07 20:24:04 ts/train.py:232 step:755K smpl:12M ep:62K epch:208.01 loss:0.001 grdn:0.131 lr:6.3e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 20:24:52 ts/train.py:232 step:755K smpl:12M ep:62K epch:208.06 loss:0.001 grdn:0.138 lr:6.3e-06 updt_s:0.159 data_s:0.076 +INFO 2025-05-07 20:25:38 ts/train.py:232 step:756K smpl:12M ep:62K epch:208.12 loss:0.001 grdn:0.134 lr:6.2e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 20:26:24 ts/train.py:232 step:756K smpl:12M ep:62K epch:208.17 loss:0.001 grdn:0.129 lr:6.2e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 20:27:09 ts/train.py:232 step:756K smpl:12M ep:62K epch:208.23 loss:0.001 grdn:0.126 lr:6.2e-06 updt_s:0.160 data_s:0.066 +INFO 2025-05-07 20:28:04 ts/train.py:232 step:756K smpl:12M ep:62K epch:208.28 loss:0.001 grdn:0.121 lr:6.2e-06 updt_s:0.159 data_s:0.116 +INFO 2025-05-07 20:28:57 ts/train.py:232 step:756K smpl:12M ep:63K epch:208.34 loss:0.001 grdn:0.116 lr:6.2e-06 updt_s:0.159 data_s:0.107 +INFO 2025-05-07 20:29:50 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.39 loss:0.001 grdn:0.130 lr:6.1e-06 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 20:30:43 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.45 loss:0.001 grdn:0.139 lr:6.1e-06 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 20:31:35 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.50 loss:0.001 grdn:0.128 lr:6.1e-06 updt_s:0.159 data_s:0.102 +INFO 2025-05-07 20:32:28 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.56 loss:0.001 grdn:0.127 lr:6.1e-06 updt_s:0.160 data_s:0.104 +INFO 2025-05-07 20:33:14 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.61 loss:0.001 grdn:0.118 lr:6.1e-06 updt_s:0.160 data_s:0.070 +INFO 2025-05-07 20:34:00 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.67 loss:0.001 grdn:0.115 lr:6.1e-06 updt_s:0.160 data_s:0.066 +INFO 2025-05-07 20:34:45 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.72 loss:0.001 grdn:0.115 lr:6.0e-06 updt_s:0.160 data_s:0.068 +INFO 2025-05-07 20:35:31 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.78 loss:0.001 grdn:0.128 lr:6.0e-06 updt_s:0.160 data_s:0.065 +INFO 2025-05-07 20:36:15 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.83 loss:0.001 grdn:0.124 lr:6.0e-06 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 20:37:01 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.89 loss:0.001 grdn:0.122 lr:6.0e-06 updt_s:0.160 data_s:0.065 +INFO 2025-05-07 20:37:46 ts/train.py:232 step:759K smpl:12M ep:63K epch:208.94 loss:0.001 grdn:0.123 lr:6.0e-06 updt_s:0.161 data_s:0.068 +INFO 2025-05-07 20:38:32 ts/train.py:232 step:759K smpl:12M ep:63K epch:209.00 loss:0.001 grdn:0.134 lr:6.0e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 20:39:17 ts/train.py:232 step:759K smpl:12M ep:63K epch:209.05 loss:0.001 grdn:0.136 lr:5.9e-06 updt_s:0.160 data_s:0.066 +INFO 2025-05-07 20:40:02 ts/train.py:232 step:759K smpl:12M ep:63K epch:209.11 loss:0.001 grdn:0.126 lr:5.9e-06 updt_s:0.159 data_s:0.063 +INFO 2025-05-07 20:40:46 ts/train.py:232 step:759K smpl:12M ep:63K epch:209.17 loss:0.001 grdn:0.136 lr:5.9e-06 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 20:41:34 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.22 loss:0.001 grdn:0.127 lr:5.9e-06 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 20:42:21 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.28 loss:0.001 grdn:0.132 lr:5.9e-06 updt_s:0.160 data_s:0.076 +INFO 2025-05-07 20:43:08 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.33 loss:0.001 grdn:0.142 lr:5.9e-06 updt_s:0.160 data_s:0.077 +INFO 2025-05-07 20:43:57 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.39 loss:0.001 grdn:0.135 lr:5.9e-06 updt_s:0.160 data_s:0.081 +INFO 2025-05-07 20:44:45 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.44 loss:0.001 grdn:0.126 lr:5.8e-06 updt_s:0.160 data_s:0.079 +INFO 2025-05-07 20:45:32 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.50 loss:0.001 grdn:0.129 lr:5.8e-06 updt_s:0.160 data_s:0.076 +INFO 2025-05-07 20:46:18 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.55 loss:0.001 grdn:0.126 lr:5.8e-06 updt_s:0.160 data_s:0.068 +INFO 2025-05-07 20:47:03 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.61 loss:0.001 grdn:0.125 lr:5.8e-06 updt_s:0.161 data_s:0.064 +INFO 2025-05-07 20:47:48 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.66 loss:0.001 grdn:0.128 lr:5.8e-06 updt_s:0.160 data_s:0.065 +INFO 2025-05-07 20:48:33 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.72 loss:0.001 grdn:0.141 lr:5.8e-06 updt_s:0.160 data_s:0.067 +INFO 2025-05-07 20:49:19 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.77 loss:0.001 grdn:0.127 lr:5.7e-06 updt_s:0.160 data_s:0.069 +INFO 2025-05-07 20:50:05 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.83 loss:0.001 grdn:0.101 lr:5.7e-06 updt_s:0.160 data_s:0.067 +INFO 2025-05-07 20:50:51 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.88 loss:0.001 grdn:0.126 lr:5.7e-06 updt_s:0.160 data_s:0.067 +INFO 2025-05-07 20:51:37 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.94 loss:0.001 grdn:0.119 lr:5.7e-06 updt_s:0.160 data_s:0.073 +INFO 2025-05-07 20:52:24 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.99 loss:0.001 grdn:0.134 lr:5.7e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 20:53:10 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.05 loss:0.001 grdn:0.126 lr:5.7e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 20:53:57 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.10 loss:0.001 grdn:0.127 lr:5.6e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 20:54:42 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.16 loss:0.001 grdn:0.126 lr:5.6e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 20:55:37 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.21 loss:0.001 grdn:0.123 lr:5.6e-06 updt_s:0.159 data_s:0.113 +INFO 2025-05-07 20:56:30 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.27 loss:0.001 grdn:0.140 lr:5.6e-06 updt_s:0.160 data_s:0.106 +INFO 2025-05-07 20:57:23 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.32 loss:0.001 grdn:0.128 lr:5.6e-06 updt_s:0.160 data_s:0.105 +INFO 2025-05-07 20:58:16 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.38 loss:0.001 grdn:0.125 lr:5.6e-06 updt_s:0.160 data_s:0.104 +INFO 2025-05-07 20:59:09 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.43 loss:0.001 grdn:0.119 lr:5.5e-06 updt_s:0.159 data_s:0.103 +INFO 2025-05-07 21:00:01 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.49 loss:0.001 grdn:0.131 lr:5.5e-06 updt_s:0.159 data_s:0.102 +INFO 2025-05-07 21:00:48 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.54 loss:0.001 grdn:0.128 lr:5.5e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 21:01:34 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.60 loss:0.001 grdn:0.144 lr:5.5e-06 updt_s:0.160 data_s:0.067 +INFO 2025-05-07 21:02:21 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.65 loss:0.001 grdn:0.123 lr:5.5e-06 updt_s:0.159 data_s:0.077 +INFO 2025-05-07 21:03:06 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.71 loss:0.001 grdn:0.122 lr:5.5e-06 updt_s:0.160 data_s:0.066 +INFO 2025-05-07 21:03:52 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.76 loss:0.001 grdn:0.110 lr:5.4e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 21:04:37 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.82 loss:0.001 grdn:0.117 lr:5.4e-06 updt_s:0.160 data_s:0.065 +INFO 2025-05-07 21:05:25 ts/train.py:232 step:766K smpl:12M ep:63K epch:210.87 loss:0.001 grdn:0.127 lr:5.4e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 21:06:13 ts/train.py:232 step:766K smpl:12M ep:63K epch:210.93 loss:0.001 grdn:0.121 lr:5.4e-06 updt_s:0.160 data_s:0.080 +INFO 2025-05-07 21:06:59 ts/train.py:232 step:766K smpl:12M ep:63K epch:210.98 loss:0.001 grdn:0.122 lr:5.4e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 21:07:43 ts/train.py:232 step:766K smpl:12M ep:63K epch:211.04 loss:0.001 grdn:0.123 lr:5.4e-06 updt_s:0.160 data_s:0.062 +INFO 2025-05-07 21:08:28 ts/train.py:232 step:766K smpl:12M ep:63K epch:211.09 loss:0.001 grdn:0.136 lr:5.4e-06 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 21:09:19 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.15 loss:0.001 grdn:0.136 lr:5.3e-06 updt_s:0.159 data_s:0.097 +INFO 2025-05-07 21:10:13 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.20 loss:0.001 grdn:0.123 lr:5.3e-06 updt_s:0.159 data_s:0.109 +INFO 2025-05-07 21:11:06 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.26 loss:0.001 grdn:0.123 lr:5.3e-06 updt_s:0.160 data_s:0.101 +INFO 2025-05-07 21:11:59 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.31 loss:0.001 grdn:0.119 lr:5.3e-06 updt_s:0.159 data_s:0.109 +INFO 2025-05-07 21:12:54 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.37 loss:0.001 grdn:0.126 lr:5.3e-06 updt_s:0.160 data_s:0.111 +INFO 2025-05-07 21:13:48 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.42 loss:0.001 grdn:0.116 lr:5.3e-06 updt_s:0.159 data_s:0.111 +INFO 2025-05-07 21:14:39 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.48 loss:0.001 grdn:0.126 lr:5.2e-06 updt_s:0.160 data_s:0.094 +INFO 2025-05-07 21:15:25 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.53 loss:0.001 grdn:0.132 lr:5.2e-06 updt_s:0.160 data_s:0.071 +INFO 2025-05-07 21:16:13 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.59 loss:0.001 grdn:0.120 lr:5.2e-06 updt_s:0.160 data_s:0.078 +INFO 2025-05-07 21:17:01 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.64 loss:0.001 grdn:0.135 lr:5.2e-06 updt_s:0.160 data_s:0.082 +INFO 2025-05-07 21:17:48 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.70 loss:0.001 grdn:0.130 lr:5.2e-06 updt_s:0.160 data_s:0.072 +INFO 2025-05-07 21:18:34 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.75 loss:0.001 grdn:0.120 lr:5.2e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 21:19:21 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.81 loss:0.001 grdn:0.133 lr:5.2e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 21:20:07 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.86 loss:0.001 grdn:0.142 lr:5.1e-06 updt_s:0.160 data_s:0.068 +INFO 2025-05-07 21:20:52 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.92 loss:0.001 grdn:0.113 lr:5.1e-06 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 21:21:37 ts/train.py:232 step:770K smpl:12M ep:64K epch:211.97 loss:0.001 grdn:0.125 lr:5.1e-06 updt_s:0.160 data_s:0.064 +INFO 2025-05-07 21:22:22 ts/train.py:232 step:770K smpl:12M ep:64K epch:212.03 loss:0.001 grdn:0.130 lr:5.1e-06 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 21:23:09 ts/train.py:232 step:770K smpl:12M ep:64K epch:212.08 loss:0.001 grdn:0.137 lr:5.1e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 21:24:04 ts/train.py:232 step:770K smpl:12M ep:64K epch:212.14 loss:0.001 grdn:0.126 lr:5.1e-06 updt_s:0.160 data_s:0.115 +INFO 2025-05-07 21:24:57 ts/train.py:232 step:770K smpl:12M ep:64K epch:212.19 loss:0.001 grdn:0.132 lr:5.0e-06 updt_s:0.160 data_s:0.103 +INFO 2025-05-07 21:25:50 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.25 loss:0.001 grdn:0.118 lr:5.0e-06 updt_s:0.160 data_s:0.105 +INFO 2025-05-07 21:26:43 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.31 loss:0.001 grdn:0.123 lr:5.0e-06 updt_s:0.160 data_s:0.104 +INFO 2025-05-07 21:27:37 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.36 loss:0.001 grdn:0.126 lr:5.0e-06 updt_s:0.160 data_s:0.106 +INFO 2025-05-07 21:28:30 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.42 loss:0.001 grdn:0.113 lr:5.0e-06 updt_s:0.160 data_s:0.104 +INFO 2025-05-07 21:29:28 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.47 loss:0.001 grdn:0.129 lr:5.0e-06 updt_s:0.160 data_s:0.134 +INFO 2025-05-07 21:30:29 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.53 loss:0.001 grdn:0.131 lr:5.0e-06 updt_s:0.159 data_s:0.141 +INFO 2025-05-07 21:31:29 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.58 loss:0.001 grdn:0.122 lr:4.9e-06 updt_s:0.160 data_s:0.140 +INFO 2025-05-07 21:32:29 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.64 loss:0.001 grdn:0.115 lr:4.9e-06 updt_s:0.160 data_s:0.143 +INFO 2025-05-07 21:33:30 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.69 loss:0.001 grdn:0.120 lr:4.9e-06 updt_s:0.159 data_s:0.142 +INFO 2025-05-07 21:34:31 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.75 loss:0.001 grdn:0.126 lr:4.9e-06 updt_s:0.159 data_s:0.146 +INFO 2025-05-07 21:35:32 ts/train.py:232 step:773K smpl:12M ep:64K epch:212.80 loss:0.001 grdn:0.115 lr:4.9e-06 updt_s:0.159 data_s:0.143 +INFO 2025-05-07 21:36:32 ts/train.py:232 step:773K smpl:12M ep:64K epch:212.86 loss:0.002 grdn:0.151 lr:4.9e-06 updt_s:0.160 data_s:0.143 +INFO 2025-05-07 21:37:32 ts/train.py:232 step:773K smpl:12M ep:64K epch:212.91 loss:0.001 grdn:0.118 lr:4.8e-06 updt_s:0.159 data_s:0.138 +INFO 2025-05-07 21:38:32 ts/train.py:232 step:773K smpl:12M ep:64K epch:212.97 loss:0.001 grdn:0.127 lr:4.8e-06 updt_s:0.159 data_s:0.140 +INFO 2025-05-07 21:39:31 ts/train.py:232 step:773K smpl:12M ep:64K epch:213.02 loss:0.001 grdn:0.124 lr:4.8e-06 updt_s:0.159 data_s:0.135 +INFO 2025-05-07 21:40:27 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.08 loss:0.001 grdn:0.120 lr:4.8e-06 updt_s:0.159 data_s:0.119 +INFO 2025-05-07 21:41:13 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.13 loss:0.001 grdn:0.118 lr:4.8e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-07 21:42:01 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.19 loss:0.001 grdn:0.130 lr:4.8e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 21:42:49 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.24 loss:0.001 grdn:0.119 lr:4.8e-06 updt_s:0.159 data_s:0.082 +INFO 2025-05-07 21:43:37 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.30 loss:0.001 grdn:0.118 lr:4.7e-06 updt_s:0.159 data_s:0.078 +INFO 2025-05-07 21:44:25 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.35 loss:0.001 grdn:0.127 lr:4.7e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 21:45:11 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.41 loss:0.001 grdn:0.130 lr:4.7e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 21:45:58 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.46 loss:0.001 grdn:0.135 lr:4.7e-06 updt_s:0.159 data_s:0.076 +INFO 2025-05-07 21:46:45 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.52 loss:0.001 grdn:0.140 lr:4.7e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 21:47:31 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.57 loss:0.001 grdn:0.114 lr:4.7e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 21:48:18 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.63 loss:0.001 grdn:0.130 lr:4.7e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 21:49:04 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.68 loss:0.001 grdn:0.127 lr:4.6e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 21:49:50 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.74 loss:0.001 grdn:0.127 lr:4.6e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 21:50:37 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.79 loss:0.001 grdn:0.123 lr:4.6e-06 updt_s:0.160 data_s:0.072 +INFO 2025-05-07 21:51:23 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.85 loss:0.001 grdn:0.128 lr:4.6e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 21:52:09 ts/train.py:232 step:777K smpl:12M ep:64K epch:213.90 loss:0.001 grdn:0.113 lr:4.6e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 21:52:55 ts/train.py:232 step:777K smpl:12M ep:64K epch:213.96 loss:0.001 grdn:0.120 lr:4.6e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 21:53:41 ts/train.py:232 step:777K smpl:12M ep:64K epch:214.01 loss:0.001 grdn:0.123 lr:4.6e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 21:54:35 ts/train.py:232 step:777K smpl:12M ep:64K epch:214.07 loss:0.001 grdn:0.124 lr:4.5e-06 updt_s:0.160 data_s:0.111 +INFO 2025-05-07 21:55:29 ts/train.py:232 step:777K smpl:12M ep:64K epch:214.12 loss:0.001 grdn:0.127 lr:4.5e-06 updt_s:0.159 data_s:0.108 +INFO 2025-05-07 21:56:22 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.18 loss:0.001 grdn:0.132 lr:4.5e-06 updt_s:0.159 data_s:0.107 +INFO 2025-05-07 21:57:15 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.23 loss:0.001 grdn:0.128 lr:4.5e-06 updt_s:0.159 data_s:0.106 +INFO 2025-05-07 21:58:08 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.29 loss:0.001 grdn:0.118 lr:4.5e-06 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 21:59:02 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.34 loss:0.001 grdn:0.118 lr:4.5e-06 updt_s:0.159 data_s:0.108 +INFO 2025-05-07 21:59:52 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.40 loss:0.001 grdn:0.120 lr:4.4e-06 updt_s:0.159 data_s:0.089 +INFO 2025-05-07 22:00:38 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.45 loss:0.001 grdn:0.141 lr:4.4e-06 updt_s:0.160 data_s:0.071 +INFO 2025-05-07 22:01:23 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.51 loss:0.001 grdn:0.120 lr:4.4e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 22:02:09 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.56 loss:0.001 grdn:0.127 lr:4.4e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 22:02:55 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.62 loss:0.001 grdn:0.136 lr:4.4e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 22:03:42 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.67 loss:0.001 grdn:0.144 lr:4.4e-06 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 22:04:27 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.73 loss:0.001 grdn:0.121 lr:4.4e-06 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 22:05:12 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.78 loss:0.001 grdn:0.128 lr:4.3e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 22:05:57 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.84 loss:0.001 grdn:0.115 lr:4.3e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 22:05:57 ts/train.py:241 Checkpoint policy after step 780000 +INFO 2025-05-07 22:06:45 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.89 loss:0.001 grdn:0.127 lr:4.3e-06 updt_s:0.158 data_s:0.063 +INFO 2025-05-07 22:07:30 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.95 loss:0.001 grdn:0.129 lr:4.3e-06 updt_s:0.158 data_s:0.069 +INFO 2025-05-07 22:08:21 ts/train.py:232 step:781K smpl:12M ep:65K epch:215.00 loss:0.001 grdn:0.133 lr:4.3e-06 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 22:09:12 ts/train.py:232 step:781K smpl:12M ep:65K epch:215.06 loss:0.001 grdn:0.122 lr:4.3e-06 updt_s:0.158 data_s:0.097 +INFO 2025-05-07 22:10:03 ts/train.py:232 step:781K smpl:12M ep:65K epch:215.11 loss:0.001 grdn:0.141 lr:4.3e-06 updt_s:0.158 data_s:0.098 +INFO 2025-05-07 22:10:55 ts/train.py:232 step:781K smpl:12M ep:65K epch:215.17 loss:0.001 grdn:0.128 lr:4.2e-06 updt_s:0.159 data_s:0.096 +INFO 2025-05-07 22:11:46 ts/train.py:232 step:781K smpl:13M ep:65K epch:215.22 loss:0.001 grdn:0.113 lr:4.2e-06 updt_s:0.158 data_s:0.096 +INFO 2025-05-07 22:12:36 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.28 loss:0.001 grdn:0.135 lr:4.2e-06 updt_s:0.158 data_s:0.093 +INFO 2025-05-07 22:13:25 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.33 loss:0.001 grdn:0.123 lr:4.2e-06 updt_s:0.158 data_s:0.084 +INFO 2025-05-07 22:14:12 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.39 loss:0.001 grdn:0.129 lr:4.2e-06 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 22:14:59 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.44 loss:0.001 grdn:0.122 lr:4.2e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 22:15:46 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.50 loss:0.001 grdn:0.133 lr:4.2e-06 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 22:16:32 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.56 loss:0.001 grdn:0.126 lr:4.2e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 22:17:17 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.61 loss:0.001 grdn:0.128 lr:4.1e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 22:18:04 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.67 loss:0.001 grdn:0.123 lr:4.1e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 22:18:51 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.72 loss:0.001 grdn:0.125 lr:4.1e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 22:19:37 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.78 loss:0.001 grdn:0.116 lr:4.1e-06 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 22:20:24 ts/train.py:232 step:784K smpl:13M ep:65K epch:215.83 loss:0.001 grdn:0.135 lr:4.1e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 22:21:11 ts/train.py:232 step:784K smpl:13M ep:65K epch:215.89 loss:0.001 grdn:0.120 lr:4.1e-06 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 22:21:58 ts/train.py:232 step:784K smpl:13M ep:65K epch:215.94 loss:0.001 grdn:0.124 lr:4.1e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 22:22:49 ts/train.py:232 step:784K smpl:13M ep:65K epch:216.00 loss:0.001 grdn:0.113 lr:4.0e-06 updt_s:0.158 data_s:0.095 +INFO 2025-05-07 22:23:36 ts/train.py:232 step:784K smpl:13M ep:65K epch:216.05 loss:0.001 grdn:0.131 lr:4.0e-06 updt_s:0.159 data_s:0.076 +INFO 2025-05-07 22:24:24 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.11 loss:0.001 grdn:0.126 lr:4.0e-06 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 22:25:13 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.16 loss:0.001 grdn:0.132 lr:4.0e-06 updt_s:0.158 data_s:0.086 +INFO 2025-05-07 22:26:02 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.22 loss:0.001 grdn:0.129 lr:4.0e-06 updt_s:0.158 data_s:0.083 +INFO 2025-05-07 22:26:51 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.27 loss:0.001 grdn:0.127 lr:4.0e-06 updt_s:0.158 data_s:0.086 +INFO 2025-05-07 22:27:37 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.33 loss:0.001 grdn:0.136 lr:4.0e-06 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 22:28:23 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.38 loss:0.001 grdn:0.123 lr:3.9e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 22:29:09 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.44 loss:0.001 grdn:0.123 lr:3.9e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 22:29:55 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.49 loss:0.001 grdn:0.128 lr:3.9e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 22:30:42 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.55 loss:0.001 grdn:0.120 lr:3.9e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 22:31:28 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.60 loss:0.001 grdn:0.118 lr:3.9e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 22:32:13 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.66 loss:0.001 grdn:0.132 lr:3.9e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 22:33:00 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.71 loss:0.001 grdn:0.141 lr:3.9e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 22:33:47 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.77 loss:0.001 grdn:0.143 lr:3.9e-06 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 22:34:34 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.82 loss:0.001 grdn:0.134 lr:3.8e-06 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 22:35:20 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.88 loss:0.001 grdn:0.107 lr:3.8e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 22:36:09 ts/train.py:232 step:788K smpl:13M ep:65K epch:216.93 loss:0.001 grdn:0.122 lr:3.8e-06 updt_s:0.158 data_s:0.086 +INFO 2025-05-07 22:36:59 ts/train.py:232 step:788K smpl:13M ep:65K epch:216.99 loss:0.001 grdn:0.125 lr:3.8e-06 updt_s:0.159 data_s:0.090 +INFO 2025-05-07 22:37:46 ts/train.py:232 step:788K smpl:13M ep:65K epch:217.04 loss:0.001 grdn:0.124 lr:3.8e-06 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 22:38:33 ts/train.py:232 step:788K smpl:13M ep:65K epch:217.10 loss:0.001 grdn:0.114 lr:3.8e-06 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 22:39:21 ts/train.py:232 step:788K smpl:13M ep:65K epch:217.15 loss:0.001 grdn:0.130 lr:3.8e-06 updt_s:0.158 data_s:0.079 +INFO 2025-05-07 22:40:07 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.21 loss:0.001 grdn:0.135 lr:3.7e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 22:40:53 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.26 loss:0.001 grdn:0.114 lr:3.7e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 22:41:40 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.32 loss:0.001 grdn:0.121 lr:3.7e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 22:42:26 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.37 loss:0.001 grdn:0.122 lr:3.7e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 22:43:12 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.43 loss:0.001 grdn:0.111 lr:3.7e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 22:43:58 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.48 loss:0.001 grdn:0.116 lr:3.7e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 22:44:45 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.54 loss:0.001 grdn:0.123 lr:3.7e-06 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 22:45:30 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.59 loss:0.001 grdn:0.124 lr:3.7e-06 updt_s:0.158 data_s:0.068 +INFO 2025-05-07 22:46:16 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.65 loss:0.001 grdn:0.120 lr:3.6e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 22:47:04 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.70 loss:0.001 grdn:0.125 lr:3.6e-06 updt_s:0.158 data_s:0.080 +INFO 2025-05-07 22:47:49 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.76 loss:0.001 grdn:0.120 lr:3.6e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 22:48:34 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.81 loss:0.001 grdn:0.112 lr:3.6e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 22:49:19 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.87 loss:0.001 grdn:0.125 lr:3.6e-06 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 22:50:14 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.92 loss:0.001 grdn:0.120 lr:3.6e-06 updt_s:0.158 data_s:0.114 +INFO 2025-05-07 22:51:07 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.98 loss:0.001 grdn:0.124 lr:3.6e-06 updt_s:0.158 data_s:0.103 +INFO 2025-05-07 22:51:59 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.03 loss:0.001 grdn:0.127 lr:3.5e-06 updt_s:0.159 data_s:0.102 +INFO 2025-05-07 22:52:52 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.09 loss:0.001 grdn:0.118 lr:3.5e-06 updt_s:0.159 data_s:0.104 +INFO 2025-05-07 22:53:44 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.14 loss:0.001 grdn:0.127 lr:3.5e-06 updt_s:0.158 data_s:0.103 +INFO 2025-05-07 22:54:36 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.20 loss:0.001 grdn:0.131 lr:3.5e-06 updt_s:0.159 data_s:0.102 +INFO 2025-05-07 22:55:22 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.25 loss:0.001 grdn:0.122 lr:3.5e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 22:56:07 ts/train.py:232 step:793K smpl:13M ep:65K epch:218.31 loss:0.001 grdn:0.125 lr:3.5e-06 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 22:56:53 ts/train.py:232 step:793K smpl:13M ep:66K epch:218.36 loss:0.001 grdn:0.144 lr:3.5e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 22:57:38 ts/train.py:232 step:793K smpl:13M ep:66K epch:218.42 loss:0.001 grdn:0.139 lr:3.5e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 22:58:24 ts/train.py:232 step:793K smpl:13M ep:66K epch:218.47 loss:0.001 grdn:0.127 lr:3.4e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 22:59:09 ts/train.py:232 step:793K smpl:13M ep:66K epch:218.53 loss:0.001 grdn:0.132 lr:3.4e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 22:59:55 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.58 loss:0.001 grdn:0.123 lr:3.4e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 23:00:41 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.64 loss:0.001 grdn:0.116 lr:3.4e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 23:01:26 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.70 loss:0.001 grdn:0.137 lr:3.4e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 23:02:11 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.75 loss:0.001 grdn:0.126 lr:3.4e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 23:02:56 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.81 loss:0.001 grdn:0.132 lr:3.4e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 23:03:45 ts/train.py:232 step:795K smpl:13M ep:66K epch:218.86 loss:0.001 grdn:0.118 lr:3.4e-06 updt_s:0.158 data_s:0.085 +INFO 2025-05-07 23:04:35 ts/train.py:232 step:795K smpl:13M ep:66K epch:218.92 loss:0.001 grdn:0.119 lr:3.3e-06 updt_s:0.159 data_s:0.090 +INFO 2025-05-07 23:05:24 ts/train.py:232 step:795K smpl:13M ep:66K epch:218.97 loss:0.001 grdn:0.121 lr:3.3e-06 updt_s:0.159 data_s:0.087 +INFO 2025-05-07 23:06:15 ts/train.py:232 step:795K smpl:13M ep:66K epch:219.03 loss:0.001 grdn:0.132 lr:3.3e-06 updt_s:0.159 data_s:0.091 +INFO 2025-05-07 23:07:03 ts/train.py:232 step:795K smpl:13M ep:66K epch:219.08 loss:0.001 grdn:0.122 lr:3.3e-06 updt_s:0.158 data_s:0.083 +INFO 2025-05-07 23:07:52 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.14 loss:0.001 grdn:0.119 lr:3.3e-06 updt_s:0.158 data_s:0.085 +INFO 2025-05-07 23:08:38 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.19 loss:0.001 grdn:0.116 lr:3.3e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 23:09:24 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.25 loss:0.001 grdn:0.116 lr:3.3e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 23:10:09 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.30 loss:0.001 grdn:0.116 lr:3.3e-06 updt_s:0.159 data_s:0.065 +INFO 2025-05-07 23:10:55 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.36 loss:0.001 grdn:0.121 lr:3.2e-06 updt_s:0.158 data_s:0.068 +INFO 2025-05-07 23:11:40 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.41 loss:0.001 grdn:0.125 lr:3.2e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 23:12:25 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.47 loss:0.001 grdn:0.119 lr:3.2e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 23:13:10 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.52 loss:0.001 grdn:0.126 lr:3.2e-06 updt_s:0.158 data_s:0.068 +INFO 2025-05-07 23:13:56 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.58 loss:0.001 grdn:0.131 lr:3.2e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 23:14:43 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.63 loss:0.001 grdn:0.134 lr:3.2e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 23:15:28 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.69 loss:0.001 grdn:0.130 lr:3.2e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 23:16:14 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.74 loss:0.001 grdn:0.122 lr:3.2e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 23:16:59 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.80 loss:0.001 grdn:0.118 lr:3.1e-06 updt_s:0.158 data_s:0.068 +INFO 2025-05-07 23:17:53 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.85 loss:0.001 grdn:0.118 lr:3.1e-06 updt_s:0.158 data_s:0.107 +INFO 2025-05-07 23:18:45 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.91 loss:0.001 grdn:0.111 lr:3.1e-06 updt_s:0.158 data_s:0.106 +INFO 2025-05-07 23:19:37 ts/train.py:232 step:799K smpl:13M ep:66K epch:219.96 loss:0.001 grdn:0.115 lr:3.1e-06 updt_s:0.158 data_s:0.098 +INFO 2025-05-07 23:20:29 ts/train.py:232 step:799K smpl:13M ep:66K epch:220.02 loss:0.001 grdn:0.126 lr:3.1e-06 updt_s:0.160 data_s:0.103 +INFO 2025-05-07 23:21:22 ts/train.py:232 step:799K smpl:13M ep:66K epch:220.07 loss:0.001 grdn:0.130 lr:3.1e-06 updt_s:0.158 data_s:0.103 +INFO 2025-05-07 23:22:14 ts/train.py:232 step:799K smpl:13M ep:66K epch:220.13 loss:0.001 grdn:0.119 lr:3.1e-06 updt_s:0.158 data_s:0.104 +INFO 2025-05-07 23:23:01 ts/train.py:232 step:799K smpl:13M ep:66K epch:220.18 loss:0.001 grdn:0.122 lr:3.1e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 23:23:46 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.24 loss:0.001 grdn:0.127 lr:3.0e-06 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 23:24:32 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.29 loss:0.001 grdn:0.117 lr:3.0e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 23:25:17 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.35 loss:0.001 grdn:0.118 lr:3.0e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 23:26:05 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.40 loss:0.001 grdn:0.114 lr:3.0e-06 updt_s:0.158 data_s:0.079 +INFO 2025-05-07 23:26:51 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.46 loss:0.001 grdn:0.123 lr:3.0e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 23:27:36 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.51 loss:0.001 grdn:0.110 lr:3.0e-06 updt_s:0.158 data_s:0.068 +INFO 2025-05-07 23:28:24 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.57 loss:0.001 grdn:0.114 lr:3.0e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-07 23:29:10 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.62 loss:0.001 grdn:0.126 lr:3.0e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 23:29:55 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.68 loss:0.001 grdn:0.121 lr:3.0e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 23:30:40 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.73 loss:0.001 grdn:0.117 lr:2.9e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 23:31:30 ts/train.py:232 step:802K smpl:13M ep:66K epch:220.79 loss:0.001 grdn:0.130 lr:2.9e-06 updt_s:0.159 data_s:0.089 +INFO 2025-05-07 23:32:20 ts/train.py:232 step:802K smpl:13M ep:66K epch:220.84 loss:0.001 grdn:0.109 lr:2.9e-06 updt_s:0.159 data_s:0.094 +INFO 2025-05-07 23:33:12 ts/train.py:232 step:802K smpl:13M ep:66K epch:220.90 loss:0.001 grdn:0.124 lr:2.9e-06 updt_s:0.159 data_s:0.099 +INFO 2025-05-07 23:34:03 ts/train.py:232 step:802K smpl:13M ep:66K epch:220.95 loss:0.001 grdn:0.126 lr:2.9e-06 updt_s:0.159 data_s:0.096 +INFO 2025-05-07 23:34:54 ts/train.py:232 step:802K smpl:13M ep:66K epch:221.01 loss:0.001 grdn:0.128 lr:2.9e-06 updt_s:0.159 data_s:0.096 +INFO 2025-05-07 23:35:45 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.06 loss:0.001 grdn:0.133 lr:2.9e-06 updt_s:0.159 data_s:0.093 +INFO 2025-05-07 23:36:34 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.12 loss:0.001 grdn:0.123 lr:2.9e-06 updt_s:0.159 data_s:0.084 +INFO 2025-05-07 23:37:19 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.17 loss:0.001 grdn:0.101 lr:2.8e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 23:38:04 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.23 loss:0.001 grdn:0.125 lr:2.8e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 23:38:51 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.28 loss:0.001 grdn:0.124 lr:2.8e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 23:39:36 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.34 loss:0.001 grdn:0.107 lr:2.8e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 23:40:22 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.39 loss:0.001 grdn:0.126 lr:2.8e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 23:41:08 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.45 loss:0.001 grdn:0.111 lr:2.8e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-07 23:41:54 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.50 loss:0.001 grdn:0.119 lr:2.8e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 23:42:40 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.56 loss:0.001 grdn:0.141 lr:2.8e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 23:43:26 ts/train.py:232 step:805K smpl:13M ep:66K epch:221.61 loss:0.001 grdn:0.123 lr:2.8e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-07 23:44:12 ts/train.py:232 step:805K smpl:13M ep:67K epch:221.67 loss:0.001 grdn:0.121 lr:2.7e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 23:44:57 ts/train.py:232 step:805K smpl:13M ep:67K epch:221.72 loss:0.001 grdn:0.115 lr:2.7e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 23:45:54 ts/train.py:232 step:805K smpl:13M ep:67K epch:221.78 loss:0.001 grdn:0.117 lr:2.7e-06 updt_s:0.159 data_s:0.125 +INFO 2025-05-07 23:46:48 ts/train.py:232 step:805K smpl:13M ep:67K epch:221.84 loss:0.001 grdn:0.119 lr:2.7e-06 updt_s:0.159 data_s:0.109 +INFO 2025-05-07 23:47:42 ts/train.py:232 step:806K smpl:13M ep:67K epch:221.89 loss:0.001 grdn:0.113 lr:2.7e-06 updt_s:0.158 data_s:0.112 +INFO 2025-05-07 23:48:36 ts/train.py:232 step:806K smpl:13M ep:67K epch:221.95 loss:0.001 grdn:0.124 lr:2.7e-06 updt_s:0.159 data_s:0.108 +INFO 2025-05-07 23:49:29 ts/train.py:232 step:806K smpl:13M ep:67K epch:222.00 loss:0.001 grdn:0.134 lr:2.7e-06 updt_s:0.159 data_s:0.105 +INFO 2025-05-07 23:50:22 ts/train.py:232 step:806K smpl:13M ep:67K epch:222.06 loss:0.001 grdn:0.122 lr:2.7e-06 updt_s:0.159 data_s:0.108 +INFO 2025-05-07 23:51:09 ts/train.py:232 step:806K smpl:13M ep:67K epch:222.11 loss:0.001 grdn:0.129 lr:2.7e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-07 23:51:55 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.17 loss:0.001 grdn:0.127 lr:2.6e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 23:52:40 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.22 loss:0.001 grdn:0.117 lr:2.6e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-07 23:53:25 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.28 loss:0.001 grdn:0.129 lr:2.6e-06 updt_s:0.159 data_s:0.064 +INFO 2025-05-07 23:54:11 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.33 loss:0.001 grdn:0.128 lr:2.6e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 23:54:56 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.39 loss:0.001 grdn:0.144 lr:2.6e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 23:55:41 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.44 loss:0.001 grdn:0.127 lr:2.6e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 23:56:27 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.50 loss:0.001 grdn:0.113 lr:2.6e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-07 23:57:12 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.55 loss:0.001 grdn:0.138 lr:2.6e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 23:57:59 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.61 loss:0.001 grdn:0.130 lr:2.6e-06 updt_s:0.160 data_s:0.070 +INFO 2025-05-07 23:58:44 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.66 loss:0.001 grdn:0.129 lr:2.5e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-07 23:59:36 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.72 loss:0.001 grdn:0.112 lr:2.5e-06 updt_s:0.159 data_s:0.097 +INFO 2025-05-08 00:00:30 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.77 loss:0.001 grdn:0.124 lr:2.5e-06 updt_s:0.159 data_s:0.109 +INFO 2025-05-08 00:01:24 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.83 loss:0.001 grdn:0.126 lr:2.5e-06 updt_s:0.159 data_s:0.112 +INFO 2025-05-08 00:02:19 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.88 loss:0.001 grdn:0.117 lr:2.5e-06 updt_s:0.159 data_s:0.114 +INFO 2025-05-08 00:03:13 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.94 loss:0.001 grdn:0.118 lr:2.5e-06 updt_s:0.159 data_s:0.112 +INFO 2025-05-08 00:04:09 ts/train.py:232 step:810K smpl:13M ep:67K epch:222.99 loss:0.001 grdn:0.129 lr:2.5e-06 updt_s:0.159 data_s:0.119 +INFO 2025-05-08 00:05:02 ts/train.py:232 step:810K smpl:13M ep:67K epch:223.05 loss:0.001 grdn:0.121 lr:2.5e-06 updt_s:0.159 data_s:0.105 +INFO 2025-05-08 00:05:50 ts/train.py:232 step:810K smpl:13M ep:67K epch:223.10 loss:0.001 grdn:0.120 lr:2.5e-06 updt_s:0.159 data_s:0.083 +INFO 2025-05-08 00:05:50 ts/train.py:241 Checkpoint policy after step 810000 +INFO 2025-05-08 00:06:41 ts/train.py:232 step:810K smpl:13M ep:67K epch:223.16 loss:0.001 grdn:0.112 lr:2.4e-06 updt_s:0.160 data_s:0.076 +INFO 2025-05-08 00:07:29 ts/train.py:232 step:810K smpl:13M ep:67K epch:223.21 loss:0.001 grdn:0.114 lr:2.4e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-08 00:08:16 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.27 loss:0.001 grdn:0.136 lr:2.4e-06 updt_s:0.159 data_s:0.076 +INFO 2025-05-08 00:09:05 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.32 loss:0.001 grdn:0.117 lr:2.4e-06 updt_s:0.160 data_s:0.083 +INFO 2025-05-08 00:09:52 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.38 loss:0.001 grdn:0.114 lr:2.4e-06 updt_s:0.161 data_s:0.076 +INFO 2025-05-08 00:10:40 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.43 loss:0.001 grdn:0.133 lr:2.4e-06 updt_s:0.160 data_s:0.077 +INFO 2025-05-08 00:11:29 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.49 loss:0.001 grdn:0.123 lr:2.4e-06 updt_s:0.160 data_s:0.083 +INFO 2025-05-08 00:12:16 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.54 loss:0.001 grdn:0.130 lr:2.4e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-08 00:13:04 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.60 loss:0.001 grdn:0.122 lr:2.4e-06 updt_s:0.159 data_s:0.077 +INFO 2025-05-08 00:13:51 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.65 loss:0.001 grdn:0.126 lr:2.3e-06 updt_s:0.159 data_s:0.078 +INFO 2025-05-08 00:14:45 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.71 loss:0.001 grdn:0.121 lr:2.3e-06 updt_s:0.159 data_s:0.108 +INFO 2025-05-08 00:15:38 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.76 loss:0.001 grdn:0.105 lr:2.3e-06 updt_s:0.159 data_s:0.105 +INFO 2025-05-08 00:16:30 ts/train.py:232 step:813K smpl:13M ep:67K epch:223.82 loss:0.001 grdn:0.120 lr:2.3e-06 updt_s:0.159 data_s:0.101 +INFO 2025-05-08 00:17:22 ts/train.py:232 step:813K smpl:13M ep:67K epch:223.87 loss:0.001 grdn:0.135 lr:2.3e-06 updt_s:0.159 data_s:0.101 +INFO 2025-05-08 00:18:15 ts/train.py:232 step:813K smpl:13M ep:67K epch:223.93 loss:0.001 grdn:0.127 lr:2.3e-06 updt_s:0.159 data_s:0.100 +INFO 2025-05-08 00:19:07 ts/train.py:232 step:813K smpl:13M ep:67K epch:223.98 loss:0.001 grdn:0.122 lr:2.3e-06 updt_s:0.159 data_s:0.102 +INFO 2025-05-08 00:19:53 ts/train.py:232 step:813K smpl:13M ep:67K epch:224.04 loss:0.001 grdn:0.125 lr:2.3e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-08 00:20:38 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.09 loss:0.001 grdn:0.120 lr:2.3e-06 updt_s:0.161 data_s:0.064 +INFO 2025-05-08 00:21:24 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.15 loss:0.001 grdn:0.115 lr:2.3e-06 updt_s:0.160 data_s:0.068 +INFO 2025-05-08 00:22:10 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.20 loss:0.001 grdn:0.121 lr:2.2e-06 updt_s:0.160 data_s:0.070 +INFO 2025-05-08 00:22:57 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.26 loss:0.001 grdn:0.130 lr:2.2e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-08 00:23:43 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.31 loss:0.001 grdn:0.130 lr:2.2e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-08 00:24:30 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.37 loss:0.001 grdn:0.130 lr:2.2e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-08 00:25:16 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.42 loss:0.001 grdn:0.129 lr:2.2e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-08 00:26:03 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.48 loss:0.001 grdn:0.116 lr:2.2e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-08 00:26:49 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.53 loss:0.001 grdn:0.125 lr:2.2e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-08 00:27:35 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.59 loss:0.001 grdn:0.115 lr:2.2e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-08 00:28:26 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.64 loss:0.001 grdn:0.132 lr:2.2e-06 updt_s:0.159 data_s:0.097 +INFO 2025-05-08 00:29:20 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.70 loss:0.001 grdn:0.128 lr:2.2e-06 updt_s:0.159 data_s:0.109 +INFO 2025-05-08 00:30:14 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.75 loss:0.001 grdn:0.126 lr:2.1e-06 updt_s:0.160 data_s:0.109 +INFO 2025-05-08 00:31:08 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.81 loss:0.001 grdn:0.121 lr:2.1e-06 updt_s:0.159 data_s:0.112 +INFO 2025-05-08 00:32:03 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.86 loss:0.001 grdn:0.115 lr:2.1e-06 updt_s:0.158 data_s:0.116 +INFO 2025-05-08 00:32:59 ts/train.py:232 step:817K smpl:13M ep:67K epch:224.92 loss:0.001 grdn:0.118 lr:2.1e-06 updt_s:0.159 data_s:0.117 +INFO 2025-05-08 00:33:50 ts/train.py:232 step:817K smpl:13M ep:67K epch:224.98 loss:0.001 grdn:0.126 lr:2.1e-06 updt_s:0.159 data_s:0.096 +INFO 2025-05-08 00:34:38 ts/train.py:232 step:817K smpl:13M ep:68K epch:225.03 loss:0.001 grdn:0.117 lr:2.1e-06 updt_s:0.159 data_s:0.084 +INFO 2025-05-08 00:35:26 ts/train.py:232 step:817K smpl:13M ep:68K epch:225.09 loss:0.001 grdn:0.119 lr:2.1e-06 updt_s:0.159 data_s:0.078 +INFO 2025-05-08 00:36:13 ts/train.py:232 step:817K smpl:13M ep:68K epch:225.14 loss:0.001 grdn:0.125 lr:2.1e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-08 00:37:01 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.20 loss:0.001 grdn:0.122 lr:2.1e-06 updt_s:0.158 data_s:0.082 +INFO 2025-05-08 00:37:49 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.25 loss:0.001 grdn:0.124 lr:2.1e-06 updt_s:0.159 data_s:0.078 +INFO 2025-05-08 00:38:36 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.31 loss:0.001 grdn:0.116 lr:2.0e-06 updt_s:0.159 data_s:0.078 +INFO 2025-05-08 00:39:23 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.36 loss:0.001 grdn:0.124 lr:2.0e-06 updt_s:0.158 data_s:0.078 +INFO 2025-05-08 00:40:11 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.42 loss:0.001 grdn:0.115 lr:2.0e-06 updt_s:0.158 data_s:0.079 +INFO 2025-05-08 00:40:59 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.47 loss:0.001 grdn:0.137 lr:2.0e-06 updt_s:0.158 data_s:0.081 +INFO 2025-05-08 00:41:47 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.53 loss:0.001 grdn:0.119 lr:2.0e-06 updt_s:0.159 data_s:0.080 +INFO 2025-05-08 00:42:35 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.58 loss:0.001 grdn:0.123 lr:2.0e-06 updt_s:0.158 data_s:0.080 +INFO 2025-05-08 00:43:31 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.64 loss:0.001 grdn:0.138 lr:2.0e-06 updt_s:0.159 data_s:0.120 +INFO 2025-05-08 00:44:23 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.69 loss:0.001 grdn:0.143 lr:2.0e-06 updt_s:0.158 data_s:0.104 +INFO 2025-05-08 00:45:17 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.75 loss:0.001 grdn:0.117 lr:2.0e-06 updt_s:0.158 data_s:0.109 +INFO 2025-05-08 00:46:10 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.80 loss:0.001 grdn:0.118 lr:2.0e-06 updt_s:0.158 data_s:0.106 +INFO 2025-05-08 00:47:04 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.86 loss:0.001 grdn:0.112 lr:1.9e-06 updt_s:0.158 data_s:0.109 +INFO 2025-05-08 00:47:57 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.91 loss:0.001 grdn:0.111 lr:1.9e-06 updt_s:0.159 data_s:0.108 +INFO 2025-05-08 00:48:44 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.97 loss:0.001 grdn:0.124 lr:1.9e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-08 00:49:29 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.02 loss:0.001 grdn:0.128 lr:1.9e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-08 00:50:15 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.08 loss:0.001 grdn:0.118 lr:1.9e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-08 00:51:01 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.13 loss:0.001 grdn:0.128 lr:1.9e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-08 00:51:47 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.19 loss:0.001 grdn:0.115 lr:1.9e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-08 00:52:33 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.24 loss:0.001 grdn:0.103 lr:1.9e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-08 00:53:19 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.30 loss:0.001 grdn:0.133 lr:1.9e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-08 00:54:04 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.35 loss:0.001 grdn:0.122 lr:1.9e-06 updt_s:0.158 data_s:0.066 +INFO 2025-05-08 00:54:50 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.41 loss:0.001 grdn:0.107 lr:1.8e-06 updt_s:0.158 data_s:0.069 +INFO 2025-05-08 00:55:35 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.46 loss:0.001 grdn:0.117 lr:1.8e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-08 00:56:21 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.52 loss:0.001 grdn:0.122 lr:1.8e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-08 00:57:09 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.57 loss:0.001 grdn:0.127 lr:1.8e-06 updt_s:0.158 data_s:0.084 +INFO 2025-05-08 00:57:55 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.63 loss:0.001 grdn:0.106 lr:1.8e-06 updt_s:0.158 data_s:0.069 +INFO 2025-05-08 00:58:41 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.68 loss:0.001 grdn:0.119 lr:1.8e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-08 00:59:25 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.74 loss:0.001 grdn:0.121 lr:1.8e-06 updt_s:0.158 data_s:0.064 +INFO 2025-05-08 01:00:11 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.79 loss:0.001 grdn:0.127 lr:1.8e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-08 01:00:56 ts/train.py:232 step:824K smpl:13M ep:68K epch:226.85 loss:0.001 grdn:0.123 lr:1.8e-06 updt_s:0.159 data_s:0.065 +INFO 2025-05-08 01:01:42 ts/train.py:232 step:824K smpl:13M ep:68K epch:226.90 loss:0.001 grdn:0.126 lr:1.8e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-08 01:02:27 ts/train.py:232 step:824K smpl:13M ep:68K epch:226.96 loss:0.001 grdn:0.119 lr:1.8e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-08 01:03:13 ts/train.py:232 step:824K smpl:13M ep:68K epch:227.01 loss:0.001 grdn:0.134 lr:1.7e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-08 01:03:59 ts/train.py:232 step:824K smpl:13M ep:68K epch:227.07 loss:0.001 grdn:0.123 lr:1.7e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-08 01:04:44 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.12 loss:0.001 grdn:0.136 lr:1.7e-06 updt_s:0.159 data_s:0.064 +INFO 2025-05-08 01:05:30 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.18 loss:0.001 grdn:0.121 lr:1.7e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-08 01:06:15 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.23 loss:0.001 grdn:0.123 lr:1.7e-06 updt_s:0.158 data_s:0.065 +INFO 2025-05-08 01:07:01 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.29 loss:0.001 grdn:0.114 lr:1.7e-06 updt_s:0.158 data_s:0.075 +INFO 2025-05-08 01:07:47 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.34 loss:0.001 grdn:0.112 lr:1.7e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-08 01:08:33 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.40 loss:0.001 grdn:0.122 lr:1.7e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-08 01:09:20 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.45 loss:0.001 grdn:0.111 lr:1.7e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-08 01:10:05 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.51 loss:0.001 grdn:0.115 lr:1.7e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-08 01:10:59 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.56 loss:0.001 grdn:0.121 lr:1.7e-06 updt_s:0.158 data_s:0.108 +INFO 2025-05-08 01:11:48 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.62 loss:0.001 grdn:0.108 lr:1.6e-06 updt_s:0.159 data_s:0.089 +INFO 2025-05-08 01:12:37 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.67 loss:0.001 grdn:0.122 lr:1.6e-06 updt_s:0.159 data_s:0.084 +INFO 2025-05-08 01:13:25 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.73 loss:0.001 grdn:0.107 lr:1.6e-06 updt_s:0.158 data_s:0.077 +INFO 2025-05-08 01:14:12 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.78 loss:0.001 grdn:0.118 lr:1.6e-06 updt_s:0.159 data_s:0.076 +INFO 2025-05-08 01:15:00 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.84 loss:0.001 grdn:0.107 lr:1.6e-06 updt_s:0.158 data_s:0.080 +INFO 2025-05-08 01:15:45 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.89 loss:0.001 grdn:0.109 lr:1.6e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-08 01:16:30 ts/train.py:232 step:828K smpl:13M ep:68K epch:227.95 loss:0.001 grdn:0.120 lr:1.6e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-08 01:17:16 ts/train.py:232 step:828K smpl:13M ep:68K epch:228.00 loss:0.001 grdn:0.134 lr:1.6e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-08 01:18:01 ts/train.py:232 step:828K smpl:13M ep:68K epch:228.06 loss:0.001 grdn:0.137 lr:1.6e-06 updt_s:0.159 data_s:0.066 +INFO 2025-05-08 01:18:47 ts/train.py:232 step:828K smpl:13M ep:68K epch:228.11 loss:0.001 grdn:0.112 lr:1.6e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-08 01:19:33 ts/train.py:232 step:828K smpl:13M ep:68K epch:228.17 loss:0.001 grdn:0.115 lr:1.6e-06 updt_s:0.160 data_s:0.069 +INFO 2025-05-08 01:20:19 ts/train.py:232 step:829K smpl:13M ep:68K epch:228.23 loss:0.001 grdn:0.131 lr:1.6e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-08 01:21:05 ts/train.py:232 step:829K smpl:13M ep:68K epch:228.28 loss:0.001 grdn:0.115 lr:1.5e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-08 01:21:50 ts/train.py:232 step:829K smpl:13M ep:69K epch:228.34 loss:0.001 grdn:0.118 lr:1.5e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-08 01:22:36 ts/train.py:232 step:829K smpl:13M ep:69K epch:228.39 loss:0.001 grdn:0.120 lr:1.5e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-08 01:23:22 ts/train.py:232 step:829K smpl:13M ep:69K epch:228.45 loss:0.001 grdn:0.130 lr:1.5e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-08 01:24:09 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.50 loss:0.001 grdn:0.124 lr:1.5e-06 updt_s:0.158 data_s:0.078 +INFO 2025-05-08 01:24:56 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.56 loss:0.001 grdn:0.120 lr:1.5e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-08 01:25:42 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.61 loss:0.001 grdn:0.121 lr:1.5e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-08 01:26:28 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.67 loss:0.001 grdn:0.116 lr:1.5e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-08 01:27:14 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.72 loss:0.001 grdn:0.120 lr:1.5e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-08 01:27:59 ts/train.py:232 step:831K smpl:13M ep:69K epch:228.78 loss:0.001 grdn:0.129 lr:1.5e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-08 01:28:45 ts/train.py:232 step:831K smpl:13M ep:69K epch:228.83 loss:0.001 grdn:0.112 lr:1.5e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-08 01:29:31 ts/train.py:232 step:831K smpl:13M ep:69K epch:228.89 loss:0.001 grdn:0.120 lr:1.4e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-08 01:30:17 ts/train.py:232 step:831K smpl:13M ep:69K epch:228.94 loss:0.001 grdn:0.108 lr:1.4e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-08 01:31:03 ts/train.py:232 step:831K smpl:13M ep:69K epch:229.00 loss:0.001 grdn:0.115 lr:1.4e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-08 01:31:50 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.05 loss:0.001 grdn:0.093 lr:1.4e-06 updt_s:0.158 data_s:0.076 +INFO 2025-05-08 01:32:36 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.11 loss:0.001 grdn:0.118 lr:1.4e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-08 01:33:22 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.16 loss:0.001 grdn:0.113 lr:1.4e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-08 01:34:08 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.22 loss:0.001 grdn:0.124 lr:1.4e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-08 01:34:54 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.27 loss:0.001 grdn:0.121 lr:1.4e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-08 01:35:40 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.33 loss:0.001 grdn:0.108 lr:1.4e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-08 01:36:27 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.38 loss:0.001 grdn:0.100 lr:1.4e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-08 01:37:12 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.44 loss:0.001 grdn:0.114 lr:1.4e-06 updt_s:0.159 data_s:0.067 +INFO 2025-05-08 01:38:00 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.49 loss:0.001 grdn:0.103 lr:1.4e-06 updt_s:0.159 data_s:0.080 +INFO 2025-05-08 01:38:47 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.55 loss:0.001 grdn:0.109 lr:1.4e-06 updt_s:0.159 data_s:0.076 +INFO 2025-05-08 01:39:34 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.60 loss:0.001 grdn:0.119 lr:1.3e-06 updt_s:0.160 data_s:0.075 +INFO 2025-05-08 01:40:21 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.66 loss:0.001 grdn:0.101 lr:1.3e-06 updt_s:0.159 data_s:0.074 +INFO 2025-05-08 01:41:07 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.71 loss:0.001 grdn:0.120 lr:1.3e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-08 01:41:53 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.77 loss:0.001 grdn:0.126 lr:1.3e-06 updt_s:0.159 data_s:0.069 +INFO 2025-05-08 01:42:39 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.82 loss:0.001 grdn:0.117 lr:1.3e-06 updt_s:0.158 data_s:0.074 +INFO 2025-05-08 01:43:25 ts/train.py:232 step:835K smpl:13M ep:69K epch:229.88 loss:0.001 grdn:0.120 lr:1.3e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-08 01:44:11 ts/train.py:232 step:835K smpl:13M ep:69K epch:229.93 loss:0.001 grdn:0.113 lr:1.3e-06 updt_s:0.158 data_s:0.075 +INFO 2025-05-08 01:44:58 ts/train.py:232 step:835K smpl:13M ep:69K epch:229.99 loss:0.001 grdn:0.126 lr:1.3e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-08 01:45:44 ts/train.py:232 step:835K smpl:13M ep:69K epch:230.04 loss:0.001 grdn:0.104 lr:1.3e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-08 01:46:31 ts/train.py:232 step:835K smpl:13M ep:69K epch:230.10 loss:0.001 grdn:0.131 lr:1.3e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-08 01:47:17 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.15 loss:0.001 grdn:0.121 lr:1.3e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-08 01:48:03 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.21 loss:0.001 grdn:0.121 lr:1.3e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-08 01:48:50 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.26 loss:0.001 grdn:0.116 lr:1.2e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-08 01:49:36 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.32 loss:0.001 grdn:0.113 lr:1.2e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-08 01:50:22 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.37 loss:0.001 grdn:0.110 lr:1.2e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-08 01:51:13 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.43 loss:0.001 grdn:0.115 lr:1.2e-06 updt_s:0.158 data_s:0.096 +INFO 2025-05-08 01:52:06 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.48 loss:0.001 grdn:0.116 lr:1.2e-06 updt_s:0.159 data_s:0.106 +INFO 2025-05-08 01:52:59 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.54 loss:0.001 grdn:0.104 lr:1.2e-06 updt_s:0.159 data_s:0.104 +INFO 2025-05-08 01:53:51 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.59 loss:0.001 grdn:0.110 lr:1.2e-06 updt_s:0.158 data_s:0.102 +INFO 2025-05-08 01:54:44 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.65 loss:0.001 grdn:0.119 lr:1.2e-06 updt_s:0.159 data_s:0.103 +INFO 2025-05-08 01:55:36 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.70 loss:0.001 grdn:0.123 lr:1.2e-06 updt_s:0.159 data_s:0.103 +INFO 2025-05-08 01:56:26 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.76 loss:0.001 grdn:0.114 lr:1.2e-06 updt_s:0.159 data_s:0.089 +INFO 2025-05-08 01:57:13 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.81 loss:0.001 grdn:0.120 lr:1.2e-06 updt_s:0.159 data_s:0.072 +INFO 2025-05-08 01:57:59 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.87 loss:0.001 grdn:0.119 lr:1.2e-06 updt_s:0.159 data_s:0.075 +INFO 2025-05-08 01:58:45 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.92 loss:0.001 grdn:0.126 lr:1.2e-06 updt_s:0.159 data_s:0.070 +INFO 2025-05-08 01:59:31 ts/train.py:232 step:839K smpl:13M ep:69K epch:230.98 loss:0.001 grdn:0.113 lr:1.1e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-08 02:00:19 ts/train.py:232 step:839K smpl:13M ep:69K epch:231.03 loss:0.001 grdn:0.115 lr:1.1e-06 updt_s:0.159 data_s:0.079 +INFO 2025-05-08 02:01:05 ts/train.py:232 step:839K smpl:13M ep:69K epch:231.09 loss:0.001 grdn:0.136 lr:1.1e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-08 02:01:51 ts/train.py:232 step:839K smpl:13M ep:69K epch:231.14 loss:0.001 grdn:0.127 lr:1.1e-06 updt_s:0.159 data_s:0.068 +INFO 2025-05-08 02:02:37 ts/train.py:232 step:839K smpl:13M ep:69K epch:231.20 loss:0.001 grdn:0.111 lr:1.1e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-08 02:03:23 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.25 loss:0.001 grdn:0.106 lr:1.1e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-08 02:04:09 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.31 loss:0.001 grdn:0.110 lr:1.1e-06 updt_s:0.159 data_s:0.071 +INFO 2025-05-08 02:04:55 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.37 loss:0.001 grdn:0.120 lr:1.1e-06 updt_s:0.158 data_s:0.069 +INFO 2025-05-08 02:04:55 ts/train.py:241 Checkpoint policy after step 840000 +INFO 2025-05-08 02:05:50 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.42 loss:0.001 grdn:0.130 lr:1.1e-06 updt_s:0.159 data_s:0.100 +INFO 2025-05-08 02:06:40 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.48 loss:0.001 grdn:0.117 lr:1.1e-06 updt_s:0.159 data_s:0.090 +INFO 2025-05-08 02:07:28 ts/train.py:232 step:841K smpl:13M ep:69K epch:231.53 loss:0.001 grdn:0.116 lr:1.1e-06 updt_s:0.159 data_s:0.081 +INFO 2025-05-08 02:08:17 ts/train.py:232 step:841K smpl:13M ep:69K epch:231.59 loss:0.001 grdn:0.110 lr:1.1e-06 updt_s:0.159 data_s:0.086 +INFO 2025-05-08 02:09:06 ts/train.py:232 step:841K smpl:13M ep:69K epch:231.64 loss:0.001 grdn:0.122 lr:1.1e-06 updt_s:0.158 data_s:0.088 +INFO 2025-05-08 02:09:57 ts/train.py:232 step:841K smpl:13M ep:70K epch:231.70 loss:0.001 grdn:0.111 lr:1.1e-06 updt_s:0.159 data_s:0.093 +INFO 2025-05-08 02:10:43 ts/train.py:232 step:841K smpl:13M ep:70K epch:231.75 loss:0.001 grdn:0.105 lr:1.0e-06 updt_s:0.158 data_s:0.072 +INFO 2025-05-08 02:11:30 ts/train.py:232 step:842K smpl:13M ep:70K epch:231.81 loss:0.001 grdn:0.102 lr:1.0e-06 updt_s:0.159 data_s:0.073 +INFO 2025-05-08 02:12:16 ts/train.py:232 step:842K smpl:13M ep:70K epch:231.86 loss:0.001 grdn:0.114 lr:1.0e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-08 02:13:02 ts/train.py:232 step:842K smpl:13M ep:70K epch:231.92 loss:0.001 grdn:0.120 lr:1.0e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-08 02:13:47 ts/train.py:232 step:842K smpl:13M ep:70K epch:231.97 loss:0.001 grdn:0.129 lr:1.0e-06 updt_s:0.159 data_s:0.066 +Traceback (most recent call last): + File "/NAS/LSY/lerobot/lerobot/scripts/train.py", line 288, in + train() + File "/NAS/LSY/lerobot/lerobot/configs/parser.py", line 227, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/NAS/LSY/lerobot/lerobot/scripts/train.py", line 212, in train + train_tracker, output_dict = update_policy( + File "/NAS/LSY/lerobot/lerobot/scripts/train.py", line 101, in update_policy + train_metrics.loss = loss.item() +KeyboardInterrupt diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/wandb-summary.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..48903d3d1d86cca3284c8d833db05501f49f91a7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":217665.344528983,"train/dataloading_s":0.06568688039726112,"train/episodes":69591.32380788431,"_step":842200,"train/samples":13475200,"train/lr":1.0188423871646376e-06,"train/steps":842200,"train/update_s":0.1587559862283524,"train/grad_norm":0.12918526319321244,"_timestamp":1.746670427359524e+09,"_wandb":{"runtime":217665},"train/epochs":231.97107935961438,"train/loss":0.0011329334909532917} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-core.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-core.log index 61f33a90b98d30c6e1144d950565f4aee06256aa..b709686a35afa422bc101b01a410a061c50a3a17 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-core.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-core.log @@ -4,3 +4,10 @@ {"time":"2025-05-05T13:46:33.42106851Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:53992"} {"time":"2025-05-05T13:46:33.42881454Z","level":"INFO","msg":"handleInformInit: received","streamId":"or9d7oy0","id":"127.0.0.1:53992"} {"time":"2025-05-05T13:46:34.472503481Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"or9d7oy0","id":"127.0.0.1:53992"} +{"time":"2025-05-08T02:14:18.772757781Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:53992"} +{"time":"2025-05-08T02:14:18.772815262Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:53992"} +{"time":"2025-05-08T02:14:18.772831383Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-05-08T02:14:18.772893164Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:53992"} +{"time":"2025-05-08T02:14:20.338426597Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:53992"} +{"time":"2025-05-08T02:14:20.338463068Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:53992"} +{"time":"2025-05-08T02:14:20.338479398Z","level":"INFO","msg":"server is closed"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-internal.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-internal.log index 0f739f5d73b02ccb5464c78f556d225706e7483b..2879aef5947db11c83a230b579e9bed0d3395884 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-internal.log @@ -14,3 +14,12 @@ {"time":"2025-05-06T22:47:41.804454606Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} {"time":"2025-05-06T23:06:22.73637838Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} {"time":"2025-05-07T10:20:45.433607767Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:33216->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-07T16:28:15.433106661Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49715->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-08T02:14:18.772849593Z","level":"INFO","msg":"stream: closing","id":"or9d7oy0"} +{"time":"2025-05-08T02:14:18.772873613Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-05-08T02:14:18.772915274Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-05-08T02:14:20.058837021Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-05-08T02:14:20.336944052Z","level":"INFO","msg":"handler: closed","stream_id":"or9d7oy0"} +{"time":"2025-05-08T02:14:20.336978492Z","level":"INFO","msg":"writer: Close: closed","stream_id":"or9d7oy0"} +{"time":"2025-05-08T02:14:20.336987063Z","level":"INFO","msg":"sender: closed","stream_id":"or9d7oy0"} +{"time":"2025-05-08T02:14:20.337076564Z","level":"INFO","msg":"stream: closed","id":"or9d7oy0"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug.log index c9380a323f5aecefbc6b4434c56996a5a5a9d062..01a06dc9569f9f115e41881f835af94d98f74df9 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug.log @@ -20,3 +20,4 @@ config: {'dataset': {'repo_id': 'shylee/so100_pengripA', 'root': '/SSD/LSY/lerob 2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2371] Wrapping output streams. 2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2394] Redirects installed. 2025-05-05 13:46:34,902 INFO MainThread:102711 [wandb_init.py:init():1056] run started, returning control to user process +2025-05-08 02:14:18,771 INFO MsgRouterThr:102711 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/run-or9d7oy0.wandb b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/run-or9d7oy0.wandb index 013f835556f95f3383e06311e31c4a4168b15998..fb145d45dc14b3bdd4bf0e22c58181e6a5774cc2 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/run-or9d7oy0.wandb +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/run-or9d7oy0.wandb @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1df54928bbec7b7068859e0ed2d4b2f01542608f94284f2aefbc61e5fec12e85 -size 37453824 +oid sha256:5195cc151acf6b6b135f26797bb6d6ee7881f23ac3f5ace3bc50b0c006625f60 +size 49709045