diff --git a/.gitattributes b/.gitattributes index 99ee7b8dbb88895c49e589ebbf218c844e65d7d3..bf4e7e8d2ff52add7ad6ef7014afc72df32dc50a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -44,3 +44,4 @@ DP_cube_downDims1_cropNo_freeze1_64_64_ema0_1e-4/wandb/run-20250502_094142-yq6yq DP_cube_downDims1_cropNo_freeze1_16_16_ema1_1e-4/wandb/run-20250503_130343-yd8ehjnc/run-yd8ehjnc.wandb filter=lfs diff=lfs merge=lfs -text DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb filter=lfs diff=lfs merge=lfs -text DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/run-ygbytty1.wandb filter=lfs diff=lfs merge=lfs -text +DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/run-or9d7oy0.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4998a17046c21179641924cf63c6a4cd8ad3c49b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5c48b51250d993c7da24c8656bf036a9e45f224e8711c8886e6fec456b58f6d +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..7d4ed5a676fc441768347a116ad61d8539fe81fa --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.973484658309718e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ea306e772bf1672c624e001029cdaed3bc2629d4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8f785adff055fb08dafe2244a1f463c694aed8ed4451633b716c00e13788e1f +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..27175ce2292a840d46b8aa57fee74051e04b7bd7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85ec521e2b6ebf54474b5626a80c7f70203b5e22c7d9e0897e4289745fbd1d3b +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bd0cc4c8bbd1814446b0a6695f94da31b23961de --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 30000, + "verbose": false, + "_step_count": 30001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.973484658309718e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..f4945f660f45b332883dccfccf18d8b8815d916a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 30000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eeb61a1feb107125609c4851370a8322c05f600d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb4a874b689bce3af5f4ba5275654a4a69535504dad3d4308a4f4b966f042a9a +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..c4b9153044afcd3a374917ee7385a8edfd2c00ce --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.892425861749099e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f6d6a690a7dc314da20092081fcdde590931452f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c2141ac0ed22421c7d37f32932c1aae102786b1484c41b8dc2e9d6ed82b85f2 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ffcd703507480819e55a0f5f809cd7d984bb9ec3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22cc0fa561811580260fc0afa216a0e12023777837f2004fffee6591f61c0089 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9c3d845454183abd4bb260e7962ae0003eff2976 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 60000, + "verbose": false, + "_step_count": 60001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.892425861749099e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d360b6037f1c8c2c85d38e951160eb02ace507dd --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 60000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0d52e445675f819d4fb3224954af7bb13c0c4b26 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a57bdebb12814ca74de082a13f6b72ef7c532ede8600c07c1cefbda35247fd1 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..844517b39b97d4b240356b58a29f5d5477ec3544 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.757705070247532e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a930088b134946d8b5528aa158e12c410840ba6f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1904e3f16d2f64fad4c1985119631d2b8acd6d3dec3e0440a2ab8b71555f1105 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cbaf38b5e1dae11cf169205d0326c8dc3985436e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12948dd2d4a3abc4cbc5d85506718e8d1a07f88faff40275624cf096457985eb +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fce6df3fd82ed4234f9a3b0fd9fc8ec6fc021844 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 90000, + "verbose": false, + "_step_count": 90001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.757705070247532e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..cec965b83b7f817ac99559792f84cbb69a721bfd --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 90000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..144353db16c5c7b3fbf2215b94d417bfb4f67c44 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adb739a640471bfd19d179d7d7b8d523ca17011fdc06348efda604368e2a6936 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..303785ff5bc367d42704d992bf6eed47bb7272a1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.57079995288704e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c470d491886dc7e2c7f257ecae90bcda5b5220ef --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09e1d67963827be4bd52925add4a9de31401c6c37bd792149d9cbe05840b6434 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..82f16e71a674aac8a7626270854ce076fb1e77c0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3476a1e3a94b97ce3d51bc9dedaf15a4cd56062a1fd7bd734d8dfead2656152b +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f978e60892059238d68988919b32bd99948d45d3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 120000, + "verbose": false, + "_step_count": 120001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.57079995288704e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbb8719463d819c865c104597046a0165eea278 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 120000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bd0178cf687a934b9113701d9af986dc6e6a9821 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82d2efad9f51c306e408d35fdaf4675298738ccf34851a3a76cbb1ccfd4c2205 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..ccf8d114f2cb5b174697d10148c2b182caaed9c6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.33376055636136e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..64f6fb94d58d8544e71c5320dd3baf8758c52562 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a97c0296dff3340cadbfa21f9b6b6e288cdcaf7f3482ecedab71a5350b931739 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c1e913d58abf098b79d6f6b850183fb46e775116 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a57b9c48613be93fd5184e080913108d22264ec5f3cfc3b23082270d558f9b5 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0767923b0a738183eb68f042ae20f5f89c2f9ddf --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 150000, + "verbose": false, + "_step_count": 150001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.33376055636136e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..941897fdfccbe971211529c488ac32a17d6dc0a0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 150000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0024d8ec6b8a852a5f67697946aa412c3bbe9cea --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66861b14d7d4fd90525e384916022ccc2f08aba1bd9b894cb42087ba71d355b5 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..a029d09f58b7b30c6b0782f9db5301ca5a7ca89b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.049186819281031e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ae4d8764cec8727feb85b66206a0128edccd4f6a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b2b76f9c68ebbf8c507879db06e076b5ac0ae909a21a14edd502615467fcfeb +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a5490fb57fc56f8f76e98bb2fe93ed91378c2da3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c89151a70d9e8884c37ec868f1562377455f9f86c9570eb8d7b78a3654752837 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7e725acefb858f85655970cd16215b4aec1acdea --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 180000, + "verbose": false, + "_step_count": 180001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.049186819281031e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..3da013f3092359f6c4ebb97dfc69e8300dbb32e4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 180000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d85974c1cd3a14aa1d7a42f3a9910d9096fa87f6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6d0b8cde743ffcfd0d8bb90b8c362c78051d0ec722488bce455b0cfed1fb2cc +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..a2ebd55b052e1e980a503765e694b0de8af480ec --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 8.720200055053977e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..28394fd5dbe36785aa339437eba0173f71833bf0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0fab0f1ac5302adb79af54436e3ab97f40231abd96a1c83429f454e96fbd9e8 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..93471ce23874fb7729301745069e1618a2fc1412 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f078eabbd82c0cad6b40a29d9a4d8549b81a015563404873d259305a11a6fd8 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..97ab942fad8a77c663b0037063c5951ba5dbc781 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 210000, + "verbose": false, + "_step_count": 210001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 8.720200055053977e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..6147d40ee6d82bb05c893a1b35d8f59f302da656 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 210000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f565b2d984ac02635619fa5b64a8ff907e1ad4a1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a443e04557612b56c303cca4e23e41ec9707538061bf7659029cc18688c52099 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..b973d80db1e60787fa3565daf1f4712fdb7b364e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 8.350408716128216e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..76171b5fb49ddc9ad1d464f1cece6568a7b650f4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b1ae6c0773b167d953d9f7612f030585dc338d2b3577b8660a4649f2b6eae9d +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..da6e3b54dc2c95608f47f7b096937568b5e6ca5c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a49453e36adb50eaf16a1a2b5cd1347e6ea1ae86a7f08f3393977f8cf1b417c +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..20c00e1b8846ac18ae698337037774699743306a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 240000, + "verbose": false, + "_step_count": 240001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 8.350408716128216e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..cd776dcc291fdcc40079da5c594255ca04238bce --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 240000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a9db84d21be13be9eea40a8fcf7290dff7cc9fb9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13ce3819d15b43e52da61c1a65181ba5ee04e0cf1b9d441051cd207b97fbfe15 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1085c71f91668de124c2f5269441008dc91a6665 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.943868815107594e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..90185b08c6c255df6da5f310453494aac98690b9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f2b2f7026dcd31ade519358d6b1cf3fec3b2684f380564b46a3769194d14a1d +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0a4e85b540018ef679a228e1457669ba4f18fbbc --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e79d3f310626cbd6f714033729b3bfad76cecf206872ddfe3fe7cf0e65fdbbb +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e2fc5c0727d511fc139fe9e28a78e368a31b04b7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 270000, + "verbose": false, + "_step_count": 270001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.943868815107594e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..537f4e9f0bf2092976dffaa70a673dde4c3fd66d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 270000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c0b7c56bafb932c1cfb9b1ba27c1855c8a71f1d3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36efb67682d0d38e08733facd05898949949570aa00a3b8f1f198d3bed5bdc2e +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..441b7eada7faf4a78ccaa1e14c1d8083b6e9035c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.505039436856827e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f10a4be7896dad7d2ab5dadc9f14bc627b6a4324 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e8eea9cef34750143ab72cf4627d07f08782f233cbdf743f47c86a785d1d093 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4066f551111c9c6611581fea6215020a9849ad64 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b3e77b8e37486811c424ea23b49990daf4d90416bb0f3a73f290ad53cb0d2b0 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..46f0924834037be23d642823ee826bc729c90d27 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 300000, + "verbose": false, + "_step_count": 300001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.505039436856827e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..db7e948ddab8257b846ad5ef28f8c555a9524598 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 300000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6961b8a837e7ba123d985167042c65befdc791 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..72d1950f894218f820d5f1f48a9f560d03cd7d76 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:725d0c3904b194dcfb86a000e822643aafad92ad4199620e749e7f98a3a649e3 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ac6effea52ddb93b573b66857d89eb463f5246e6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..cb2ac96baeaf8f3382dc8f0f62087f73edc7f970 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.038733829556077e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2dd21ce2b7c16b3446545ebdb13a3ebbfde31beb --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85efd3e090961e3648919a0daf1e031ec3c875f7bf867781600793f99f08d240 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a28ae5f2bb6de18886ec219ea9bb28110f7c737e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3200f454cd8a224b27e2b9166272386d2076dbc3831553e054c22cc316dfd3f5 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..594d01a0a1b45b388016e6e6c58a1074f424337e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 330000, + "verbose": false, + "_step_count": 330001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.038733829556077e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..f5ea2cc3e4569f3f5076d82101c911d3ca4dadb3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 330000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..95577922fe177a6276ba796a7fd52ba26c00c0aa --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log @@ -0,0 +1,13 @@ +{"time":"2025-05-05T13:46:33.42896044Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-core.log"} +{"time":"2025-05-05T13:46:34.472474241Z","level":"INFO","msg":"created new stream","id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.472498521Z","level":"INFO","msg":"stream: started","id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.472695701Z","level":"INFO","msg":"writer: Do: started","stream_id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.472704801Z","level":"INFO","msg":"sender: started","stream_id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.47285267Z","level":"INFO","msg":"handler: started","stream_id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.815364301Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-05-05T18:28:10.438000676Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49720->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T09:33:25.436758862Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:41668->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T11:02:10.437470095Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:58472->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:15:55.437088156Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:45422->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:30:55.436340164Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:51617->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:42:40.435841749Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:46107->141.223.7.51:53: i/o timeout"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c9380a323f5aecefbc6b4434c56996a5a5a9d062 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/debug.log @@ -0,0 +1,22 @@ +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Configure stats pid to 102711 +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug.log +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-internal.log +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:init():781] calling init triggers +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'shylee/so100_pengripA', 'root': '/SSD/LSY/lerobot_dataset/pengripA', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'horizon': 32, 'n_action_steps': 32, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ema': True, 'ema_lambda': 0.5}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4', 'job_name': 'DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 900000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 30000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:init():809] starting backend +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:init():813] sending inform_init request +2025-05-05 13:46:33,427 INFO MainThread:102711 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-05-05 13:46:33,427 INFO MainThread:102711 [wandb_init.py:init():823] backend started and connected +2025-05-05 13:46:33,429 INFO MainThread:102711 [wandb_init.py:init():915] updated telemetry +2025-05-05 13:46:33,587 INFO MainThread:102711 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-05-05 13:46:34,771 INFO MainThread:102711 [wandb_init.py:init():1014] starting run threads in backend +2025-05-05 13:46:34,900 INFO MainThread:102711 [wandb_run.py:_console_start():2454] atexit reg +2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2394] Redirects installed. +2025-05-05 13:46:34,902 INFO MainThread:102711 [wandb_init.py:init():1056] run started, returning control to user process diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/output.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..874f9a3a7ea007e2dcee6780ca3eff8605c8c4a1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/output.log @@ -0,0 +1,1706 @@ +Logs will be synced with wandb. +INFO 2025-05-05 13:46:34 ndb_utils.py:96 Track this run --> https://wandb.ai/marchmelo0923-postech/lerobot/runs/or9d7oy0 +INFO 2025-05-05 13:46:34 ts/train.py:127 Creating dataset +Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 300/300 [00:00<00:00, 40666.12it/s] +INFO 2025-05-05 13:46:36 ts/train.py:138 Creating policy +INFO 2025-05-05 13:46:42 ts/train.py:144 Creating optimizer and scheduler +INFO 2025-05-05 13:46:42 ts/train.py:156 Output dir: /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4 +INFO 2025-05-05 13:46:42 ts/train.py:159 cfg.steps=900000 (900K) +INFO 2025-05-05 13:46:42 ts/train.py:160 dataset.num_frames=58090 (58K) +INFO 2025-05-05 13:46:42 ts/train.py:161 dataset.num_episodes=300 +INFO 2025-05-05 13:46:42 ts/train.py:162 num_learnable_params=92242982 (92M) +INFO 2025-05-05 13:46:42 ts/train.py:163 num_total_params=92243036 (92M) +INFO 2025-05-05 13:46:42 ts/train.py:202 Start offline training on a fixed dataset +INFO 2025-05-05 13:47:43 ts/train.py:232 step:200 smpl:3K ep:17 epch:0.06 loss:0.688 grdn:3.182 lr:2.0e-05 updt_s:0.197 data_s:0.111 +INFO 2025-05-05 13:48:37 ts/train.py:232 step:400 smpl:6K ep:33 epch:0.11 loss:0.127 grdn:2.566 lr:6.0e-05 updt_s:0.163 data_s:0.105 +INFO 2025-05-05 13:49:31 ts/train.py:232 step:600 smpl:10K ep:50 epch:0.17 loss:0.083 grdn:1.852 lr:9.5e-05 updt_s:0.163 data_s:0.106 +INFO 2025-05-05 13:50:26 ts/train.py:232 step:800 smpl:13K ep:66 epch:0.22 loss:0.056 grdn:1.181 lr:1.0e-04 updt_s:0.163 data_s:0.112 +INFO 2025-05-05 13:51:22 ts/train.py:232 step:1K smpl:16K ep:83 epch:0.28 loss:0.051 grdn:1.004 lr:1.0e-04 updt_s:0.163 data_s:0.116 +INFO 2025-05-05 13:52:19 ts/train.py:232 step:1K smpl:19K ep:99 epch:0.33 loss:0.045 grdn:0.854 lr:1.0e-04 updt_s:0.164 data_s:0.120 +INFO 2025-05-05 13:53:08 ts/train.py:232 step:1K smpl:22K ep:116 epch:0.39 loss:0.040 grdn:0.790 lr:1.0e-04 updt_s:0.163 data_s:0.079 +INFO 2025-05-05 13:53:56 ts/train.py:232 step:2K smpl:26K ep:132 epch:0.44 loss:0.034 grdn:0.688 lr:1.0e-04 updt_s:0.164 data_s:0.077 +INFO 2025-05-05 13:54:44 ts/train.py:232 step:2K smpl:29K ep:149 epch:0.50 loss:0.033 grdn:0.615 lr:1.0e-04 updt_s:0.164 data_s:0.074 +INFO 2025-05-05 13:55:31 ts/train.py:232 step:2K smpl:32K ep:165 epch:0.55 loss:0.032 grdn:0.638 lr:1.0e-04 updt_s:0.165 data_s:0.070 +INFO 2025-05-05 13:56:20 ts/train.py:232 step:2K smpl:35K ep:182 epch:0.61 loss:0.032 grdn:0.605 lr:1.0e-04 updt_s:0.165 data_s:0.076 +INFO 2025-05-05 13:57:08 ts/train.py:232 step:2K smpl:38K ep:198 epch:0.66 loss:0.032 grdn:0.586 lr:1.0e-04 updt_s:0.164 data_s:0.076 +INFO 2025-05-05 13:57:59 ts/train.py:232 step:3K smpl:42K ep:215 epch:0.72 loss:0.031 grdn:0.565 lr:1.0e-04 updt_s:0.164 data_s:0.092 +INFO 2025-05-05 13:58:48 ts/train.py:232 step:3K smpl:45K ep:231 epch:0.77 loss:0.028 grdn:0.510 lr:1.0e-04 updt_s:0.165 data_s:0.078 +INFO 2025-05-05 13:59:35 ts/train.py:232 step:3K smpl:48K ep:248 epch:0.83 loss:0.028 grdn:0.498 lr:1.0e-04 updt_s:0.164 data_s:0.072 +INFO 2025-05-05 14:00:23 ts/train.py:232 step:3K smpl:51K ep:264 epch:0.88 loss:0.027 grdn:0.455 lr:1.0e-04 updt_s:0.166 data_s:0.070 +INFO 2025-05-05 14:01:10 ts/train.py:232 step:3K smpl:54K ep:281 epch:0.94 loss:0.026 grdn:0.449 lr:1.0e-04 updt_s:0.164 data_s:0.072 +INFO 2025-05-05 14:02:10 ts/train.py:232 step:4K smpl:58K ep:297 epch:0.99 loss:0.027 grdn:0.456 lr:1.0e-04 updt_s:0.182 data_s:0.115 +INFO 2025-05-05 14:03:13 ts/train.py:232 step:4K smpl:61K ep:314 epch:1.05 loss:0.025 grdn:0.423 lr:1.0e-04 updt_s:0.166 data_s:0.151 +INFO 2025-05-05 14:04:17 ts/train.py:232 step:4K smpl:64K ep:331 epch:1.10 loss:0.025 grdn:0.409 lr:1.0e-04 updt_s:0.165 data_s:0.149 +INFO 2025-05-05 14:05:18 ts/train.py:232 step:4K smpl:67K ep:347 epch:1.16 loss:0.024 grdn:0.393 lr:1.0e-04 updt_s:0.164 data_s:0.143 +INFO 2025-05-05 14:06:21 ts/train.py:232 step:4K smpl:70K ep:364 epch:1.21 loss:0.023 grdn:0.391 lr:1.0e-04 updt_s:0.164 data_s:0.148 +INFO 2025-05-05 14:07:23 ts/train.py:232 step:5K smpl:74K ep:380 epch:1.27 loss:0.024 grdn:0.403 lr:1.0e-04 updt_s:0.164 data_s:0.145 +INFO 2025-05-05 14:08:20 ts/train.py:232 step:5K smpl:77K ep:397 epch:1.32 loss:0.023 grdn:0.365 lr:1.0e-04 updt_s:0.164 data_s:0.122 +INFO 2025-05-05 14:09:09 ts/train.py:232 step:5K smpl:80K ep:413 epch:1.38 loss:0.024 grdn:0.387 lr:1.0e-04 updt_s:0.166 data_s:0.080 +INFO 2025-05-05 14:09:57 ts/train.py:232 step:5K smpl:83K ep:430 epch:1.43 loss:0.023 grdn:0.367 lr:1.0e-04 updt_s:0.164 data_s:0.072 +INFO 2025-05-05 14:10:44 ts/train.py:232 step:5K smpl:86K ep:446 epch:1.49 loss:0.023 grdn:0.376 lr:1.0e-04 updt_s:0.164 data_s:0.071 +INFO 2025-05-05 14:11:32 ts/train.py:232 step:6K smpl:90K ep:463 epch:1.54 loss:0.023 grdn:0.360 lr:1.0e-04 updt_s:0.164 data_s:0.073 +INFO 2025-05-05 14:12:22 ts/train.py:232 step:6K smpl:93K ep:479 epch:1.60 loss:0.022 grdn:0.327 lr:1.0e-04 updt_s:0.164 data_s:0.084 +INFO 2025-05-05 14:13:12 ts/train.py:232 step:6K smpl:96K ep:496 epch:1.65 loss:0.022 grdn:0.318 lr:1.0e-04 updt_s:0.163 data_s:0.089 +INFO 2025-05-05 14:14:01 ts/train.py:232 step:6K smpl:99K ep:512 epch:1.71 loss:0.022 grdn:0.333 lr:1.0e-04 updt_s:0.164 data_s:0.080 +INFO 2025-05-05 14:14:50 ts/train.py:232 step:6K smpl:102K ep:529 epch:1.76 loss:0.023 grdn:0.341 lr:1.0e-04 updt_s:0.163 data_s:0.079 +INFO 2025-05-05 14:15:39 ts/train.py:232 step:7K smpl:106K ep:545 epch:1.82 loss:0.021 grdn:0.325 lr:1.0e-04 updt_s:0.163 data_s:0.082 +INFO 2025-05-05 14:16:27 ts/train.py:232 step:7K smpl:109K ep:562 epch:1.87 loss:0.022 grdn:0.300 lr:1.0e-04 updt_s:0.163 data_s:0.078 +INFO 2025-05-05 14:17:14 ts/train.py:232 step:7K smpl:112K ep:578 epch:1.93 loss:0.022 grdn:0.327 lr:1.0e-04 updt_s:0.163 data_s:0.070 +INFO 2025-05-05 14:18:09 ts/train.py:232 step:7K smpl:115K ep:595 epch:1.98 loss:0.022 grdn:0.313 lr:1.0e-04 updt_s:0.162 data_s:0.113 +INFO 2025-05-05 14:19:03 ts/train.py:232 step:7K smpl:118K ep:611 epch:2.04 loss:0.021 grdn:0.289 lr:1.0e-04 updt_s:0.162 data_s:0.104 +INFO 2025-05-05 14:19:55 ts/train.py:232 step:8K smpl:122K ep:628 epch:2.09 loss:0.020 grdn:0.288 lr:1.0e-04 updt_s:0.162 data_s:0.100 +INFO 2025-05-05 14:20:48 ts/train.py:232 step:8K smpl:125K ep:645 epch:2.15 loss:0.019 grdn:0.282 lr:1.0e-04 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 14:21:41 ts/train.py:232 step:8K smpl:128K ep:661 epch:2.20 loss:0.021 grdn:0.301 lr:1.0e-04 updt_s:0.163 data_s:0.101 +INFO 2025-05-05 14:22:34 ts/train.py:232 step:8K smpl:131K ep:678 epch:2.26 loss:0.020 grdn:0.277 lr:1.0e-04 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 14:23:23 ts/train.py:232 step:8K smpl:134K ep:694 epch:2.31 loss:0.021 grdn:0.303 lr:1.0e-04 updt_s:0.163 data_s:0.078 +INFO 2025-05-05 14:24:11 ts/train.py:232 step:9K smpl:138K ep:711 epch:2.37 loss:0.021 grdn:0.287 lr:1.0e-04 updt_s:0.164 data_s:0.077 +INFO 2025-05-05 14:24:59 ts/train.py:232 step:9K smpl:141K ep:727 epch:2.42 loss:0.021 grdn:0.296 lr:1.0e-04 updt_s:0.163 data_s:0.078 +INFO 2025-05-05 14:25:48 ts/train.py:232 step:9K smpl:144K ep:744 epch:2.48 loss:0.019 grdn:0.262 lr:1.0e-04 updt_s:0.163 data_s:0.079 +INFO 2025-05-05 14:26:36 ts/train.py:232 step:9K smpl:147K ep:760 epch:2.53 loss:0.019 grdn:0.261 lr:1.0e-04 updt_s:0.163 data_s:0.078 +INFO 2025-05-05 14:27:25 ts/train.py:232 step:9K smpl:150K ep:777 epch:2.59 loss:0.021 grdn:0.278 lr:1.0e-04 updt_s:0.163 data_s:0.080 +INFO 2025-05-05 14:28:14 ts/train.py:232 step:10K smpl:154K ep:793 epch:2.64 loss:0.019 grdn:0.251 lr:1.0e-04 updt_s:0.163 data_s:0.080 +INFO 2025-05-05 14:29:03 ts/train.py:232 step:10K smpl:157K ep:810 epch:2.70 loss:0.020 grdn:0.257 lr:1.0e-04 updt_s:0.163 data_s:0.081 +INFO 2025-05-05 14:29:52 ts/train.py:232 step:10K smpl:160K ep:826 epch:2.75 loss:0.020 grdn:0.269 lr:1.0e-04 updt_s:0.163 data_s:0.080 +INFO 2025-05-05 14:30:42 ts/train.py:232 step:10K smpl:163K ep:843 epch:2.81 loss:0.019 grdn:0.254 lr:1.0e-04 updt_s:0.163 data_s:0.086 +INFO 2025-05-05 14:31:29 ts/train.py:232 step:10K smpl:166K ep:859 epch:2.86 loss:0.018 grdn:0.244 lr:1.0e-04 updt_s:0.162 data_s:0.075 +INFO 2025-05-05 14:32:20 ts/train.py:232 step:11K smpl:170K ep:876 epch:2.92 loss:0.018 grdn:0.253 lr:1.0e-04 updt_s:0.162 data_s:0.093 +INFO 2025-05-05 14:33:11 ts/train.py:232 step:11K smpl:173K ep:892 epch:2.97 loss:0.018 grdn:0.246 lr:1.0e-04 updt_s:0.162 data_s:0.090 +INFO 2025-05-05 14:34:01 ts/train.py:232 step:11K smpl:176K ep:909 epch:3.03 loss:0.020 grdn:0.255 lr:1.0e-04 updt_s:0.163 data_s:0.087 +INFO 2025-05-05 14:34:49 ts/train.py:232 step:11K smpl:179K ep:925 epch:3.08 loss:0.019 grdn:0.255 lr:1.0e-04 updt_s:0.162 data_s:0.076 +INFO 2025-05-05 14:35:38 ts/train.py:232 step:11K smpl:182K ep:942 epch:3.14 loss:0.019 grdn:0.255 lr:1.0e-04 updt_s:0.163 data_s:0.080 +INFO 2025-05-05 14:36:28 ts/train.py:232 step:12K smpl:186K ep:959 epch:3.20 loss:0.018 grdn:0.243 lr:1.0e-04 updt_s:0.163 data_s:0.086 +INFO 2025-05-05 14:37:16 ts/train.py:232 step:12K smpl:189K ep:975 epch:3.25 loss:0.019 grdn:0.257 lr:1.0e-04 updt_s:0.163 data_s:0.077 +INFO 2025-05-05 14:38:02 ts/train.py:232 step:12K smpl:192K ep:992 epch:3.31 loss:0.018 grdn:0.242 lr:1.0e-04 updt_s:0.162 data_s:0.068 +INFO 2025-05-05 14:38:49 ts/train.py:232 step:12K smpl:195K ep:1K epch:3.36 loss:0.018 grdn:0.237 lr:1.0e-04 updt_s:0.162 data_s:0.072 +INFO 2025-05-05 14:39:35 ts/train.py:232 step:12K smpl:198K ep:1K epch:3.42 loss:0.017 grdn:0.228 lr:1.0e-04 updt_s:0.163 data_s:0.067 +INFO 2025-05-05 14:40:23 ts/train.py:232 step:13K smpl:202K ep:1K epch:3.47 loss:0.018 grdn:0.241 lr:1.0e-04 updt_s:0.164 data_s:0.071 +INFO 2025-05-05 14:41:09 ts/train.py:232 step:13K smpl:205K ep:1K epch:3.53 loss:0.017 grdn:0.232 lr:1.0e-04 updt_s:0.164 data_s:0.066 +INFO 2025-05-05 14:41:56 ts/train.py:232 step:13K smpl:208K ep:1K epch:3.58 loss:0.017 grdn:0.236 lr:1.0e-04 updt_s:0.163 data_s:0.070 +INFO 2025-05-05 14:42:44 ts/train.py:232 step:13K smpl:211K ep:1K epch:3.64 loss:0.018 grdn:0.234 lr:1.0e-04 updt_s:0.163 data_s:0.077 +INFO 2025-05-05 14:43:32 ts/train.py:232 step:13K smpl:214K ep:1K epch:3.69 loss:0.017 grdn:0.230 lr:1.0e-04 updt_s:0.164 data_s:0.076 +INFO 2025-05-05 14:44:20 ts/train.py:232 step:14K smpl:218K ep:1K epch:3.75 loss:0.017 grdn:0.246 lr:1.0e-04 updt_s:0.164 data_s:0.076 +INFO 2025-05-05 14:45:09 ts/train.py:232 step:14K smpl:221K ep:1K epch:3.80 loss:0.017 grdn:0.224 lr:1.0e-04 updt_s:0.164 data_s:0.079 +INFO 2025-05-05 14:45:56 ts/train.py:232 step:14K smpl:224K ep:1K epch:3.86 loss:0.017 grdn:0.219 lr:1.0e-04 updt_s:0.164 data_s:0.070 +INFO 2025-05-05 14:46:51 ts/train.py:232 step:14K smpl:227K ep:1K epch:3.91 loss:0.017 grdn:0.234 lr:1.0e-04 updt_s:0.163 data_s:0.113 +INFO 2025-05-05 14:47:46 ts/train.py:232 step:14K smpl:230K ep:1K epch:3.97 loss:0.016 grdn:0.222 lr:1.0e-04 updt_s:0.163 data_s:0.107 +INFO 2025-05-05 14:48:40 ts/train.py:232 step:15K smpl:234K ep:1K epch:4.02 loss:0.016 grdn:0.219 lr:1.0e-04 updt_s:0.163 data_s:0.110 +INFO 2025-05-05 14:49:35 ts/train.py:232 step:15K smpl:237K ep:1K epch:4.08 loss:0.016 grdn:0.204 lr:1.0e-04 updt_s:0.163 data_s:0.109 +INFO 2025-05-05 14:50:29 ts/train.py:232 step:15K smpl:240K ep:1K epch:4.13 loss:0.016 grdn:0.222 lr:1.0e-04 updt_s:0.163 data_s:0.108 +INFO 2025-05-05 14:51:24 ts/train.py:232 step:15K smpl:243K ep:1K epch:4.19 loss:0.016 grdn:0.214 lr:1.0e-04 updt_s:0.163 data_s:0.109 +INFO 2025-05-05 14:52:12 ts/train.py:232 step:15K smpl:246K ep:1K epch:4.24 loss:0.016 grdn:0.217 lr:1.0e-04 updt_s:0.164 data_s:0.077 +INFO 2025-05-05 14:53:00 ts/train.py:232 step:16K smpl:250K ep:1K epch:4.30 loss:0.018 grdn:0.236 lr:1.0e-04 updt_s:0.164 data_s:0.072 +INFO 2025-05-05 14:53:47 ts/train.py:232 step:16K smpl:253K ep:1K epch:4.35 loss:0.016 grdn:0.220 lr:1.0e-04 updt_s:0.164 data_s:0.071 +INFO 2025-05-05 14:54:35 ts/train.py:232 step:16K smpl:256K ep:1K epch:4.41 loss:0.017 grdn:0.216 lr:1.0e-04 updt_s:0.164 data_s:0.077 +INFO 2025-05-05 14:55:22 ts/train.py:232 step:16K smpl:259K ep:1K epch:4.46 loss:0.017 grdn:0.216 lr:1.0e-04 updt_s:0.164 data_s:0.069 +INFO 2025-05-05 14:56:09 ts/train.py:232 step:16K smpl:262K ep:1K epch:4.52 loss:0.016 grdn:0.207 lr:1.0e-04 updt_s:0.164 data_s:0.073 +INFO 2025-05-05 14:56:57 ts/train.py:232 step:17K smpl:266K ep:1K epch:4.57 loss:0.016 grdn:0.214 lr:1.0e-04 updt_s:0.164 data_s:0.073 +INFO 2025-05-05 14:57:45 ts/train.py:232 step:17K smpl:269K ep:1K epch:4.63 loss:0.016 grdn:0.219 lr:1.0e-04 updt_s:0.164 data_s:0.073 +INFO 2025-05-05 14:58:33 ts/train.py:232 step:17K smpl:272K ep:1K epch:4.68 loss:0.015 grdn:0.200 lr:1.0e-04 updt_s:0.164 data_s:0.075 +INFO 2025-05-05 14:59:19 ts/train.py:232 step:17K smpl:275K ep:1K epch:4.74 loss:0.016 grdn:0.194 lr:1.0e-04 updt_s:0.164 data_s:0.068 +INFO 2025-05-05 15:00:07 ts/train.py:232 step:17K smpl:278K ep:1K epch:4.79 loss:0.016 grdn:0.213 lr:1.0e-04 updt_s:0.164 data_s:0.073 +INFO 2025-05-05 15:00:57 ts/train.py:232 step:18K smpl:282K ep:1K epch:4.85 loss:0.016 grdn:0.212 lr:1.0e-04 updt_s:0.163 data_s:0.087 +INFO 2025-05-05 15:01:46 ts/train.py:232 step:18K smpl:285K ep:1K epch:4.90 loss:0.015 grdn:0.211 lr:1.0e-04 updt_s:0.164 data_s:0.080 +INFO 2025-05-05 15:02:35 ts/train.py:232 step:18K smpl:288K ep:1K epch:4.96 loss:0.016 grdn:0.212 lr:1.0e-04 updt_s:0.163 data_s:0.081 +INFO 2025-05-05 15:03:23 ts/train.py:232 step:18K smpl:291K ep:2K epch:5.01 loss:0.015 grdn:0.212 lr:1.0e-04 updt_s:0.164 data_s:0.077 +INFO 2025-05-05 15:04:14 ts/train.py:232 step:18K smpl:294K ep:2K epch:5.07 loss:0.016 grdn:0.219 lr:1.0e-04 updt_s:0.164 data_s:0.088 +INFO 2025-05-05 15:05:03 ts/train.py:232 step:19K smpl:298K ep:2K epch:5.12 loss:0.015 grdn:0.211 lr:1.0e-04 updt_s:0.164 data_s:0.081 +INFO 2025-05-05 15:05:51 ts/train.py:232 step:19K smpl:301K ep:2K epch:5.18 loss:0.015 grdn:0.204 lr:1.0e-04 updt_s:0.164 data_s:0.076 +INFO 2025-05-05 15:06:39 ts/train.py:232 step:19K smpl:304K ep:2K epch:5.23 loss:0.016 grdn:0.210 lr:1.0e-04 updt_s:0.164 data_s:0.074 +INFO 2025-05-05 15:07:26 ts/train.py:232 step:19K smpl:307K ep:2K epch:5.29 loss:0.015 grdn:0.201 lr:1.0e-04 updt_s:0.165 data_s:0.069 +INFO 2025-05-05 15:08:14 ts/train.py:232 step:19K smpl:310K ep:2K epch:5.34 loss:0.014 grdn:0.193 lr:1.0e-04 updt_s:0.165 data_s:0.073 +INFO 2025-05-05 15:09:02 ts/train.py:232 step:20K smpl:314K ep:2K epch:5.40 loss:0.015 grdn:0.209 lr:1.0e-04 updt_s:0.165 data_s:0.072 +INFO 2025-05-05 15:09:49 ts/train.py:232 step:20K smpl:317K ep:2K epch:5.45 loss:0.014 grdn:0.192 lr:1.0e-04 updt_s:0.165 data_s:0.071 +INFO 2025-05-05 15:10:37 ts/train.py:232 step:20K smpl:320K ep:2K epch:5.51 loss:0.015 grdn:0.201 lr:1.0e-04 updt_s:0.165 data_s:0.073 +INFO 2025-05-05 15:11:24 ts/train.py:232 step:20K smpl:323K ep:2K epch:5.56 loss:0.015 grdn:0.199 lr:1.0e-04 updt_s:0.165 data_s:0.070 +INFO 2025-05-05 15:12:11 ts/train.py:232 step:20K smpl:326K ep:2K epch:5.62 loss:0.015 grdn:0.194 lr:1.0e-04 updt_s:0.164 data_s:0.071 +INFO 2025-05-05 15:12:58 ts/train.py:232 step:21K smpl:330K ep:2K epch:5.67 loss:0.014 grdn:0.194 lr:1.0e-04 updt_s:0.164 data_s:0.071 +INFO 2025-05-05 15:13:45 ts/train.py:232 step:21K smpl:333K ep:2K epch:5.73 loss:0.014 grdn:0.205 lr:1.0e-04 updt_s:0.164 data_s:0.070 +INFO 2025-05-05 15:14:32 ts/train.py:232 step:21K smpl:336K ep:2K epch:5.78 loss:0.015 grdn:0.208 lr:1.0e-04 updt_s:0.163 data_s:0.070 +INFO 2025-05-05 15:15:28 ts/train.py:232 step:21K smpl:339K ep:2K epch:5.84 loss:0.015 grdn:0.199 lr:1.0e-04 updt_s:0.164 data_s:0.113 +INFO 2025-05-05 15:16:19 ts/train.py:232 step:21K smpl:342K ep:2K epch:5.89 loss:0.014 grdn:0.187 lr:1.0e-04 updt_s:0.164 data_s:0.091 +INFO 2025-05-05 15:17:14 ts/train.py:232 step:22K smpl:346K ep:2K epch:5.95 loss:0.015 grdn:0.209 lr:1.0e-04 updt_s:0.165 data_s:0.107 +INFO 2025-05-05 15:18:05 ts/train.py:232 step:22K smpl:349K ep:2K epch:6.00 loss:0.014 grdn:0.192 lr:1.0e-04 updt_s:0.164 data_s:0.095 +INFO 2025-05-05 15:18:59 ts/train.py:232 step:22K smpl:352K ep:2K epch:6.06 loss:0.014 grdn:0.188 lr:1.0e-04 updt_s:0.163 data_s:0.101 +INFO 2025-05-05 15:19:52 ts/train.py:232 step:22K smpl:355K ep:2K epch:6.11 loss:0.014 grdn:0.191 lr:1.0e-04 updt_s:0.163 data_s:0.102 +INFO 2025-05-05 15:20:39 ts/train.py:232 step:22K smpl:358K ep:2K epch:6.17 loss:0.015 grdn:0.205 lr:1.0e-04 updt_s:0.163 data_s:0.074 +INFO 2025-05-05 15:21:28 ts/train.py:232 step:23K smpl:362K ep:2K epch:6.22 loss:0.013 grdn:0.193 lr:1.0e-04 updt_s:0.165 data_s:0.075 +INFO 2025-05-05 15:22:18 ts/train.py:232 step:23K smpl:365K ep:2K epch:6.28 loss:0.014 grdn:0.193 lr:1.0e-04 updt_s:0.164 data_s:0.087 +INFO 2025-05-05 15:23:06 ts/train.py:232 step:23K smpl:368K ep:2K epch:6.33 loss:0.014 grdn:0.190 lr:1.0e-04 updt_s:0.164 data_s:0.077 +INFO 2025-05-05 15:23:57 ts/train.py:232 step:23K smpl:371K ep:2K epch:6.39 loss:0.014 grdn:0.196 lr:1.0e-04 updt_s:0.163 data_s:0.089 +INFO 2025-05-05 15:24:45 ts/train.py:232 step:23K smpl:374K ep:2K epch:6.45 loss:0.013 grdn:0.186 lr:1.0e-04 updt_s:0.163 data_s:0.074 +INFO 2025-05-05 15:25:34 ts/train.py:232 step:24K smpl:378K ep:2K epch:6.50 loss:0.014 grdn:0.195 lr:1.0e-04 updt_s:0.164 data_s:0.083 +INFO 2025-05-05 15:26:22 ts/train.py:232 step:24K smpl:381K ep:2K epch:6.56 loss:0.013 grdn:0.189 lr:1.0e-04 updt_s:0.163 data_s:0.074 +INFO 2025-05-05 15:27:09 ts/train.py:232 step:24K smpl:384K ep:2K epch:6.61 loss:0.014 grdn:0.184 lr:1.0e-04 updt_s:0.162 data_s:0.074 +INFO 2025-05-05 15:27:58 ts/train.py:232 step:24K smpl:387K ep:2K epch:6.67 loss:0.013 grdn:0.191 lr:1.0e-04 updt_s:0.163 data_s:0.083 +INFO 2025-05-05 15:28:48 ts/train.py:232 step:24K smpl:390K ep:2K epch:6.72 loss:0.013 grdn:0.180 lr:1.0e-04 updt_s:0.164 data_s:0.081 +INFO 2025-05-05 15:29:40 ts/train.py:232 step:25K smpl:394K ep:2K epch:6.78 loss:0.014 grdn:0.186 lr:1.0e-04 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 15:30:33 ts/train.py:232 step:25K smpl:397K ep:2K epch:6.83 loss:0.013 grdn:0.189 lr:1.0e-04 updt_s:0.163 data_s:0.101 +INFO 2025-05-05 15:31:26 ts/train.py:232 step:25K smpl:400K ep:2K epch:6.89 loss:0.013 grdn:0.187 lr:1.0e-04 updt_s:0.164 data_s:0.100 +INFO 2025-05-05 15:32:19 ts/train.py:232 step:25K smpl:403K ep:2K epch:6.94 loss:0.014 grdn:0.185 lr:1.0e-04 updt_s:0.163 data_s:0.104 +INFO 2025-05-05 15:33:13 ts/train.py:232 step:25K smpl:406K ep:2K epch:7.00 loss:0.013 grdn:0.185 lr:1.0e-04 updt_s:0.163 data_s:0.105 +INFO 2025-05-05 15:34:06 ts/train.py:232 step:26K smpl:410K ep:2K epch:7.05 loss:0.013 grdn:0.182 lr:1.0e-04 updt_s:0.163 data_s:0.104 +INFO 2025-05-05 15:35:00 ts/train.py:232 step:26K smpl:413K ep:2K epch:7.11 loss:0.013 grdn:0.186 lr:1.0e-04 updt_s:0.163 data_s:0.106 +INFO 2025-05-05 15:35:55 ts/train.py:232 step:26K smpl:416K ep:2K epch:7.16 loss:0.013 grdn:0.181 lr:1.0e-04 updt_s:0.163 data_s:0.108 +INFO 2025-05-05 15:36:50 ts/train.py:232 step:26K smpl:419K ep:2K epch:7.22 loss:0.013 grdn:0.193 lr:1.0e-04 updt_s:0.163 data_s:0.112 +INFO 2025-05-05 15:37:43 ts/train.py:232 step:26K smpl:422K ep:2K epch:7.27 loss:0.014 grdn:0.184 lr:1.0e-04 updt_s:0.163 data_s:0.104 +INFO 2025-05-05 15:38:37 ts/train.py:232 step:27K smpl:426K ep:2K epch:7.33 loss:0.013 grdn:0.189 lr:1.0e-04 updt_s:0.163 data_s:0.101 +INFO 2025-05-05 15:39:31 ts/train.py:232 step:27K smpl:429K ep:2K epch:7.38 loss:0.014 grdn:0.185 lr:1.0e-04 updt_s:0.163 data_s:0.107 +INFO 2025-05-05 15:40:24 ts/train.py:232 step:27K smpl:432K ep:2K epch:7.44 loss:0.012 grdn:0.178 lr:1.0e-04 updt_s:0.163 data_s:0.102 +INFO 2025-05-05 15:41:17 ts/train.py:232 step:27K smpl:435K ep:2K epch:7.49 loss:0.014 grdn:0.192 lr:1.0e-04 updt_s:0.163 data_s:0.103 +INFO 2025-05-05 15:42:12 ts/train.py:232 step:27K smpl:438K ep:2K epch:7.55 loss:0.013 grdn:0.188 lr:1.0e-04 updt_s:0.163 data_s:0.113 +INFO 2025-05-05 15:43:07 ts/train.py:232 step:28K smpl:442K ep:2K epch:7.60 loss:0.014 grdn:0.191 lr:1.0e-04 updt_s:0.163 data_s:0.106 +INFO 2025-05-05 15:44:01 ts/train.py:232 step:28K smpl:445K ep:2K epch:7.66 loss:0.013 grdn:0.182 lr:1.0e-04 updt_s:0.163 data_s:0.107 +INFO 2025-05-05 15:44:54 ts/train.py:232 step:28K smpl:448K ep:2K epch:7.71 loss:0.012 grdn:0.172 lr:1.0e-04 updt_s:0.163 data_s:0.105 +INFO 2025-05-05 15:45:47 ts/train.py:232 step:28K smpl:451K ep:2K epch:7.77 loss:0.013 grdn:0.192 lr:1.0e-04 updt_s:0.164 data_s:0.100 +INFO 2025-05-05 15:46:40 ts/train.py:232 step:28K smpl:454K ep:2K epch:7.82 loss:0.013 grdn:0.181 lr:1.0e-04 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 15:47:31 ts/train.py:232 step:29K smpl:458K ep:2K epch:7.88 loss:0.013 grdn:0.180 lr:1.0e-04 updt_s:0.164 data_s:0.095 +INFO 2025-05-05 15:48:24 ts/train.py:232 step:29K smpl:461K ep:2K epch:7.93 loss:0.013 grdn:0.171 lr:1.0e-04 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 15:49:16 ts/train.py:232 step:29K smpl:464K ep:2K epch:7.99 loss:0.013 grdn:0.183 lr:1.0e-04 updt_s:0.164 data_s:0.096 +INFO 2025-05-05 15:50:07 ts/train.py:232 step:29K smpl:467K ep:2K epch:8.04 loss:0.013 grdn:0.176 lr:1.0e-04 updt_s:0.163 data_s:0.091 +INFO 2025-05-05 15:50:56 ts/train.py:232 step:29K smpl:470K ep:2K epch:8.10 loss:0.013 grdn:0.188 lr:1.0e-04 updt_s:0.163 data_s:0.084 +INFO 2025-05-05 15:51:43 ts/train.py:232 step:30K smpl:474K ep:2K epch:8.15 loss:0.013 grdn:0.182 lr:1.0e-04 updt_s:0.163 data_s:0.070 +INFO 2025-05-05 15:52:30 ts/train.py:232 step:30K smpl:477K ep:2K epch:8.21 loss:0.012 grdn:0.174 lr:1.0e-04 updt_s:0.163 data_s:0.069 +INFO 2025-05-05 15:53:17 ts/train.py:232 step:30K smpl:480K ep:2K epch:8.26 loss:0.013 grdn:0.190 lr:1.0e-04 updt_s:0.164 data_s:0.071 +INFO 2025-05-05 15:53:17 ts/train.py:241 Checkpoint policy after step 30000 +INFO 2025-05-05 15:54:06 ts/train.py:232 step:30K smpl:483K ep:2K epch:8.32 loss:0.013 grdn:0.184 lr:1.0e-04 updt_s:0.162 data_s:0.066 +INFO 2025-05-05 15:54:53 ts/train.py:232 step:30K smpl:486K ep:3K epch:8.37 loss:0.013 grdn:0.187 lr:1.0e-04 updt_s:0.162 data_s:0.072 +INFO 2025-05-05 15:55:40 ts/train.py:232 step:31K smpl:490K ep:3K epch:8.43 loss:0.013 grdn:0.186 lr:1.0e-04 updt_s:0.163 data_s:0.071 +INFO 2025-05-05 15:56:27 ts/train.py:232 step:31K smpl:493K ep:3K epch:8.48 loss:0.012 grdn:0.167 lr:1.0e-04 updt_s:0.162 data_s:0.068 +INFO 2025-05-05 15:57:13 ts/train.py:232 step:31K smpl:496K ep:3K epch:8.54 loss:0.012 grdn:0.179 lr:1.0e-04 updt_s:0.163 data_s:0.068 +INFO 2025-05-05 15:58:00 ts/train.py:232 step:31K smpl:499K ep:3K epch:8.59 loss:0.014 grdn:0.190 lr:1.0e-04 updt_s:0.163 data_s:0.069 +INFO 2025-05-05 15:58:48 ts/train.py:232 step:31K smpl:502K ep:3K epch:8.65 loss:0.012 grdn:0.183 lr:1.0e-04 updt_s:0.164 data_s:0.076 +INFO 2025-05-05 15:59:39 ts/train.py:232 step:32K smpl:506K ep:3K epch:8.70 loss:0.012 grdn:0.190 lr:1.0e-04 updt_s:0.162 data_s:0.091 +INFO 2025-05-05 16:00:32 ts/train.py:232 step:32K smpl:509K ep:3K epch:8.76 loss:0.013 grdn:0.182 lr:1.0e-04 updt_s:0.163 data_s:0.102 +INFO 2025-05-05 16:01:25 ts/train.py:232 step:32K smpl:512K ep:3K epch:8.81 loss:0.012 grdn:0.177 lr:1.0e-04 updt_s:0.163 data_s:0.102 +INFO 2025-05-05 16:02:18 ts/train.py:232 step:32K smpl:515K ep:3K epch:8.87 loss:0.012 grdn:0.171 lr:1.0e-04 updt_s:0.163 data_s:0.101 +INFO 2025-05-05 16:03:12 ts/train.py:232 step:32K smpl:518K ep:3K epch:8.92 loss:0.012 grdn:0.183 lr:1.0e-04 updt_s:0.163 data_s:0.105 +INFO 2025-05-05 16:04:05 ts/train.py:232 step:33K smpl:522K ep:3K epch:8.98 loss:0.012 grdn:0.176 lr:1.0e-04 updt_s:0.163 data_s:0.102 +INFO 2025-05-05 16:04:58 ts/train.py:232 step:33K smpl:525K ep:3K epch:9.03 loss:0.012 grdn:0.178 lr:1.0e-04 updt_s:0.162 data_s:0.099 +INFO 2025-05-05 16:05:49 ts/train.py:232 step:33K smpl:528K ep:3K epch:9.09 loss:0.012 grdn:0.179 lr:1.0e-04 updt_s:0.162 data_s:0.096 +INFO 2025-05-05 16:06:41 ts/train.py:232 step:33K smpl:531K ep:3K epch:9.14 loss:0.012 grdn:0.179 lr:1.0e-04 updt_s:0.163 data_s:0.095 +INFO 2025-05-05 16:07:33 ts/train.py:232 step:33K smpl:534K ep:3K epch:9.20 loss:0.012 grdn:0.185 lr:1.0e-04 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 16:08:24 ts/train.py:232 step:34K smpl:538K ep:3K epch:9.25 loss:0.012 grdn:0.181 lr:1.0e-04 updt_s:0.163 data_s:0.092 +INFO 2025-05-05 16:09:16 ts/train.py:232 step:34K smpl:541K ep:3K epch:9.31 loss:0.013 grdn:0.177 lr:1.0e-04 updt_s:0.162 data_s:0.095 +INFO 2025-05-05 16:10:08 ts/train.py:232 step:34K smpl:544K ep:3K epch:9.36 loss:0.012 grdn:0.174 lr:1.0e-04 updt_s:0.162 data_s:0.099 +INFO 2025-05-05 16:11:00 ts/train.py:232 step:34K smpl:547K ep:3K epch:9.42 loss:0.013 grdn:0.188 lr:1.0e-04 updt_s:0.161 data_s:0.096 +INFO 2025-05-05 16:11:51 ts/train.py:232 step:34K smpl:550K ep:3K epch:9.47 loss:0.012 grdn:0.176 lr:1.0e-04 updt_s:0.162 data_s:0.093 +INFO 2025-05-05 16:12:43 ts/train.py:232 step:35K smpl:554K ep:3K epch:9.53 loss:0.012 grdn:0.180 lr:1.0e-04 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 16:13:36 ts/train.py:232 step:35K smpl:557K ep:3K epch:9.59 loss:0.013 grdn:0.189 lr:1.0e-04 updt_s:0.163 data_s:0.098 +INFO 2025-05-05 16:14:28 ts/train.py:232 step:35K smpl:560K ep:3K epch:9.64 loss:0.011 grdn:0.161 lr:1.0e-04 updt_s:0.163 data_s:0.097 +INFO 2025-05-05 16:15:22 ts/train.py:232 step:35K smpl:563K ep:3K epch:9.70 loss:0.012 grdn:0.167 lr:1.0e-04 updt_s:0.163 data_s:0.110 +INFO 2025-05-05 16:16:16 ts/train.py:232 step:35K smpl:566K ep:3K epch:9.75 loss:0.012 grdn:0.179 lr:1.0e-04 updt_s:0.163 data_s:0.105 +INFO 2025-05-05 16:17:09 ts/train.py:232 step:36K smpl:570K ep:3K epch:9.81 loss:0.012 grdn:0.173 lr:1.0e-04 updt_s:0.163 data_s:0.103 +INFO 2025-05-05 16:18:03 ts/train.py:232 step:36K smpl:573K ep:3K epch:9.86 loss:0.011 grdn:0.166 lr:1.0e-04 updt_s:0.163 data_s:0.103 +INFO 2025-05-05 16:18:56 ts/train.py:232 step:36K smpl:576K ep:3K epch:9.92 loss:0.011 grdn:0.174 lr:1.0e-04 updt_s:0.164 data_s:0.101 +INFO 2025-05-05 16:19:50 ts/train.py:232 step:36K smpl:579K ep:3K epch:9.97 loss:0.012 grdn:0.175 lr:1.0e-04 updt_s:0.163 data_s:0.104 +INFO 2025-05-05 16:20:42 ts/train.py:232 step:36K smpl:582K ep:3K epch:10.03 loss:0.012 grdn:0.178 lr:1.0e-04 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 16:21:33 ts/train.py:232 step:37K smpl:586K ep:3K epch:10.08 loss:0.012 grdn:0.173 lr:1.0e-04 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 16:22:26 ts/train.py:232 step:37K smpl:589K ep:3K epch:10.14 loss:0.012 grdn:0.179 lr:1.0e-04 updt_s:0.163 data_s:0.098 +INFO 2025-05-05 16:23:18 ts/train.py:232 step:37K smpl:592K ep:3K epch:10.19 loss:0.012 grdn:0.182 lr:1.0e-04 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 16:24:10 ts/train.py:232 step:37K smpl:595K ep:3K epch:10.25 loss:0.012 grdn:0.177 lr:1.0e-04 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 16:25:02 ts/train.py:232 step:37K smpl:598K ep:3K epch:10.30 loss:0.011 grdn:0.169 lr:1.0e-04 updt_s:0.163 data_s:0.095 +INFO 2025-05-05 16:25:54 ts/train.py:232 step:38K smpl:602K ep:3K epch:10.36 loss:0.012 grdn:0.181 lr:1.0e-04 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 16:26:46 ts/train.py:232 step:38K smpl:605K ep:3K epch:10.41 loss:0.012 grdn:0.177 lr:1.0e-04 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 16:27:38 ts/train.py:232 step:38K smpl:608K ep:3K epch:10.47 loss:0.011 grdn:0.167 lr:1.0e-04 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 16:28:30 ts/train.py:232 step:38K smpl:611K ep:3K epch:10.52 loss:0.012 grdn:0.168 lr:1.0e-04 updt_s:0.163 data_s:0.098 +INFO 2025-05-05 16:29:23 ts/train.py:232 step:38K smpl:614K ep:3K epch:10.58 loss:0.012 grdn:0.173 lr:1.0e-04 updt_s:0.163 data_s:0.098 +INFO 2025-05-05 16:30:16 ts/train.py:232 step:39K smpl:618K ep:3K epch:10.63 loss:0.012 grdn:0.176 lr:1.0e-04 updt_s:0.163 data_s:0.100 +INFO 2025-05-05 16:31:05 ts/train.py:232 step:39K smpl:621K ep:3K epch:10.69 loss:0.011 grdn:0.175 lr:1.0e-04 updt_s:0.164 data_s:0.081 +INFO 2025-05-05 16:31:54 ts/train.py:232 step:39K smpl:624K ep:3K epch:10.74 loss:0.012 grdn:0.180 lr:1.0e-04 updt_s:0.163 data_s:0.081 +INFO 2025-05-05 16:32:42 ts/train.py:232 step:39K smpl:627K ep:3K epch:10.80 loss:0.012 grdn:0.177 lr:1.0e-04 updt_s:0.163 data_s:0.075 +INFO 2025-05-05 16:33:31 ts/train.py:232 step:39K smpl:630K ep:3K epch:10.85 loss:0.012 grdn:0.163 lr:1.0e-04 updt_s:0.164 data_s:0.083 +INFO 2025-05-05 16:34:21 ts/train.py:232 step:40K smpl:634K ep:3K epch:10.91 loss:0.011 grdn:0.171 lr:1.0e-04 updt_s:0.163 data_s:0.082 +INFO 2025-05-05 16:35:11 ts/train.py:232 step:40K smpl:637K ep:3K epch:10.96 loss:0.011 grdn:0.172 lr:1.0e-04 updt_s:0.164 data_s:0.089 +INFO 2025-05-05 16:36:00 ts/train.py:232 step:40K smpl:640K ep:3K epch:11.02 loss:0.012 grdn:0.172 lr:1.0e-04 updt_s:0.163 data_s:0.081 +INFO 2025-05-05 16:36:50 ts/train.py:232 step:40K smpl:643K ep:3K epch:11.07 loss:0.012 grdn:0.173 lr:1.0e-04 updt_s:0.163 data_s:0.088 +INFO 2025-05-05 16:37:39 ts/train.py:232 step:40K smpl:646K ep:3K epch:11.13 loss:0.012 grdn:0.170 lr:1.0e-04 updt_s:0.163 data_s:0.077 +INFO 2025-05-05 16:38:27 ts/train.py:232 step:41K smpl:650K ep:3K epch:11.18 loss:0.011 grdn:0.169 lr:1.0e-04 updt_s:0.162 data_s:0.080 +INFO 2025-05-05 16:39:15 ts/train.py:232 step:41K smpl:653K ep:3K epch:11.24 loss:0.011 grdn:0.171 lr:1.0e-04 updt_s:0.161 data_s:0.079 +INFO 2025-05-05 16:40:04 ts/train.py:232 step:41K smpl:656K ep:3K epch:11.29 loss:0.012 grdn:0.175 lr:1.0e-04 updt_s:0.162 data_s:0.079 +INFO 2025-05-05 16:40:52 ts/train.py:232 step:41K smpl:659K ep:3K epch:11.35 loss:0.011 grdn:0.162 lr:9.9e-05 updt_s:0.161 data_s:0.079 +INFO 2025-05-05 16:41:41 ts/train.py:232 step:41K smpl:662K ep:3K epch:11.40 loss:0.012 grdn:0.173 lr:9.9e-05 updt_s:0.161 data_s:0.084 +INFO 2025-05-05 16:42:29 ts/train.py:232 step:42K smpl:666K ep:3K epch:11.46 loss:0.012 grdn:0.173 lr:9.9e-05 updt_s:0.162 data_s:0.077 +INFO 2025-05-05 16:43:18 ts/train.py:232 step:42K smpl:669K ep:3K epch:11.51 loss:0.011 grdn:0.165 lr:9.9e-05 updt_s:0.162 data_s:0.081 +INFO 2025-05-05 16:44:06 ts/train.py:232 step:42K smpl:672K ep:3K epch:11.57 loss:0.012 grdn:0.180 lr:9.9e-05 updt_s:0.161 data_s:0.078 +INFO 2025-05-05 16:45:02 ts/train.py:232 step:42K smpl:675K ep:3K epch:11.62 loss:0.011 grdn:0.172 lr:9.9e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 16:45:54 ts/train.py:232 step:42K smpl:678K ep:4K epch:11.68 loss:0.011 grdn:0.171 lr:9.9e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 16:46:47 ts/train.py:232 step:43K smpl:682K ep:4K epch:11.73 loss:0.011 grdn:0.171 lr:9.9e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 16:47:40 ts/train.py:232 step:43K smpl:685K ep:4K epch:11.79 loss:0.012 grdn:0.168 lr:9.9e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-05 16:48:34 ts/train.py:232 step:43K smpl:688K ep:4K epch:11.84 loss:0.012 grdn:0.172 lr:9.9e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 16:49:27 ts/train.py:232 step:43K smpl:691K ep:4K epch:11.90 loss:0.011 grdn:0.164 lr:9.9e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-05 16:50:21 ts/train.py:232 step:43K smpl:694K ep:4K epch:11.95 loss:0.011 grdn:0.167 lr:9.9e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-05 16:51:15 ts/train.py:232 step:44K smpl:698K ep:4K epch:12.01 loss:0.010 grdn:0.156 lr:9.9e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 16:52:08 ts/train.py:232 step:44K smpl:701K ep:4K epch:12.06 loss:0.011 grdn:0.166 lr:9.9e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-05 16:53:02 ts/train.py:232 step:44K smpl:704K ep:4K epch:12.12 loss:0.011 grdn:0.182 lr:9.9e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-05 16:53:54 ts/train.py:232 step:44K smpl:707K ep:4K epch:12.17 loss:0.012 grdn:0.177 lr:9.9e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-05 16:54:46 ts/train.py:232 step:44K smpl:710K ep:4K epch:12.23 loss:0.012 grdn:0.177 lr:9.9e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 16:55:39 ts/train.py:232 step:45K smpl:714K ep:4K epch:12.28 loss:0.011 grdn:0.163 lr:9.9e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-05 16:56:31 ts/train.py:232 step:45K smpl:717K ep:4K epch:12.34 loss:0.011 grdn:0.166 lr:9.9e-05 updt_s:0.162 data_s:0.097 +INFO 2025-05-05 16:57:23 ts/train.py:232 step:45K smpl:720K ep:4K epch:12.39 loss:0.012 grdn:0.174 lr:9.9e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-05 16:58:16 ts/train.py:232 step:45K smpl:723K ep:4K epch:12.45 loss:0.012 grdn:0.176 lr:9.9e-05 updt_s:0.163 data_s:0.099 +INFO 2025-05-05 16:59:08 ts/train.py:232 step:45K smpl:726K ep:4K epch:12.50 loss:0.011 grdn:0.168 lr:9.9e-05 updt_s:0.163 data_s:0.099 +INFO 2025-05-05 17:00:00 ts/train.py:232 step:46K smpl:730K ep:4K epch:12.56 loss:0.011 grdn:0.161 lr:9.9e-05 updt_s:0.163 data_s:0.098 +INFO 2025-05-05 17:00:49 ts/train.py:232 step:46K smpl:733K ep:4K epch:12.61 loss:0.011 grdn:0.171 lr:9.9e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-05 17:01:39 ts/train.py:232 step:46K smpl:736K ep:4K epch:12.67 loss:0.011 grdn:0.174 lr:9.9e-05 updt_s:0.163 data_s:0.083 +INFO 2025-05-05 17:02:27 ts/train.py:232 step:46K smpl:739K ep:4K epch:12.73 loss:0.011 grdn:0.162 lr:9.9e-05 updt_s:0.163 data_s:0.076 +INFO 2025-05-05 17:03:18 ts/train.py:232 step:46K smpl:742K ep:4K epch:12.78 loss:0.011 grdn:0.165 lr:9.9e-05 updt_s:0.163 data_s:0.094 +INFO 2025-05-05 17:04:10 ts/train.py:232 step:47K smpl:746K ep:4K epch:12.84 loss:0.010 grdn:0.161 lr:9.9e-05 updt_s:0.163 data_s:0.093 +INFO 2025-05-05 17:05:00 ts/train.py:232 step:47K smpl:749K ep:4K epch:12.89 loss:0.011 grdn:0.160 lr:9.9e-05 updt_s:0.163 data_s:0.089 +INFO 2025-05-05 17:05:51 ts/train.py:232 step:47K smpl:752K ep:4K epch:12.95 loss:0.010 grdn:0.162 lr:9.9e-05 updt_s:0.163 data_s:0.088 +INFO 2025-05-05 17:06:41 ts/train.py:232 step:47K smpl:755K ep:4K epch:13.00 loss:0.010 grdn:0.163 lr:9.9e-05 updt_s:0.163 data_s:0.087 +INFO 2025-05-05 17:07:31 ts/train.py:232 step:47K smpl:758K ep:4K epch:13.06 loss:0.011 grdn:0.175 lr:9.9e-05 updt_s:0.163 data_s:0.086 +INFO 2025-05-05 17:08:20 ts/train.py:232 step:48K smpl:762K ep:4K epch:13.11 loss:0.011 grdn:0.166 lr:9.9e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-05 17:09:08 ts/train.py:232 step:48K smpl:765K ep:4K epch:13.17 loss:0.011 grdn:0.170 lr:9.9e-05 updt_s:0.161 data_s:0.082 +INFO 2025-05-05 17:09:58 ts/train.py:232 step:48K smpl:768K ep:4K epch:13.22 loss:0.011 grdn:0.176 lr:9.9e-05 updt_s:0.163 data_s:0.086 +INFO 2025-05-05 17:10:47 ts/train.py:232 step:48K smpl:771K ep:4K epch:13.28 loss:0.011 grdn:0.167 lr:9.9e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-05 17:11:37 ts/train.py:232 step:48K smpl:774K ep:4K epch:13.33 loss:0.011 grdn:0.171 lr:9.9e-05 updt_s:0.162 data_s:0.084 +INFO 2025-05-05 17:12:27 ts/train.py:232 step:49K smpl:778K ep:4K epch:13.39 loss:0.011 grdn:0.160 lr:9.9e-05 updt_s:0.162 data_s:0.087 +INFO 2025-05-05 17:13:18 ts/train.py:232 step:49K smpl:781K ep:4K epch:13.44 loss:0.011 grdn:0.166 lr:9.9e-05 updt_s:0.163 data_s:0.092 +INFO 2025-05-05 17:14:06 ts/train.py:232 step:49K smpl:784K ep:4K epch:13.50 loss:0.012 grdn:0.176 lr:9.9e-05 updt_s:0.162 data_s:0.078 +INFO 2025-05-05 17:15:03 ts/train.py:232 step:49K smpl:787K ep:4K epch:13.55 loss:0.011 grdn:0.165 lr:9.9e-05 updt_s:0.162 data_s:0.123 +INFO 2025-05-05 17:15:58 ts/train.py:232 step:49K smpl:790K ep:4K epch:13.61 loss:0.010 grdn:0.159 lr:9.9e-05 updt_s:0.162 data_s:0.113 +INFO 2025-05-05 17:16:54 ts/train.py:232 step:50K smpl:794K ep:4K epch:13.66 loss:0.011 grdn:0.165 lr:9.9e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-05 17:17:49 ts/train.py:232 step:50K smpl:797K ep:4K epch:13.72 loss:0.011 grdn:0.166 lr:9.9e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 17:18:45 ts/train.py:232 step:50K smpl:800K ep:4K epch:13.77 loss:0.010 grdn:0.161 lr:9.9e-05 updt_s:0.162 data_s:0.114 +INFO 2025-05-05 17:19:41 ts/train.py:232 step:50K smpl:803K ep:4K epch:13.83 loss:0.010 grdn:0.166 lr:9.9e-05 updt_s:0.162 data_s:0.117 +INFO 2025-05-05 17:20:35 ts/train.py:232 step:50K smpl:806K ep:4K epch:13.88 loss:0.010 grdn:0.164 lr:9.9e-05 updt_s:0.163 data_s:0.107 +INFO 2025-05-05 17:21:29 ts/train.py:232 step:51K smpl:810K ep:4K epch:13.94 loss:0.011 grdn:0.158 lr:9.9e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 17:22:24 ts/train.py:232 step:51K smpl:813K ep:4K epch:13.99 loss:0.010 grdn:0.152 lr:9.9e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-05 17:23:18 ts/train.py:232 step:51K smpl:816K ep:4K epch:14.05 loss:0.010 grdn:0.163 lr:9.9e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-05 17:24:12 ts/train.py:232 step:51K smpl:819K ep:4K epch:14.10 loss:0.010 grdn:0.164 lr:9.9e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-05 17:25:04 ts/train.py:232 step:51K smpl:822K ep:4K epch:14.16 loss:0.010 grdn:0.158 lr:9.9e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-05 17:25:57 ts/train.py:232 step:52K smpl:826K ep:4K epch:14.21 loss:0.010 grdn:0.166 lr:9.9e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 17:26:50 ts/train.py:232 step:52K smpl:829K ep:4K epch:14.27 loss:0.011 grdn:0.170 lr:9.9e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-05 17:27:43 ts/train.py:232 step:52K smpl:832K ep:4K epch:14.32 loss:0.011 grdn:0.167 lr:9.9e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 17:28:37 ts/train.py:232 step:52K smpl:835K ep:4K epch:14.38 loss:0.012 grdn:0.178 lr:9.9e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-05 17:29:30 ts/train.py:232 step:52K smpl:838K ep:4K epch:14.43 loss:0.010 grdn:0.164 lr:9.9e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-05 17:30:22 ts/train.py:232 step:53K smpl:842K ep:4K epch:14.49 loss:0.011 grdn:0.175 lr:9.9e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-05 17:31:11 ts/train.py:232 step:53K smpl:845K ep:4K epch:14.54 loss:0.010 grdn:0.155 lr:9.9e-05 updt_s:0.162 data_s:0.079 +INFO 2025-05-05 17:31:59 ts/train.py:232 step:53K smpl:848K ep:4K epch:14.60 loss:0.011 grdn:0.162 lr:9.9e-05 updt_s:0.162 data_s:0.078 +INFO 2025-05-05 17:32:48 ts/train.py:232 step:53K smpl:851K ep:4K epch:14.65 loss:0.011 grdn:0.164 lr:9.9e-05 updt_s:0.162 data_s:0.081 +INFO 2025-05-05 17:33:36 ts/train.py:232 step:53K smpl:854K ep:4K epch:14.71 loss:0.011 grdn:0.165 lr:9.9e-05 updt_s:0.164 data_s:0.079 +INFO 2025-05-05 17:34:25 ts/train.py:232 step:54K smpl:858K ep:4K epch:14.76 loss:0.010 grdn:0.163 lr:9.9e-05 updt_s:0.162 data_s:0.081 +INFO 2025-05-05 17:35:15 ts/train.py:232 step:54K smpl:861K ep:4K epch:14.82 loss:0.010 grdn:0.155 lr:9.9e-05 updt_s:0.163 data_s:0.084 +INFO 2025-05-05 17:36:03 ts/train.py:232 step:54K smpl:864K ep:4K epch:14.87 loss:0.011 grdn:0.168 lr:9.9e-05 updt_s:0.163 data_s:0.079 +INFO 2025-05-05 17:36:51 ts/train.py:232 step:54K smpl:867K ep:4K epch:14.93 loss:0.010 grdn:0.163 lr:9.9e-05 updt_s:0.162 data_s:0.078 +INFO 2025-05-05 17:37:39 ts/train.py:232 step:54K smpl:870K ep:4K epch:14.98 loss:0.010 grdn:0.169 lr:9.9e-05 updt_s:0.162 data_s:0.077 +INFO 2025-05-05 17:38:28 ts/train.py:232 step:55K smpl:874K ep:5K epch:15.04 loss:0.010 grdn:0.160 lr:9.9e-05 updt_s:0.163 data_s:0.080 +INFO 2025-05-05 17:39:19 ts/train.py:232 step:55K smpl:877K ep:5K epch:15.09 loss:0.010 grdn:0.157 lr:9.9e-05 updt_s:0.162 data_s:0.091 +INFO 2025-05-05 17:40:09 ts/train.py:232 step:55K smpl:880K ep:5K epch:15.15 loss:0.010 grdn:0.158 lr:9.9e-05 updt_s:0.162 data_s:0.085 +INFO 2025-05-05 17:40:58 ts/train.py:232 step:55K smpl:883K ep:5K epch:15.20 loss:0.011 grdn:0.171 lr:9.9e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-05 17:41:45 ts/train.py:232 step:55K smpl:886K ep:5K epch:15.26 loss:0.011 grdn:0.177 lr:9.9e-05 updt_s:0.162 data_s:0.076 +INFO 2025-05-05 17:42:34 ts/train.py:232 step:56K smpl:890K ep:5K epch:15.31 loss:0.011 grdn:0.165 lr:9.9e-05 updt_s:0.162 data_s:0.079 +INFO 2025-05-05 17:43:22 ts/train.py:232 step:56K smpl:893K ep:5K epch:15.37 loss:0.010 grdn:0.167 lr:9.9e-05 updt_s:0.163 data_s:0.079 +INFO 2025-05-05 17:44:10 ts/train.py:232 step:56K smpl:896K ep:5K epch:15.42 loss:0.010 grdn:0.162 lr:9.9e-05 updt_s:0.161 data_s:0.078 +INFO 2025-05-05 17:45:03 ts/train.py:232 step:56K smpl:899K ep:5K epch:15.48 loss:0.010 grdn:0.158 lr:9.9e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 17:45:56 ts/train.py:232 step:56K smpl:902K ep:5K epch:15.53 loss:0.010 grdn:0.153 lr:9.9e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 17:46:47 ts/train.py:232 step:57K smpl:906K ep:5K epch:15.59 loss:0.011 grdn:0.176 lr:9.9e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-05 17:47:39 ts/train.py:232 step:57K smpl:909K ep:5K epch:15.64 loss:0.010 grdn:0.165 lr:9.9e-05 updt_s:0.162 data_s:0.093 +INFO 2025-05-05 17:48:31 ts/train.py:232 step:57K smpl:912K ep:5K epch:15.70 loss:0.011 grdn:0.167 lr:9.9e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-05 17:49:22 ts/train.py:232 step:57K smpl:915K ep:5K epch:15.75 loss:0.010 grdn:0.165 lr:9.9e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-05 17:50:15 ts/train.py:232 step:57K smpl:918K ep:5K epch:15.81 loss:0.011 grdn:0.161 lr:9.9e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-05 17:51:08 ts/train.py:232 step:58K smpl:922K ep:5K epch:15.87 loss:0.010 grdn:0.169 lr:9.9e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-05 17:52:00 ts/train.py:232 step:58K smpl:925K ep:5K epch:15.92 loss:0.011 grdn:0.161 lr:9.9e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 17:52:53 ts/train.py:232 step:58K smpl:928K ep:5K epch:15.98 loss:0.011 grdn:0.175 lr:9.9e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-05 17:53:46 ts/train.py:232 step:58K smpl:931K ep:5K epch:16.03 loss:0.010 grdn:0.161 lr:9.9e-05 updt_s:0.162 data_s:0.100 +INFO 2025-05-05 17:54:39 ts/train.py:232 step:58K smpl:934K ep:5K epch:16.09 loss:0.010 grdn:0.161 lr:9.9e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 17:55:31 ts/train.py:232 step:59K smpl:938K ep:5K epch:16.14 loss:0.010 grdn:0.156 lr:9.9e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 17:56:31 ts/train.py:232 step:59K smpl:941K ep:5K epch:16.20 loss:0.010 grdn:0.163 lr:9.9e-05 updt_s:0.168 data_s:0.130 +INFO 2025-05-05 17:57:35 ts/train.py:232 step:59K smpl:944K ep:5K epch:16.25 loss:0.010 grdn:0.157 lr:9.9e-05 updt_s:0.235 data_s:0.085 +INFO 2025-05-05 17:58:28 ts/train.py:232 step:59K smpl:947K ep:5K epch:16.31 loss:0.010 grdn:0.162 lr:9.9e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 17:59:21 ts/train.py:232 step:59K smpl:950K ep:5K epch:16.36 loss:0.010 grdn:0.161 lr:9.9e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 18:00:17 ts/train.py:232 step:60K smpl:954K ep:5K epch:16.42 loss:0.010 grdn:0.156 lr:9.9e-05 updt_s:0.161 data_s:0.115 +INFO 2025-05-05 18:01:12 ts/train.py:232 step:60K smpl:957K ep:5K epch:16.47 loss:0.011 grdn:0.173 lr:9.9e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-05 18:02:09 ts/train.py:232 step:60K smpl:960K ep:5K epch:16.53 loss:0.010 grdn:0.159 lr:9.9e-05 updt_s:0.162 data_s:0.123 +INFO 2025-05-05 18:02:09 ts/train.py:241 Checkpoint policy after step 60000 +INFO 2025-05-05 18:03:07 ts/train.py:232 step:60K smpl:963K ep:5K epch:16.58 loss:0.011 grdn:0.167 lr:9.9e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 18:04:06 ts/train.py:232 step:60K smpl:966K ep:5K epch:16.64 loss:0.010 grdn:0.153 lr:9.9e-05 updt_s:0.163 data_s:0.129 +INFO 2025-05-05 18:05:05 ts/train.py:232 step:61K smpl:970K ep:5K epch:16.69 loss:0.010 grdn:0.158 lr:9.9e-05 updt_s:0.163 data_s:0.133 +INFO 2025-05-05 18:06:02 ts/train.py:232 step:61K smpl:973K ep:5K epch:16.75 loss:0.010 grdn:0.154 lr:9.9e-05 updt_s:0.162 data_s:0.124 +INFO 2025-05-05 18:06:59 ts/train.py:232 step:61K smpl:976K ep:5K epch:16.80 loss:0.010 grdn:0.156 lr:9.9e-05 updt_s:0.163 data_s:0.120 +INFO 2025-05-05 18:07:57 ts/train.py:232 step:61K smpl:979K ep:5K epch:16.86 loss:0.010 grdn:0.163 lr:9.9e-05 updt_s:0.163 data_s:0.126 +INFO 2025-05-05 18:08:55 ts/train.py:232 step:61K smpl:982K ep:5K epch:16.91 loss:0.009 grdn:0.152 lr:9.9e-05 updt_s:0.163 data_s:0.127 +INFO 2025-05-05 18:09:51 ts/train.py:232 step:62K smpl:986K ep:5K epch:16.97 loss:0.010 grdn:0.157 lr:9.9e-05 updt_s:0.162 data_s:0.118 +INFO 2025-05-05 18:10:47 ts/train.py:232 step:62K smpl:989K ep:5K epch:17.02 loss:0.009 grdn:0.160 lr:9.9e-05 updt_s:0.162 data_s:0.115 +INFO 2025-05-05 18:11:41 ts/train.py:232 step:62K smpl:992K ep:5K epch:17.08 loss:0.010 grdn:0.161 lr:9.9e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 18:12:35 ts/train.py:232 step:62K smpl:995K ep:5K epch:17.13 loss:0.009 grdn:0.152 lr:9.9e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 18:13:29 ts/train.py:232 step:62K smpl:998K ep:5K epch:17.19 loss:0.011 grdn:0.170 lr:9.9e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-05 18:14:24 ts/train.py:232 step:63K smpl:1M ep:5K epch:17.24 loss:0.009 grdn:0.146 lr:9.9e-05 updt_s:0.163 data_s:0.113 +INFO 2025-05-05 18:15:19 ts/train.py:232 step:63K smpl:1M ep:5K epch:17.30 loss:0.010 grdn:0.157 lr:9.9e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-05 18:16:13 ts/train.py:232 step:63K smpl:1M ep:5K epch:17.35 loss:0.010 grdn:0.166 lr:9.9e-05 updt_s:0.163 data_s:0.106 +INFO 2025-05-05 18:17:09 ts/train.py:232 step:63K smpl:1M ep:5K epch:17.41 loss:0.009 grdn:0.155 lr:9.9e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 18:18:04 ts/train.py:232 step:63K smpl:1M ep:5K epch:17.46 loss:0.010 grdn:0.156 lr:9.9e-05 updt_s:0.163 data_s:0.109 +INFO 2025-05-05 18:18:59 ts/train.py:232 step:64K smpl:1M ep:5K epch:17.52 loss:0.011 grdn:0.167 lr:9.9e-05 updt_s:0.163 data_s:0.113 +INFO 2025-05-05 18:19:55 ts/train.py:232 step:64K smpl:1M ep:5K epch:17.57 loss:0.011 grdn:0.170 lr:9.9e-05 updt_s:0.163 data_s:0.118 +INFO 2025-05-05 18:20:51 ts/train.py:232 step:64K smpl:1M ep:5K epch:17.63 loss:0.010 grdn:0.157 lr:9.9e-05 updt_s:0.163 data_s:0.116 +INFO 2025-05-05 18:21:50 ts/train.py:232 step:64K smpl:1M ep:5K epch:17.68 loss:0.010 grdn:0.153 lr:9.9e-05 updt_s:0.163 data_s:0.128 +INFO 2025-05-05 18:22:44 ts/train.py:232 step:64K smpl:1M ep:5K epch:17.74 loss:0.009 grdn:0.151 lr:9.9e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 18:23:39 ts/train.py:232 step:65K smpl:1M ep:5K epch:17.79 loss:0.009 grdn:0.147 lr:9.9e-05 updt_s:0.163 data_s:0.111 +INFO 2025-05-05 18:24:35 ts/train.py:232 step:65K smpl:1M ep:5K epch:17.85 loss:0.010 grdn:0.161 lr:9.9e-05 updt_s:0.163 data_s:0.116 +INFO 2025-05-05 18:25:29 ts/train.py:232 step:65K smpl:1M ep:5K epch:17.90 loss:0.010 grdn:0.156 lr:9.9e-05 updt_s:0.163 data_s:0.110 +INFO 2025-05-05 18:26:22 ts/train.py:232 step:65K smpl:1M ep:5K epch:17.96 loss:0.010 grdn:0.159 lr:9.9e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-05 18:27:15 ts/train.py:232 step:65K smpl:1M ep:5K epch:18.01 loss:0.010 grdn:0.157 lr:9.9e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-05 18:28:08 ts/train.py:232 step:66K smpl:1M ep:5K epch:18.07 loss:0.010 grdn:0.157 lr:9.9e-05 updt_s:0.162 data_s:0.100 +INFO 2025-05-05 18:29:01 ts/train.py:232 step:66K smpl:1M ep:5K epch:18.12 loss:0.010 grdn:0.162 lr:9.9e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-05 18:29:55 ts/train.py:232 step:66K smpl:1M ep:5K epch:18.18 loss:0.010 grdn:0.170 lr:9.9e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-05 18:30:48 ts/train.py:232 step:66K smpl:1M ep:5K epch:18.23 loss:0.010 grdn:0.154 lr:9.9e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 18:31:41 ts/train.py:232 step:66K smpl:1M ep:5K epch:18.29 loss:0.009 grdn:0.150 lr:9.9e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 18:32:37 ts/train.py:232 step:67K smpl:1M ep:6K epch:18.34 loss:0.010 grdn:0.165 lr:9.9e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 18:33:32 ts/train.py:232 step:67K smpl:1M ep:6K epch:18.40 loss:0.010 grdn:0.159 lr:9.9e-05 updt_s:0.162 data_s:0.114 +INFO 2025-05-05 18:34:28 ts/train.py:232 step:67K smpl:1M ep:6K epch:18.45 loss:0.009 grdn:0.156 lr:9.9e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 18:35:24 ts/train.py:232 step:67K smpl:1M ep:6K epch:18.51 loss:0.009 grdn:0.152 lr:9.9e-05 updt_s:0.162 data_s:0.118 +INFO 2025-05-05 18:36:19 ts/train.py:232 step:67K smpl:1M ep:6K epch:18.56 loss:0.010 grdn:0.170 lr:9.9e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-05 18:37:14 ts/train.py:232 step:68K smpl:1M ep:6K epch:18.62 loss:0.010 grdn:0.160 lr:9.9e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-05 18:38:07 ts/train.py:232 step:68K smpl:1M ep:6K epch:18.67 loss:0.009 grdn:0.151 lr:9.9e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-05 18:39:02 ts/train.py:232 step:68K smpl:1M ep:6K epch:18.73 loss:0.010 grdn:0.156 lr:9.9e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-05 18:39:57 ts/train.py:232 step:68K smpl:1M ep:6K epch:18.78 loss:0.010 grdn:0.162 lr:9.9e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-05 18:40:51 ts/train.py:232 step:68K smpl:1M ep:6K epch:18.84 loss:0.009 grdn:0.155 lr:9.9e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-05 18:41:48 ts/train.py:232 step:69K smpl:1M ep:6K epch:18.89 loss:0.009 grdn:0.153 lr:9.9e-05 updt_s:0.161 data_s:0.122 +INFO 2025-05-05 18:42:45 ts/train.py:232 step:69K smpl:1M ep:6K epch:18.95 loss:0.009 grdn:0.154 lr:9.9e-05 updt_s:0.161 data_s:0.120 +INFO 2025-05-05 18:43:42 ts/train.py:232 step:69K smpl:1M ep:6K epch:19.00 loss:0.010 grdn:0.157 lr:9.9e-05 updt_s:0.161 data_s:0.123 +INFO 2025-05-05 18:44:37 ts/train.py:232 step:69K smpl:1M ep:6K epch:19.06 loss:0.010 grdn:0.158 lr:9.9e-05 updt_s:0.162 data_s:0.117 +INFO 2025-05-05 18:45:34 ts/train.py:232 step:69K smpl:1M ep:6K epch:19.12 loss:0.009 grdn:0.148 lr:9.9e-05 updt_s:0.161 data_s:0.121 +INFO 2025-05-05 18:46:31 ts/train.py:232 step:70K smpl:1M ep:6K epch:19.17 loss:0.009 grdn:0.153 lr:9.9e-05 updt_s:0.161 data_s:0.121 +INFO 2025-05-05 18:47:27 ts/train.py:232 step:70K smpl:1M ep:6K epch:19.23 loss:0.009 grdn:0.155 lr:9.9e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-05 18:48:24 ts/train.py:232 step:70K smpl:1M ep:6K epch:19.28 loss:0.010 grdn:0.159 lr:9.9e-05 updt_s:0.161 data_s:0.123 +INFO 2025-05-05 18:49:24 ts/train.py:232 step:70K smpl:1M ep:6K epch:19.34 loss:0.010 grdn:0.160 lr:9.9e-05 updt_s:0.161 data_s:0.137 +INFO 2025-05-05 18:50:22 ts/train.py:232 step:70K smpl:1M ep:6K epch:19.39 loss:0.010 grdn:0.153 lr:9.9e-05 updt_s:0.161 data_s:0.131 +INFO 2025-05-05 18:51:21 ts/train.py:232 step:71K smpl:1M ep:6K epch:19.45 loss:0.009 grdn:0.157 lr:9.9e-05 updt_s:0.162 data_s:0.128 +INFO 2025-05-05 18:52:18 ts/train.py:232 step:71K smpl:1M ep:6K epch:19.50 loss:0.009 grdn:0.145 lr:9.9e-05 updt_s:0.162 data_s:0.126 +INFO 2025-05-05 18:53:16 ts/train.py:232 step:71K smpl:1M ep:6K epch:19.56 loss:0.009 grdn:0.164 lr:9.8e-05 updt_s:0.161 data_s:0.129 +INFO 2025-05-05 18:54:15 ts/train.py:232 step:71K smpl:1M ep:6K epch:19.61 loss:0.010 grdn:0.158 lr:9.8e-05 updt_s:0.162 data_s:0.129 +INFO 2025-05-05 18:55:12 ts/train.py:232 step:71K smpl:1M ep:6K epch:19.67 loss:0.010 grdn:0.154 lr:9.8e-05 updt_s:0.161 data_s:0.124 +INFO 2025-05-05 18:56:09 ts/train.py:232 step:72K smpl:1M ep:6K epch:19.72 loss:0.009 grdn:0.153 lr:9.8e-05 updt_s:0.160 data_s:0.124 +INFO 2025-05-05 18:57:06 ts/train.py:232 step:72K smpl:1M ep:6K epch:19.78 loss:0.010 grdn:0.158 lr:9.8e-05 updt_s:0.161 data_s:0.123 +INFO 2025-05-05 18:58:02 ts/train.py:232 step:72K smpl:1M ep:6K epch:19.83 loss:0.009 grdn:0.149 lr:9.8e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-05 18:58:57 ts/train.py:232 step:72K smpl:1M ep:6K epch:19.89 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-05 18:59:53 ts/train.py:232 step:72K smpl:1M ep:6K epch:19.94 loss:0.009 grdn:0.153 lr:9.8e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-05 19:00:48 ts/train.py:232 step:73K smpl:1M ep:6K epch:20.00 loss:0.009 grdn:0.159 lr:9.8e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-05 19:01:44 ts/train.py:232 step:73K smpl:1M ep:6K epch:20.05 loss:0.009 grdn:0.152 lr:9.8e-05 updt_s:0.160 data_s:0.118 +INFO 2025-05-05 19:02:40 ts/train.py:232 step:73K smpl:1M ep:6K epch:20.11 loss:0.009 grdn:0.152 lr:9.8e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-05 19:03:36 ts/train.py:232 step:73K smpl:1M ep:6K epch:20.16 loss:0.009 grdn:0.158 lr:9.8e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-05 19:04:32 ts/train.py:232 step:73K smpl:1M ep:6K epch:20.22 loss:0.009 grdn:0.155 lr:9.8e-05 updt_s:0.162 data_s:0.119 +INFO 2025-05-05 19:05:29 ts/train.py:232 step:74K smpl:1M ep:6K epch:20.27 loss:0.009 grdn:0.155 lr:9.8e-05 updt_s:0.161 data_s:0.120 +INFO 2025-05-05 19:06:22 ts/train.py:232 step:74K smpl:1M ep:6K epch:20.33 loss:0.009 grdn:0.148 lr:9.8e-05 updt_s:0.164 data_s:0.099 +INFO 2025-05-05 19:07:15 ts/train.py:232 step:74K smpl:1M ep:6K epch:20.38 loss:0.009 grdn:0.153 lr:9.8e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-05 19:08:09 ts/train.py:232 step:74K smpl:1M ep:6K epch:20.44 loss:0.009 grdn:0.154 lr:9.8e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 19:09:03 ts/train.py:232 step:74K smpl:1M ep:6K epch:20.49 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-05 19:09:55 ts/train.py:232 step:75K smpl:1M ep:6K epch:20.55 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 19:10:49 ts/train.py:232 step:75K smpl:1M ep:6K epch:20.60 loss:0.009 grdn:0.149 lr:9.8e-05 updt_s:0.162 data_s:0.109 +INFO 2025-05-05 19:11:43 ts/train.py:232 step:75K smpl:1M ep:6K epch:20.66 loss:0.009 grdn:0.159 lr:9.8e-05 updt_s:0.162 data_s:0.109 +INFO 2025-05-05 19:12:38 ts/train.py:232 step:75K smpl:1M ep:6K epch:20.71 loss:0.009 grdn:0.158 lr:9.8e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-05 19:13:32 ts/train.py:232 step:75K smpl:1M ep:6K epch:20.77 loss:0.010 grdn:0.163 lr:9.8e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-05 19:14:27 ts/train.py:232 step:76K smpl:1M ep:6K epch:20.82 loss:0.009 grdn:0.156 lr:9.8e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-05 19:15:21 ts/train.py:232 step:76K smpl:1M ep:6K epch:20.88 loss:0.009 grdn:0.147 lr:9.8e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-05 19:16:16 ts/train.py:232 step:76K smpl:1M ep:6K epch:20.93 loss:0.010 grdn:0.158 lr:9.8e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-05 19:17:12 ts/train.py:232 step:76K smpl:1M ep:6K epch:20.99 loss:0.009 grdn:0.154 lr:9.8e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 19:18:06 ts/train.py:232 step:76K smpl:1M ep:6K epch:21.04 loss:0.010 grdn:0.157 lr:9.8e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 19:19:02 ts/train.py:232 step:77K smpl:1M ep:6K epch:21.10 loss:0.009 grdn:0.158 lr:9.8e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 19:19:56 ts/train.py:232 step:77K smpl:1M ep:6K epch:21.15 loss:0.009 grdn:0.154 lr:9.8e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 19:20:51 ts/train.py:232 step:77K smpl:1M ep:6K epch:21.21 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-05 19:21:49 ts/train.py:232 step:77K smpl:1M ep:6K epch:21.26 loss:0.009 grdn:0.161 lr:9.8e-05 updt_s:0.162 data_s:0.125 +INFO 2025-05-05 19:22:43 ts/train.py:232 step:77K smpl:1M ep:6K epch:21.32 loss:0.009 grdn:0.158 lr:9.8e-05 updt_s:0.162 data_s:0.109 +INFO 2025-05-05 19:23:38 ts/train.py:232 step:78K smpl:1M ep:6K epch:21.37 loss:0.009 grdn:0.159 lr:9.8e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-05 19:24:32 ts/train.py:232 step:78K smpl:1M ep:6K epch:21.43 loss:0.010 grdn:0.155 lr:9.8e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 19:25:26 ts/train.py:232 step:78K smpl:1M ep:6K epch:21.48 loss:0.008 grdn:0.141 lr:9.8e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-05 19:26:19 ts/train.py:232 step:78K smpl:1M ep:6K epch:21.54 loss:0.009 grdn:0.160 lr:9.8e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-05 19:27:12 ts/train.py:232 step:78K smpl:1M ep:6K epch:21.59 loss:0.009 grdn:0.161 lr:9.8e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-05 19:28:05 ts/train.py:232 step:79K smpl:1M ep:6K epch:21.65 loss:0.009 grdn:0.152 lr:9.8e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-05 19:28:57 ts/train.py:232 step:79K smpl:1M ep:7K epch:21.70 loss:0.009 grdn:0.153 lr:9.8e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 19:29:49 ts/train.py:232 step:79K smpl:1M ep:7K epch:21.76 loss:0.009 grdn:0.152 lr:9.8e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-05 19:30:41 ts/train.py:232 step:79K smpl:1M ep:7K epch:21.81 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-05 19:31:33 ts/train.py:232 step:79K smpl:1M ep:7K epch:21.87 loss:0.009 grdn:0.157 lr:9.8e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-05 19:32:28 ts/train.py:232 step:80K smpl:1M ep:7K epch:21.92 loss:0.009 grdn:0.156 lr:9.8e-05 updt_s:0.163 data_s:0.110 +INFO 2025-05-05 19:33:22 ts/train.py:232 step:80K smpl:1M ep:7K epch:21.98 loss:0.010 grdn:0.161 lr:9.8e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-05 19:34:18 ts/train.py:232 step:80K smpl:1M ep:7K epch:22.03 loss:0.010 grdn:0.169 lr:9.8e-05 updt_s:0.162 data_s:0.118 +INFO 2025-05-05 19:35:13 ts/train.py:232 step:80K smpl:1M ep:7K epch:22.09 loss:0.009 grdn:0.150 lr:9.8e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-05 19:36:09 ts/train.py:232 step:80K smpl:1M ep:7K epch:22.14 loss:0.009 grdn:0.143 lr:9.8e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-05 19:37:02 ts/train.py:232 step:81K smpl:1M ep:7K epch:22.20 loss:0.009 grdn:0.155 lr:9.8e-05 updt_s:0.163 data_s:0.105 +INFO 2025-05-05 19:37:54 ts/train.py:232 step:81K smpl:1M ep:7K epch:22.26 loss:0.009 grdn:0.152 lr:9.8e-05 updt_s:0.162 data_s:0.096 +INFO 2025-05-05 19:38:46 ts/train.py:232 step:81K smpl:1M ep:7K epch:22.31 loss:0.009 grdn:0.149 lr:9.8e-05 updt_s:0.163 data_s:0.097 +INFO 2025-05-05 19:39:37 ts/train.py:232 step:81K smpl:1M ep:7K epch:22.37 loss:0.009 grdn:0.159 lr:9.8e-05 updt_s:0.162 data_s:0.091 +INFO 2025-05-05 19:40:28 ts/train.py:232 step:81K smpl:1M ep:7K epch:22.42 loss:0.008 grdn:0.155 lr:9.8e-05 updt_s:0.162 data_s:0.089 +INFO 2025-05-05 19:41:18 ts/train.py:232 step:82K smpl:1M ep:7K epch:22.48 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.163 data_s:0.090 +INFO 2025-05-05 19:42:07 ts/train.py:232 step:82K smpl:1M ep:7K epch:22.53 loss:0.010 grdn:0.167 lr:9.8e-05 updt_s:0.162 data_s:0.079 +INFO 2025-05-05 19:42:55 ts/train.py:232 step:82K smpl:1M ep:7K epch:22.59 loss:0.008 grdn:0.147 lr:9.8e-05 updt_s:0.162 data_s:0.077 +INFO 2025-05-05 19:43:43 ts/train.py:232 step:82K smpl:1M ep:7K epch:22.64 loss:0.009 grdn:0.146 lr:9.8e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-05 19:44:30 ts/train.py:232 step:82K smpl:1M ep:7K epch:22.70 loss:0.009 grdn:0.158 lr:9.8e-05 updt_s:0.161 data_s:0.078 +INFO 2025-05-05 19:45:18 ts/train.py:232 step:83K smpl:1M ep:7K epch:22.75 loss:0.009 grdn:0.159 lr:9.8e-05 updt_s:0.161 data_s:0.078 +INFO 2025-05-05 19:46:07 ts/train.py:232 step:83K smpl:1M ep:7K epch:22.81 loss:0.010 grdn:0.164 lr:9.8e-05 updt_s:0.161 data_s:0.081 +INFO 2025-05-05 19:46:55 ts/train.py:232 step:83K smpl:1M ep:7K epch:22.86 loss:0.009 grdn:0.152 lr:9.8e-05 updt_s:0.161 data_s:0.079 +INFO 2025-05-05 19:47:43 ts/train.py:232 step:83K smpl:1M ep:7K epch:22.92 loss:0.009 grdn:0.154 lr:9.8e-05 updt_s:0.162 data_s:0.077 +INFO 2025-05-05 19:48:36 ts/train.py:232 step:83K smpl:1M ep:7K epch:22.97 loss:0.009 grdn:0.148 lr:9.8e-05 updt_s:0.164 data_s:0.099 +INFO 2025-05-05 19:49:28 ts/train.py:232 step:84K smpl:1M ep:7K epch:23.03 loss:0.009 grdn:0.162 lr:9.8e-05 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 19:50:17 ts/train.py:232 step:84K smpl:1M ep:7K epch:23.08 loss:0.009 grdn:0.158 lr:9.8e-05 updt_s:0.163 data_s:0.082 +INFO 2025-05-05 19:51:05 ts/train.py:232 step:84K smpl:1M ep:7K epch:23.14 loss:0.009 grdn:0.153 lr:9.8e-05 updt_s:0.163 data_s:0.076 +INFO 2025-05-05 19:52:02 ts/train.py:232 step:84K smpl:1M ep:7K epch:23.19 loss:0.009 grdn:0.160 lr:9.8e-05 updt_s:0.162 data_s:0.124 +INFO 2025-05-05 19:52:58 ts/train.py:232 step:84K smpl:1M ep:7K epch:23.25 loss:0.009 grdn:0.150 lr:9.8e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 19:53:54 ts/train.py:232 step:85K smpl:1M ep:7K epch:23.30 loss:0.009 grdn:0.155 lr:9.8e-05 updt_s:0.162 data_s:0.114 +INFO 2025-05-05 19:54:50 ts/train.py:232 step:85K smpl:1M ep:7K epch:23.36 loss:0.008 grdn:0.155 lr:9.8e-05 updt_s:0.162 data_s:0.117 +INFO 2025-05-05 19:55:45 ts/train.py:232 step:85K smpl:1M ep:7K epch:23.41 loss:0.009 grdn:0.153 lr:9.8e-05 updt_s:0.162 data_s:0.114 +INFO 2025-05-05 19:56:41 ts/train.py:232 step:85K smpl:1M ep:7K epch:23.47 loss:0.009 grdn:0.147 lr:9.8e-05 updt_s:0.162 data_s:0.115 +INFO 2025-05-05 19:57:35 ts/train.py:232 step:85K smpl:1M ep:7K epch:23.52 loss:0.009 grdn:0.152 lr:9.8e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 19:58:29 ts/train.py:232 step:86K smpl:1M ep:7K epch:23.58 loss:0.009 grdn:0.156 lr:9.8e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 19:59:23 ts/train.py:232 step:86K smpl:1M ep:7K epch:23.63 loss:0.010 grdn:0.163 lr:9.8e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-05 20:00:16 ts/train.py:232 step:86K smpl:1M ep:7K epch:23.69 loss:0.009 grdn:0.158 lr:9.8e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-05 20:01:10 ts/train.py:232 step:86K smpl:1M ep:7K epch:23.74 loss:0.010 grdn:0.153 lr:9.8e-05 updt_s:0.163 data_s:0.104 +INFO 2025-05-05 20:02:05 ts/train.py:232 step:86K smpl:1M ep:7K epch:23.80 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-05 20:02:59 ts/train.py:232 step:87K smpl:1M ep:7K epch:23.85 loss:0.009 grdn:0.156 lr:9.8e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-05 20:03:55 ts/train.py:232 step:87K smpl:1M ep:7K epch:23.91 loss:0.008 grdn:0.148 lr:9.8e-05 updt_s:0.163 data_s:0.115 +INFO 2025-05-05 20:04:49 ts/train.py:232 step:87K smpl:1M ep:7K epch:23.96 loss:0.009 grdn:0.159 lr:9.8e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 20:05:44 ts/train.py:232 step:87K smpl:1M ep:7K epch:24.02 loss:0.009 grdn:0.152 lr:9.8e-05 updt_s:0.162 data_s:0.114 +INFO 2025-05-05 20:06:39 ts/train.py:232 step:87K smpl:1M ep:7K epch:24.07 loss:0.009 grdn:0.143 lr:9.8e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-05 20:07:32 ts/train.py:232 step:88K smpl:1M ep:7K epch:24.13 loss:0.008 grdn:0.146 lr:9.8e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-05 20:08:22 ts/train.py:232 step:88K smpl:1M ep:7K epch:24.18 loss:0.009 grdn:0.155 lr:9.8e-05 updt_s:0.162 data_s:0.084 +INFO 2025-05-05 20:09:12 ts/train.py:232 step:88K smpl:1M ep:7K epch:24.24 loss:0.008 grdn:0.145 lr:9.8e-05 updt_s:0.163 data_s:0.091 +INFO 2025-05-05 20:10:01 ts/train.py:232 step:88K smpl:1M ep:7K epch:24.29 loss:0.009 grdn:0.157 lr:9.8e-05 updt_s:0.163 data_s:0.081 +INFO 2025-05-05 20:10:51 ts/train.py:232 step:88K smpl:1M ep:7K epch:24.35 loss:0.008 grdn:0.142 lr:9.8e-05 updt_s:0.163 data_s:0.082 +INFO 2025-05-05 20:11:40 ts/train.py:232 step:89K smpl:1M ep:7K epch:24.40 loss:0.008 grdn:0.142 lr:9.8e-05 updt_s:0.164 data_s:0.080 +INFO 2025-05-05 20:12:28 ts/train.py:232 step:89K smpl:1M ep:7K epch:24.46 loss:0.008 grdn:0.153 lr:9.8e-05 updt_s:0.162 data_s:0.077 +INFO 2025-05-05 20:13:16 ts/train.py:232 step:89K smpl:1M ep:7K epch:24.51 loss:0.009 grdn:0.160 lr:9.8e-05 updt_s:0.162 data_s:0.078 +INFO 2025-05-05 20:14:05 ts/train.py:232 step:89K smpl:1M ep:7K epch:24.57 loss:0.009 grdn:0.153 lr:9.8e-05 updt_s:0.162 data_s:0.083 +INFO 2025-05-05 20:14:53 ts/train.py:232 step:89K smpl:1M ep:7K epch:24.62 loss:0.008 grdn:0.150 lr:9.8e-05 updt_s:0.161 data_s:0.078 +INFO 2025-05-05 20:15:41 ts/train.py:232 step:90K smpl:1M ep:7K epch:24.68 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.162 data_s:0.075 +INFO 2025-05-05 20:16:30 ts/train.py:232 step:90K smpl:1M ep:7K epch:24.73 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.162 data_s:0.085 +INFO 2025-05-05 20:17:19 ts/train.py:232 step:90K smpl:1M ep:7K epch:24.79 loss:0.009 grdn:0.151 lr:9.8e-05 updt_s:0.161 data_s:0.078 +INFO 2025-05-05 20:17:19 ts/train.py:241 Checkpoint policy after step 90000 +INFO 2025-05-05 20:18:09 ts/train.py:232 step:90K smpl:1M ep:7K epch:24.84 loss:0.009 grdn:0.153 lr:9.8e-05 updt_s:0.160 data_s:0.073 +INFO 2025-05-05 20:18:57 ts/train.py:232 step:90K smpl:1M ep:7K epch:24.90 loss:0.009 grdn:0.148 lr:9.8e-05 updt_s:0.162 data_s:0.077 +INFO 2025-05-05 20:19:45 ts/train.py:232 step:91K smpl:1M ep:7K epch:24.95 loss:0.009 grdn:0.149 lr:9.8e-05 updt_s:0.161 data_s:0.078 +INFO 2025-05-05 20:20:33 ts/train.py:232 step:91K smpl:1M ep:8K epch:25.01 loss:0.009 grdn:0.159 lr:9.8e-05 updt_s:0.162 data_s:0.078 +INFO 2025-05-05 20:21:22 ts/train.py:232 step:91K smpl:1M ep:8K epch:25.06 loss:0.009 grdn:0.159 lr:9.8e-05 updt_s:0.162 data_s:0.081 +INFO 2025-05-05 20:22:14 ts/train.py:232 step:91K smpl:1M ep:8K epch:25.12 loss:0.009 grdn:0.149 lr:9.8e-05 updt_s:0.163 data_s:0.096 +INFO 2025-05-05 20:23:06 ts/train.py:232 step:91K smpl:1M ep:8K epch:25.17 loss:0.009 grdn:0.155 lr:9.8e-05 updt_s:0.162 data_s:0.097 +INFO 2025-05-05 20:23:56 ts/train.py:232 step:92K smpl:1M ep:8K epch:25.23 loss:0.010 grdn:0.160 lr:9.7e-05 updt_s:0.162 data_s:0.085 +INFO 2025-05-05 20:24:46 ts/train.py:232 step:92K smpl:1M ep:8K epch:25.28 loss:0.010 grdn:0.170 lr:9.7e-05 updt_s:0.162 data_s:0.090 +INFO 2025-05-05 20:25:39 ts/train.py:232 step:92K smpl:1M ep:8K epch:25.34 loss:0.009 grdn:0.151 lr:9.7e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-05 20:26:27 ts/train.py:232 step:92K smpl:1M ep:8K epch:25.40 loss:0.008 grdn:0.148 lr:9.7e-05 updt_s:0.163 data_s:0.080 +INFO 2025-05-05 20:27:16 ts/train.py:232 step:92K smpl:1M ep:8K epch:25.45 loss:0.008 grdn:0.154 lr:9.7e-05 updt_s:0.161 data_s:0.080 +INFO 2025-05-05 20:28:07 ts/train.py:232 step:93K smpl:1M ep:8K epch:25.51 loss:0.009 grdn:0.152 lr:9.7e-05 updt_s:0.162 data_s:0.094 +INFO 2025-05-05 20:29:00 ts/train.py:232 step:93K smpl:1M ep:8K epch:25.56 loss:0.008 grdn:0.149 lr:9.7e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-05 20:29:52 ts/train.py:232 step:93K smpl:1M ep:8K epch:25.62 loss:0.009 grdn:0.151 lr:9.7e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-05 20:30:41 ts/train.py:232 step:93K smpl:1M ep:8K epch:25.67 loss:0.008 grdn:0.146 lr:9.7e-05 updt_s:0.161 data_s:0.086 +INFO 2025-05-05 20:31:35 ts/train.py:232 step:93K smpl:1M ep:8K epch:25.73 loss:0.008 grdn:0.150 lr:9.7e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-05 20:32:25 ts/train.py:232 step:94K smpl:1M ep:8K epch:25.78 loss:0.008 grdn:0.145 lr:9.7e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-05 20:33:15 ts/train.py:232 step:94K smpl:2M ep:8K epch:25.84 loss:0.009 grdn:0.157 lr:9.7e-05 updt_s:0.161 data_s:0.089 +INFO 2025-05-05 20:34:07 ts/train.py:232 step:94K smpl:2M ep:8K epch:25.89 loss:0.009 grdn:0.155 lr:9.7e-05 updt_s:0.161 data_s:0.094 +INFO 2025-05-05 20:34:58 ts/train.py:232 step:94K smpl:2M ep:8K epch:25.95 loss:0.008 grdn:0.149 lr:9.7e-05 updt_s:0.162 data_s:0.093 +INFO 2025-05-05 20:35:49 ts/train.py:232 step:94K smpl:2M ep:8K epch:26.00 loss:0.008 grdn:0.146 lr:9.7e-05 updt_s:0.162 data_s:0.091 +INFO 2025-05-05 20:36:42 ts/train.py:232 step:95K smpl:2M ep:8K epch:26.06 loss:0.008 grdn:0.145 lr:9.7e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 20:37:34 ts/train.py:232 step:95K smpl:2M ep:8K epch:26.11 loss:0.009 grdn:0.148 lr:9.7e-05 updt_s:0.162 data_s:0.100 +INFO 2025-05-05 20:38:26 ts/train.py:232 step:95K smpl:2M ep:8K epch:26.17 loss:0.009 grdn:0.146 lr:9.7e-05 updt_s:0.162 data_s:0.097 +INFO 2025-05-05 20:39:19 ts/train.py:232 step:95K smpl:2M ep:8K epch:26.22 loss:0.009 grdn:0.149 lr:9.7e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-05 20:40:12 ts/train.py:232 step:95K smpl:2M ep:8K epch:26.28 loss:0.008 grdn:0.142 lr:9.7e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-05 20:41:04 ts/train.py:232 step:96K smpl:2M ep:8K epch:26.33 loss:0.009 grdn:0.158 lr:9.7e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-05 20:41:58 ts/train.py:232 step:96K smpl:2M ep:8K epch:26.39 loss:0.010 grdn:0.154 lr:9.7e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-05 20:42:53 ts/train.py:232 step:96K smpl:2M ep:8K epch:26.44 loss:0.008 grdn:0.146 lr:9.7e-05 updt_s:0.163 data_s:0.113 +INFO 2025-05-05 20:43:49 ts/train.py:232 step:96K smpl:2M ep:8K epch:26.50 loss:0.009 grdn:0.155 lr:9.7e-05 updt_s:0.162 data_s:0.113 +INFO 2025-05-05 20:44:43 ts/train.py:232 step:96K smpl:2M ep:8K epch:26.55 loss:0.008 grdn:0.140 lr:9.7e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-05 20:45:40 ts/train.py:232 step:97K smpl:2M ep:8K epch:26.61 loss:0.008 grdn:0.147 lr:9.7e-05 updt_s:0.162 data_s:0.119 +INFO 2025-05-05 20:46:36 ts/train.py:232 step:97K smpl:2M ep:8K epch:26.66 loss:0.009 grdn:0.158 lr:9.7e-05 updt_s:0.162 data_s:0.118 +INFO 2025-05-05 20:47:31 ts/train.py:232 step:97K smpl:2M ep:8K epch:26.72 loss:0.008 grdn:0.151 lr:9.7e-05 updt_s:0.162 data_s:0.114 +INFO 2025-05-05 20:48:27 ts/train.py:232 step:97K smpl:2M ep:8K epch:26.77 loss:0.008 grdn:0.141 lr:9.7e-05 updt_s:0.162 data_s:0.115 +INFO 2025-05-05 20:49:20 ts/train.py:232 step:97K smpl:2M ep:8K epch:26.83 loss:0.009 grdn:0.157 lr:9.7e-05 updt_s:0.163 data_s:0.103 +INFO 2025-05-05 20:50:14 ts/train.py:232 step:98K smpl:2M ep:8K epch:26.88 loss:0.009 grdn:0.151 lr:9.7e-05 updt_s:0.163 data_s:0.106 +INFO 2025-05-05 20:51:09 ts/train.py:232 step:98K smpl:2M ep:8K epch:26.94 loss:0.009 grdn:0.155 lr:9.7e-05 updt_s:0.162 data_s:0.113 +INFO 2025-05-05 20:52:05 ts/train.py:232 step:98K smpl:2M ep:8K epch:26.99 loss:0.008 grdn:0.144 lr:9.7e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 20:53:03 ts/train.py:232 step:98K smpl:2M ep:8K epch:27.05 loss:0.008 grdn:0.156 lr:9.7e-05 updt_s:0.162 data_s:0.126 +INFO 2025-05-05 20:53:59 ts/train.py:232 step:98K smpl:2M ep:8K epch:27.10 loss:0.009 grdn:0.151 lr:9.7e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 20:54:55 ts/train.py:232 step:99K smpl:2M ep:8K epch:27.16 loss:0.009 grdn:0.158 lr:9.7e-05 updt_s:0.162 data_s:0.117 +INFO 2025-05-05 20:55:50 ts/train.py:232 step:99K smpl:2M ep:8K epch:27.21 loss:0.009 grdn:0.155 lr:9.7e-05 updt_s:0.162 data_s:0.115 +INFO 2025-05-05 20:56:47 ts/train.py:232 step:99K smpl:2M ep:8K epch:27.27 loss:0.008 grdn:0.149 lr:9.7e-05 updt_s:0.162 data_s:0.120 +INFO 2025-05-05 20:57:43 ts/train.py:232 step:99K smpl:2M ep:8K epch:27.32 loss:0.008 grdn:0.149 lr:9.7e-05 updt_s:0.162 data_s:0.118 +INFO 2025-05-05 20:58:38 ts/train.py:232 step:99K smpl:2M ep:8K epch:27.38 loss:0.009 grdn:0.149 lr:9.7e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-05 20:59:35 ts/train.py:232 step:100K smpl:2M ep:8K epch:27.43 loss:0.008 grdn:0.144 lr:9.7e-05 updt_s:0.161 data_s:0.122 +INFO 2025-05-05 21:00:32 ts/train.py:232 step:100K smpl:2M ep:8K epch:27.49 loss:0.009 grdn:0.150 lr:9.7e-05 updt_s:0.163 data_s:0.121 +INFO 2025-05-05 21:01:27 ts/train.py:232 step:100K smpl:2M ep:8K epch:27.54 loss:0.009 grdn:0.150 lr:9.7e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-05 21:02:21 ts/train.py:232 step:100K smpl:2M ep:8K epch:27.60 loss:0.008 grdn:0.152 lr:9.7e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-05 21:03:16 ts/train.py:232 step:100K smpl:2M ep:8K epch:27.65 loss:0.008 grdn:0.142 lr:9.7e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-05 21:04:11 ts/train.py:232 step:101K smpl:2M ep:8K epch:27.71 loss:0.008 grdn:0.146 lr:9.7e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-05 21:05:06 ts/train.py:232 step:101K smpl:2M ep:8K epch:27.76 loss:0.009 grdn:0.148 lr:9.7e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-05 21:06:01 ts/train.py:232 step:101K smpl:2M ep:8K epch:27.82 loss:0.008 grdn:0.141 lr:9.7e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 21:06:56 ts/train.py:232 step:101K smpl:2M ep:8K epch:27.87 loss:0.008 grdn:0.148 lr:9.7e-05 updt_s:0.162 data_s:0.113 +INFO 2025-05-05 21:07:52 ts/train.py:232 step:101K smpl:2M ep:8K epch:27.93 loss:0.009 grdn:0.157 lr:9.7e-05 updt_s:0.162 data_s:0.115 +INFO 2025-05-05 21:08:48 ts/train.py:232 step:102K smpl:2M ep:8K epch:27.98 loss:0.008 grdn:0.147 lr:9.7e-05 updt_s:0.161 data_s:0.121 +INFO 2025-05-05 21:09:45 ts/train.py:232 step:102K smpl:2M ep:8K epch:28.04 loss:0.009 grdn:0.151 lr:9.7e-05 updt_s:0.162 data_s:0.121 +INFO 2025-05-05 21:10:41 ts/train.py:232 step:102K smpl:2M ep:8K epch:28.09 loss:0.008 grdn:0.149 lr:9.7e-05 updt_s:0.161 data_s:0.119 +INFO 2025-05-05 21:11:39 ts/train.py:232 step:102K smpl:2M ep:8K epch:28.15 loss:0.008 grdn:0.146 lr:9.7e-05 updt_s:0.161 data_s:0.124 +INFO 2025-05-05 21:12:35 ts/train.py:232 step:102K smpl:2M ep:8K epch:28.20 loss:0.008 grdn:0.136 lr:9.7e-05 updt_s:0.161 data_s:0.119 +INFO 2025-05-05 21:13:32 ts/train.py:232 step:103K smpl:2M ep:8K epch:28.26 loss:0.009 grdn:0.147 lr:9.7e-05 updt_s:0.162 data_s:0.124 +INFO 2025-05-05 21:14:27 ts/train.py:232 step:103K smpl:2M ep:8K epch:28.31 loss:0.009 grdn:0.147 lr:9.7e-05 updt_s:0.164 data_s:0.111 +INFO 2025-05-05 21:15:24 ts/train.py:232 step:103K smpl:2M ep:9K epch:28.37 loss:0.008 grdn:0.143 lr:9.7e-05 updt_s:0.162 data_s:0.122 +INFO 2025-05-05 21:16:21 ts/train.py:232 step:103K smpl:2M ep:9K epch:28.42 loss:0.008 grdn:0.148 lr:9.7e-05 updt_s:0.162 data_s:0.118 +INFO 2025-05-05 21:17:15 ts/train.py:232 step:103K smpl:2M ep:9K epch:28.48 loss:0.008 grdn:0.149 lr:9.7e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-05 21:18:10 ts/train.py:232 step:104K smpl:2M ep:9K epch:28.54 loss:0.008 grdn:0.150 lr:9.7e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-05 21:19:03 ts/train.py:232 step:104K smpl:2M ep:9K epch:28.59 loss:0.008 grdn:0.145 lr:9.7e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-05 21:19:55 ts/train.py:232 step:104K smpl:2M ep:9K epch:28.65 loss:0.008 grdn:0.141 lr:9.7e-05 updt_s:0.162 data_s:0.095 +INFO 2025-05-05 21:20:49 ts/train.py:232 step:104K smpl:2M ep:9K epch:28.70 loss:0.009 grdn:0.146 lr:9.7e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-05 21:21:44 ts/train.py:232 step:104K smpl:2M ep:9K epch:28.76 loss:0.008 grdn:0.144 lr:9.7e-05 updt_s:0.162 data_s:0.115 +INFO 2025-05-05 21:22:39 ts/train.py:232 step:105K smpl:2M ep:9K epch:28.81 loss:0.008 grdn:0.150 lr:9.7e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 21:23:32 ts/train.py:232 step:105K smpl:2M ep:9K epch:28.87 loss:0.009 grdn:0.155 lr:9.7e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-05 21:24:26 ts/train.py:232 step:105K smpl:2M ep:9K epch:28.92 loss:0.008 grdn:0.149 lr:9.7e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-05 21:25:24 ts/train.py:232 step:105K smpl:2M ep:9K epch:28.98 loss:0.008 grdn:0.141 lr:9.7e-05 updt_s:0.163 data_s:0.126 +INFO 2025-05-05 21:26:22 ts/train.py:232 step:105K smpl:2M ep:9K epch:29.03 loss:0.008 grdn:0.143 lr:9.7e-05 updt_s:0.162 data_s:0.124 +INFO 2025-05-05 21:27:19 ts/train.py:232 step:106K smpl:2M ep:9K epch:29.09 loss:0.008 grdn:0.141 lr:9.7e-05 updt_s:0.162 data_s:0.126 +INFO 2025-05-05 21:28:18 ts/train.py:232 step:106K smpl:2M ep:9K epch:29.14 loss:0.008 grdn:0.152 lr:9.7e-05 updt_s:0.162 data_s:0.129 +INFO 2025-05-05 21:29:15 ts/train.py:232 step:106K smpl:2M ep:9K epch:29.20 loss:0.009 grdn:0.151 lr:9.7e-05 updt_s:0.162 data_s:0.123 +INFO 2025-05-05 21:30:12 ts/train.py:232 step:106K smpl:2M ep:9K epch:29.25 loss:0.008 grdn:0.151 lr:9.7e-05 updt_s:0.162 data_s:0.123 +INFO 2025-05-05 21:31:07 ts/train.py:232 step:106K smpl:2M ep:9K epch:29.31 loss:0.008 grdn:0.149 lr:9.7e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-05 21:32:03 ts/train.py:232 step:107K smpl:2M ep:9K epch:29.36 loss:0.008 grdn:0.146 lr:9.7e-05 updt_s:0.162 data_s:0.117 +INFO 2025-05-05 21:32:57 ts/train.py:232 step:107K smpl:2M ep:9K epch:29.42 loss:0.009 grdn:0.163 lr:9.7e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-05 21:33:51 ts/train.py:232 step:107K smpl:2M ep:9K epch:29.47 loss:0.009 grdn:0.154 lr:9.7e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-05 21:34:47 ts/train.py:232 step:107K smpl:2M ep:9K epch:29.53 loss:0.008 grdn:0.148 lr:9.7e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-05 21:35:43 ts/train.py:232 step:107K smpl:2M ep:9K epch:29.58 loss:0.008 grdn:0.144 lr:9.7e-05 updt_s:0.161 data_s:0.115 +INFO 2025-05-05 21:36:38 ts/train.py:232 step:108K smpl:2M ep:9K epch:29.64 loss:0.009 grdn:0.151 lr:9.7e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-05 21:37:34 ts/train.py:232 step:108K smpl:2M ep:9K epch:29.69 loss:0.008 grdn:0.141 lr:9.7e-05 updt_s:0.161 data_s:0.119 +INFO 2025-05-05 21:38:30 ts/train.py:232 step:108K smpl:2M ep:9K epch:29.75 loss:0.008 grdn:0.151 lr:9.7e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-05 21:39:26 ts/train.py:232 step:108K smpl:2M ep:9K epch:29.80 loss:0.008 grdn:0.144 lr:9.7e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-05 21:40:22 ts/train.py:232 step:108K smpl:2M ep:9K epch:29.86 loss:0.008 grdn:0.149 lr:9.6e-05 updt_s:0.162 data_s:0.119 +INFO 2025-05-05 21:41:20 ts/train.py:232 step:109K smpl:2M ep:9K epch:29.91 loss:0.008 grdn:0.150 lr:9.6e-05 updt_s:0.161 data_s:0.126 +INFO 2025-05-05 21:42:15 ts/train.py:232 step:109K smpl:2M ep:9K epch:29.97 loss:0.007 grdn:0.136 lr:9.6e-05 updt_s:0.162 data_s:0.116 +INFO 2025-05-05 21:43:11 ts/train.py:232 step:109K smpl:2M ep:9K epch:30.02 loss:0.008 grdn:0.149 lr:9.6e-05 updt_s:0.162 data_s:0.117 +INFO 2025-05-05 21:44:08 ts/train.py:232 step:109K smpl:2M ep:9K epch:30.08 loss:0.008 grdn:0.146 lr:9.6e-05 updt_s:0.162 data_s:0.121 +INFO 2025-05-05 21:45:05 ts/train.py:232 step:109K smpl:2M ep:9K epch:30.13 loss:0.008 grdn:0.145 lr:9.6e-05 updt_s:0.162 data_s:0.119 +INFO 2025-05-05 21:46:01 ts/train.py:232 step:110K smpl:2M ep:9K epch:30.19 loss:0.008 grdn:0.142 lr:9.6e-05 updt_s:0.162 data_s:0.120 +INFO 2025-05-05 21:46:57 ts/train.py:232 step:110K smpl:2M ep:9K epch:30.24 loss:0.008 grdn:0.141 lr:9.6e-05 updt_s:0.161 data_s:0.115 +INFO 2025-05-05 21:47:52 ts/train.py:232 step:110K smpl:2M ep:9K epch:30.30 loss:0.008 grdn:0.142 lr:9.6e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-05 21:48:47 ts/train.py:232 step:110K smpl:2M ep:9K epch:30.35 loss:0.008 grdn:0.145 lr:9.6e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-05 21:49:42 ts/train.py:232 step:110K smpl:2M ep:9K epch:30.41 loss:0.008 grdn:0.148 lr:9.6e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-05 21:50:37 ts/train.py:232 step:111K smpl:2M ep:9K epch:30.46 loss:0.008 grdn:0.151 lr:9.6e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-05 21:51:32 ts/train.py:232 step:111K smpl:2M ep:9K epch:30.52 loss:0.008 grdn:0.146 lr:9.6e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-05 21:52:26 ts/train.py:232 step:111K smpl:2M ep:9K epch:30.57 loss:0.008 grdn:0.142 lr:9.6e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 21:53:20 ts/train.py:232 step:111K smpl:2M ep:9K epch:30.63 loss:0.008 grdn:0.144 lr:9.6e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-05 21:54:15 ts/train.py:232 step:111K smpl:2M ep:9K epch:30.68 loss:0.008 grdn:0.144 lr:9.6e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-05 21:55:10 ts/train.py:232 step:112K smpl:2M ep:9K epch:30.74 loss:0.008 grdn:0.155 lr:9.6e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-05 21:56:04 ts/train.py:232 step:112K smpl:2M ep:9K epch:30.79 loss:0.008 grdn:0.149 lr:9.6e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 21:56:59 ts/train.py:232 step:112K smpl:2M ep:9K epch:30.85 loss:0.008 grdn:0.152 lr:9.6e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-05 21:57:53 ts/train.py:232 step:112K smpl:2M ep:9K epch:30.90 loss:0.008 grdn:0.157 lr:9.6e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-05 21:58:44 ts/train.py:232 step:112K smpl:2M ep:9K epch:30.96 loss:0.008 grdn:0.138 lr:9.6e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-05 21:59:35 ts/train.py:232 step:113K smpl:2M ep:9K epch:31.01 loss:0.008 grdn:0.151 lr:9.6e-05 updt_s:0.161 data_s:0.091 +INFO 2025-05-05 22:00:26 ts/train.py:232 step:113K smpl:2M ep:9K epch:31.07 loss:0.008 grdn:0.143 lr:9.6e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-05 22:01:17 ts/train.py:232 step:113K smpl:2M ep:9K epch:31.12 loss:0.009 grdn:0.157 lr:9.6e-05 updt_s:0.161 data_s:0.092 +INFO 2025-05-05 22:02:07 ts/train.py:232 step:113K smpl:2M ep:9K epch:31.18 loss:0.008 grdn:0.142 lr:9.6e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-05 22:02:57 ts/train.py:232 step:113K smpl:2M ep:9K epch:31.23 loss:0.008 grdn:0.140 lr:9.6e-05 updt_s:0.160 data_s:0.090 +INFO 2025-05-05 22:03:48 ts/train.py:232 step:114K smpl:2M ep:9K epch:31.29 loss:0.007 grdn:0.145 lr:9.6e-05 updt_s:0.161 data_s:0.091 +INFO 2025-05-05 22:04:37 ts/train.py:232 step:114K smpl:2M ep:9K epch:31.34 loss:0.008 grdn:0.149 lr:9.6e-05 updt_s:0.161 data_s:0.086 +INFO 2025-05-05 22:05:27 ts/train.py:232 step:114K smpl:2M ep:9K epch:31.40 loss:0.008 grdn:0.153 lr:9.6e-05 updt_s:0.160 data_s:0.086 +INFO 2025-05-05 22:06:18 ts/train.py:232 step:114K smpl:2M ep:9K epch:31.45 loss:0.009 grdn:0.154 lr:9.6e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-05 22:07:09 ts/train.py:232 step:114K smpl:2M ep:9K epch:31.51 loss:0.008 grdn:0.147 lr:9.6e-05 updt_s:0.162 data_s:0.094 +INFO 2025-05-05 22:08:01 ts/train.py:232 step:115K smpl:2M ep:9K epch:31.56 loss:0.008 grdn:0.137 lr:9.6e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-05 22:08:53 ts/train.py:232 step:115K smpl:2M ep:9K epch:31.62 loss:0.008 grdn:0.145 lr:9.6e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-05 22:09:44 ts/train.py:232 step:115K smpl:2M ep:10K epch:31.67 loss:0.009 grdn:0.157 lr:9.6e-05 updt_s:0.161 data_s:0.090 +INFO 2025-05-05 22:10:35 ts/train.py:232 step:115K smpl:2M ep:10K epch:31.73 loss:0.008 grdn:0.148 lr:9.6e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-05 22:11:26 ts/train.py:232 step:115K smpl:2M ep:10K epch:31.79 loss:0.007 grdn:0.142 lr:9.6e-05 updt_s:0.161 data_s:0.092 +INFO 2025-05-05 22:12:21 ts/train.py:232 step:116K smpl:2M ep:10K epch:31.84 loss:0.008 grdn:0.151 lr:9.6e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-05 22:13:15 ts/train.py:232 step:116K smpl:2M ep:10K epch:31.90 loss:0.008 grdn:0.153 lr:9.6e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-05 22:14:10 ts/train.py:232 step:116K smpl:2M ep:10K epch:31.95 loss:0.008 grdn:0.156 lr:9.6e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-05 22:15:06 ts/train.py:232 step:116K smpl:2M ep:10K epch:32.01 loss:0.008 grdn:0.148 lr:9.6e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-05 22:16:02 ts/train.py:232 step:116K smpl:2M ep:10K epch:32.06 loss:0.008 grdn:0.150 lr:9.6e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-05 22:16:58 ts/train.py:232 step:117K smpl:2M ep:10K epch:32.12 loss:0.008 grdn:0.142 lr:9.6e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-05 22:17:51 ts/train.py:232 step:117K smpl:2M ep:10K epch:32.17 loss:0.008 grdn:0.147 lr:9.6e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 22:18:45 ts/train.py:232 step:117K smpl:2M ep:10K epch:32.23 loss:0.008 grdn:0.147 lr:9.6e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-05 22:19:37 ts/train.py:232 step:117K smpl:2M ep:10K epch:32.28 loss:0.008 grdn:0.145 lr:9.6e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-05 22:20:31 ts/train.py:232 step:117K smpl:2M ep:10K epch:32.34 loss:0.008 grdn:0.158 lr:9.6e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-05 22:21:25 ts/train.py:232 step:118K smpl:2M ep:10K epch:32.39 loss:0.008 grdn:0.142 lr:9.6e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-05 22:22:19 ts/train.py:232 step:118K smpl:2M ep:10K epch:32.45 loss:0.008 grdn:0.147 lr:9.6e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 22:23:10 ts/train.py:232 step:118K smpl:2M ep:10K epch:32.50 loss:0.008 grdn:0.138 lr:9.6e-05 updt_s:0.161 data_s:0.090 +INFO 2025-05-05 22:24:02 ts/train.py:232 step:118K smpl:2M ep:10K epch:32.56 loss:0.008 grdn:0.148 lr:9.6e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-05 22:24:57 ts/train.py:232 step:118K smpl:2M ep:10K epch:32.61 loss:0.008 grdn:0.146 lr:9.6e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-05 22:25:52 ts/train.py:232 step:119K smpl:2M ep:10K epch:32.67 loss:0.008 grdn:0.154 lr:9.6e-05 updt_s:0.163 data_s:0.114 +INFO 2025-05-05 22:26:47 ts/train.py:232 step:119K smpl:2M ep:10K epch:32.72 loss:0.007 grdn:0.140 lr:9.6e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 22:27:41 ts/train.py:232 step:119K smpl:2M ep:10K epch:32.78 loss:0.007 grdn:0.136 lr:9.6e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-05 22:28:36 ts/train.py:232 step:119K smpl:2M ep:10K epch:32.83 loss:0.008 grdn:0.148 lr:9.6e-05 updt_s:0.163 data_s:0.115 +INFO 2025-05-05 22:29:29 ts/train.py:232 step:119K smpl:2M ep:10K epch:32.89 loss:0.008 grdn:0.142 lr:9.6e-05 updt_s:0.163 data_s:0.101 +INFO 2025-05-05 22:30:23 ts/train.py:232 step:120K smpl:2M ep:10K epch:32.94 loss:0.008 grdn:0.143 lr:9.6e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-05 22:31:17 ts/train.py:232 step:120K smpl:2M ep:10K epch:33.00 loss:0.008 grdn:0.150 lr:9.6e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-05 22:32:11 ts/train.py:232 step:120K smpl:2M ep:10K epch:33.05 loss:0.008 grdn:0.149 lr:9.6e-05 updt_s:0.163 data_s:0.107 +INFO 2025-05-05 22:32:11 ts/train.py:241 Checkpoint policy after step 120000 +INFO 2025-05-05 22:33:06 ts/train.py:232 step:120K smpl:2M ep:10K epch:33.11 loss:0.007 grdn:0.143 lr:9.6e-05 updt_s:0.160 data_s:0.096 +INFO 2025-05-05 22:33:59 ts/train.py:232 step:120K smpl:2M ep:10K epch:33.16 loss:0.008 grdn:0.151 lr:9.6e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-05 22:34:52 ts/train.py:232 step:121K smpl:2M ep:10K epch:33.22 loss:0.008 grdn:0.143 lr:9.6e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-05 22:35:47 ts/train.py:232 step:121K smpl:2M ep:10K epch:33.27 loss:0.008 grdn:0.149 lr:9.6e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-05 22:36:40 ts/train.py:232 step:121K smpl:2M ep:10K epch:33.33 loss:0.008 grdn:0.143 lr:9.6e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-05 22:37:33 ts/train.py:232 step:121K smpl:2M ep:10K epch:33.38 loss:0.008 grdn:0.154 lr:9.6e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-05 22:38:26 ts/train.py:232 step:121K smpl:2M ep:10K epch:33.44 loss:0.008 grdn:0.149 lr:9.6e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-05 22:39:21 ts/train.py:232 step:122K smpl:2M ep:10K epch:33.49 loss:0.008 grdn:0.146 lr:9.6e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-05 22:40:16 ts/train.py:232 step:122K smpl:2M ep:10K epch:33.55 loss:0.008 grdn:0.153 lr:9.6e-05 updt_s:0.159 data_s:0.112 +INFO 2025-05-05 22:41:10 ts/train.py:232 step:122K smpl:2M ep:10K epch:33.60 loss:0.008 grdn:0.148 lr:9.6e-05 updt_s:0.160 data_s:0.113 +INFO 2025-05-05 22:42:04 ts/train.py:232 step:122K smpl:2M ep:10K epch:33.66 loss:0.008 grdn:0.145 lr:9.6e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-05 22:43:00 ts/train.py:232 step:122K smpl:2M ep:10K epch:33.71 loss:0.007 grdn:0.139 lr:9.6e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-05 22:43:57 ts/train.py:232 step:123K smpl:2M ep:10K epch:33.77 loss:0.008 grdn:0.146 lr:9.6e-05 updt_s:0.160 data_s:0.126 +INFO 2025-05-05 22:44:52 ts/train.py:232 step:123K smpl:2M ep:10K epch:33.82 loss:0.008 grdn:0.149 lr:9.6e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-05 22:45:48 ts/train.py:232 step:123K smpl:2M ep:10K epch:33.88 loss:0.009 grdn:0.157 lr:9.6e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-05 22:46:44 ts/train.py:232 step:123K smpl:2M ep:10K epch:33.93 loss:0.008 grdn:0.145 lr:9.5e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-05 22:47:41 ts/train.py:232 step:123K smpl:2M ep:10K epch:33.99 loss:0.008 grdn:0.152 lr:9.5e-05 updt_s:0.161 data_s:0.121 +INFO 2025-05-05 22:48:37 ts/train.py:232 step:124K smpl:2M ep:10K epch:34.04 loss:0.008 grdn:0.142 lr:9.5e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-05 22:49:32 ts/train.py:232 step:124K smpl:2M ep:10K epch:34.10 loss:0.008 grdn:0.135 lr:9.5e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-05 22:50:26 ts/train.py:232 step:124K smpl:2M ep:10K epch:34.15 loss:0.008 grdn:0.150 lr:9.5e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-05 22:51:21 ts/train.py:232 step:124K smpl:2M ep:10K epch:34.21 loss:0.008 grdn:0.145 lr:9.5e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-05 22:52:15 ts/train.py:232 step:124K smpl:2M ep:10K epch:34.26 loss:0.008 grdn:0.143 lr:9.5e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-05 22:53:09 ts/train.py:232 step:125K smpl:2M ep:10K epch:34.32 loss:0.008 grdn:0.141 lr:9.5e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-05 22:54:03 ts/train.py:232 step:125K smpl:2M ep:10K epch:34.37 loss:0.008 grdn:0.146 lr:9.5e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-05 22:54:57 ts/train.py:232 step:125K smpl:2M ep:10K epch:34.43 loss:0.008 grdn:0.152 lr:9.5e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-05 22:55:51 ts/train.py:232 step:125K smpl:2M ep:10K epch:34.48 loss:0.008 grdn:0.142 lr:9.5e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-05 22:56:45 ts/train.py:232 step:125K smpl:2M ep:10K epch:34.54 loss:0.008 grdn:0.145 lr:9.5e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-05 22:57:41 ts/train.py:232 step:126K smpl:2M ep:10K epch:34.59 loss:0.008 grdn:0.144 lr:9.5e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-05 22:58:36 ts/train.py:232 step:126K smpl:2M ep:10K epch:34.65 loss:0.008 grdn:0.146 lr:9.5e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-05 22:59:30 ts/train.py:232 step:126K smpl:2M ep:10K epch:34.70 loss:0.007 grdn:0.141 lr:9.5e-05 updt_s:0.159 data_s:0.111 +INFO 2025-05-05 23:00:29 ts/train.py:232 step:126K smpl:2M ep:10K epch:34.76 loss:0.008 grdn:0.150 lr:9.5e-05 updt_s:0.160 data_s:0.131 +INFO 2025-05-05 23:01:26 ts/train.py:232 step:126K smpl:2M ep:10K epch:34.81 loss:0.007 grdn:0.140 lr:9.5e-05 updt_s:0.160 data_s:0.124 +INFO 2025-05-05 23:02:23 ts/train.py:232 step:127K smpl:2M ep:10K epch:34.87 loss:0.008 grdn:0.141 lr:9.5e-05 updt_s:0.160 data_s:0.128 +INFO 2025-05-05 23:03:19 ts/train.py:232 step:127K smpl:2M ep:10K epch:34.93 loss:0.008 grdn:0.146 lr:9.5e-05 updt_s:0.159 data_s:0.118 +INFO 2025-05-05 23:04:14 ts/train.py:232 step:127K smpl:2M ep:10K epch:34.98 loss:0.008 grdn:0.142 lr:9.5e-05 updt_s:0.159 data_s:0.115 +INFO 2025-05-05 23:05:10 ts/train.py:232 step:127K smpl:2M ep:11K epch:35.04 loss:0.008 grdn:0.144 lr:9.5e-05 updt_s:0.160 data_s:0.118 +INFO 2025-05-05 23:06:04 ts/train.py:232 step:127K smpl:2M ep:11K epch:35.09 loss:0.008 grdn:0.147 lr:9.5e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-05 23:06:59 ts/train.py:232 step:128K smpl:2M ep:11K epch:35.15 loss:0.007 grdn:0.144 lr:9.5e-05 updt_s:0.160 data_s:0.113 +INFO 2025-05-05 23:07:53 ts/train.py:232 step:128K smpl:2M ep:11K epch:35.20 loss:0.007 grdn:0.145 lr:9.5e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-05 23:08:47 ts/train.py:232 step:128K smpl:2M ep:11K epch:35.26 loss:0.008 grdn:0.146 lr:9.5e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-05 23:09:42 ts/train.py:232 step:128K smpl:2M ep:11K epch:35.31 loss:0.008 grdn:0.147 lr:9.5e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-05 23:10:36 ts/train.py:232 step:128K smpl:2M ep:11K epch:35.37 loss:0.008 grdn:0.146 lr:9.5e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-05 23:11:30 ts/train.py:232 step:129K smpl:2M ep:11K epch:35.42 loss:0.008 grdn:0.144 lr:9.5e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-05 23:12:25 ts/train.py:232 step:129K smpl:2M ep:11K epch:35.48 loss:0.007 grdn:0.136 lr:9.5e-05 updt_s:0.159 data_s:0.110 +INFO 2025-05-05 23:13:20 ts/train.py:232 step:129K smpl:2M ep:11K epch:35.53 loss:0.009 grdn:0.154 lr:9.5e-05 updt_s:0.159 data_s:0.117 +INFO 2025-05-05 23:14:15 ts/train.py:232 step:129K smpl:2M ep:11K epch:35.59 loss:0.008 grdn:0.151 lr:9.5e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-05 23:15:11 ts/train.py:232 step:129K smpl:2M ep:11K epch:35.64 loss:0.007 grdn:0.142 lr:9.5e-05 updt_s:0.159 data_s:0.116 +INFO 2025-05-05 23:16:09 ts/train.py:232 step:130K smpl:2M ep:11K epch:35.70 loss:0.008 grdn:0.139 lr:9.5e-05 updt_s:0.160 data_s:0.129 +INFO 2025-05-05 23:17:05 ts/train.py:232 step:130K smpl:2M ep:11K epch:35.75 loss:0.008 grdn:0.142 lr:9.5e-05 updt_s:0.160 data_s:0.123 +INFO 2025-05-05 23:18:02 ts/train.py:232 step:130K smpl:2M ep:11K epch:35.81 loss:0.008 grdn:0.149 lr:9.5e-05 updt_s:0.160 data_s:0.120 +INFO 2025-05-05 23:18:58 ts/train.py:232 step:130K smpl:2M ep:11K epch:35.86 loss:0.007 grdn:0.137 lr:9.5e-05 updt_s:0.160 data_s:0.120 +INFO 2025-05-05 23:19:52 ts/train.py:232 step:130K smpl:2M ep:11K epch:35.92 loss:0.007 grdn:0.137 lr:9.5e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-05 23:20:47 ts/train.py:232 step:131K smpl:2M ep:11K epch:35.97 loss:0.008 grdn:0.147 lr:9.5e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-05 23:21:42 ts/train.py:232 step:131K smpl:2M ep:11K epch:36.03 loss:0.008 grdn:0.142 lr:9.5e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-05 23:22:36 ts/train.py:232 step:131K smpl:2M ep:11K epch:36.08 loss:0.008 grdn:0.148 lr:9.5e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-05 23:23:29 ts/train.py:232 step:131K smpl:2M ep:11K epch:36.14 loss:0.008 grdn:0.144 lr:9.5e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-05 23:24:22 ts/train.py:232 step:131K smpl:2M ep:11K epch:36.19 loss:0.007 grdn:0.139 lr:9.5e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-05 23:25:15 ts/train.py:232 step:132K smpl:2M ep:11K epch:36.25 loss:0.007 grdn:0.138 lr:9.5e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-05 23:26:09 ts/train.py:232 step:132K smpl:2M ep:11K epch:36.30 loss:0.007 grdn:0.137 lr:9.5e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-05 23:27:03 ts/train.py:232 step:132K smpl:2M ep:11K epch:36.36 loss:0.008 grdn:0.148 lr:9.5e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-05 23:27:57 ts/train.py:232 step:132K smpl:2M ep:11K epch:36.41 loss:0.008 grdn:0.149 lr:9.5e-05 updt_s:0.160 data_s:0.108 +INFO 2025-05-05 23:28:50 ts/train.py:232 step:132K smpl:2M ep:11K epch:36.47 loss:0.008 grdn:0.149 lr:9.5e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-05 23:29:44 ts/train.py:232 step:133K smpl:2M ep:11K epch:36.52 loss:0.008 grdn:0.140 lr:9.5e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-05 23:30:37 ts/train.py:232 step:133K smpl:2M ep:11K epch:36.58 loss:0.008 grdn:0.145 lr:9.5e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-05 23:31:30 ts/train.py:232 step:133K smpl:2M ep:11K epch:36.63 loss:0.008 grdn:0.146 lr:9.5e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-05 23:32:26 ts/train.py:232 step:133K smpl:2M ep:11K epch:36.69 loss:0.008 grdn:0.142 lr:9.5e-05 updt_s:0.160 data_s:0.121 +INFO 2025-05-05 23:33:21 ts/train.py:232 step:133K smpl:2M ep:11K epch:36.74 loss:0.008 grdn:0.157 lr:9.5e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-05 23:34:17 ts/train.py:232 step:134K smpl:2M ep:11K epch:36.80 loss:0.008 grdn:0.140 lr:9.5e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-05 23:35:12 ts/train.py:232 step:134K smpl:2M ep:11K epch:36.85 loss:0.007 grdn:0.145 lr:9.5e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-05 23:36:07 ts/train.py:232 step:134K smpl:2M ep:11K epch:36.91 loss:0.007 grdn:0.144 lr:9.5e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-05 23:37:02 ts/train.py:232 step:134K smpl:2M ep:11K epch:36.96 loss:0.007 grdn:0.137 lr:9.5e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-05 23:37:57 ts/train.py:232 step:134K smpl:2M ep:11K epch:37.02 loss:0.007 grdn:0.139 lr:9.5e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-05 23:38:52 ts/train.py:232 step:135K smpl:2M ep:11K epch:37.07 loss:0.007 grdn:0.145 lr:9.5e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-05 23:39:47 ts/train.py:232 step:135K smpl:2M ep:11K epch:37.13 loss:0.007 grdn:0.141 lr:9.5e-05 updt_s:0.160 data_s:0.113 +INFO 2025-05-05 23:40:42 ts/train.py:232 step:135K smpl:2M ep:11K epch:37.18 loss:0.007 grdn:0.138 lr:9.5e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-05 23:41:37 ts/train.py:232 step:135K smpl:2M ep:11K epch:37.24 loss:0.007 grdn:0.147 lr:9.5e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-05 23:42:32 ts/train.py:232 step:135K smpl:2M ep:11K epch:37.29 loss:0.008 grdn:0.133 lr:9.5e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-05 23:43:27 ts/train.py:232 step:136K smpl:2M ep:11K epch:37.35 loss:0.007 grdn:0.142 lr:9.5e-05 updt_s:0.160 data_s:0.113 +INFO 2025-05-05 23:44:21 ts/train.py:232 step:136K smpl:2M ep:11K epch:37.40 loss:0.008 grdn:0.143 lr:9.5e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-05 23:45:16 ts/train.py:232 step:136K smpl:2M ep:11K epch:37.46 loss:0.008 grdn:0.156 lr:9.5e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-05 23:46:11 ts/train.py:232 step:136K smpl:2M ep:11K epch:37.51 loss:0.007 grdn:0.138 lr:9.4e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-05 23:47:08 ts/train.py:232 step:136K smpl:2M ep:11K epch:37.57 loss:0.007 grdn:0.140 lr:9.4e-05 updt_s:0.160 data_s:0.120 +INFO 2025-05-05 23:48:03 ts/train.py:232 step:137K smpl:2M ep:11K epch:37.62 loss:0.008 grdn:0.144 lr:9.4e-05 updt_s:0.159 data_s:0.116 +INFO 2025-05-05 23:48:56 ts/train.py:232 step:137K smpl:2M ep:11K epch:37.68 loss:0.007 grdn:0.140 lr:9.4e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-05 23:49:49 ts/train.py:232 step:137K smpl:2M ep:11K epch:37.73 loss:0.007 grdn:0.143 lr:9.4e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-05 23:50:40 ts/train.py:232 step:137K smpl:2M ep:11K epch:37.79 loss:0.007 grdn:0.149 lr:9.4e-05 updt_s:0.161 data_s:0.095 +INFO 2025-05-05 23:51:31 ts/train.py:232 step:137K smpl:2M ep:11K epch:37.84 loss:0.008 grdn:0.148 lr:9.4e-05 updt_s:0.161 data_s:0.094 +INFO 2025-05-05 23:52:21 ts/train.py:232 step:138K smpl:2M ep:11K epch:37.90 loss:0.007 grdn:0.134 lr:9.4e-05 updt_s:0.160 data_s:0.089 +INFO 2025-05-05 23:53:11 ts/train.py:232 step:138K smpl:2M ep:11K epch:37.95 loss:0.008 grdn:0.142 lr:9.4e-05 updt_s:0.160 data_s:0.088 +INFO 2025-05-05 23:54:02 ts/train.py:232 step:138K smpl:2M ep:11K epch:38.01 loss:0.007 grdn:0.146 lr:9.4e-05 updt_s:0.160 data_s:0.096 +INFO 2025-05-05 23:54:54 ts/train.py:232 step:138K smpl:2M ep:11K epch:38.07 loss:0.007 grdn:0.145 lr:9.4e-05 updt_s:0.162 data_s:0.097 +INFO 2025-05-05 23:55:46 ts/train.py:232 step:138K smpl:2M ep:11K epch:38.12 loss:0.007 grdn:0.144 lr:9.4e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-05 23:56:37 ts/train.py:232 step:139K smpl:2M ep:11K epch:38.18 loss:0.008 grdn:0.141 lr:9.4e-05 updt_s:0.161 data_s:0.095 +INFO 2025-05-05 23:57:29 ts/train.py:232 step:139K smpl:2M ep:11K epch:38.23 loss:0.007 grdn:0.141 lr:9.4e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-05 23:58:20 ts/train.py:232 step:139K smpl:2M ep:11K epch:38.29 loss:0.007 grdn:0.138 lr:9.4e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-05 23:59:12 ts/train.py:232 step:139K smpl:2M ep:12K epch:38.34 loss:0.008 grdn:0.149 lr:9.4e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 00:00:03 ts/train.py:232 step:139K smpl:2M ep:12K epch:38.40 loss:0.007 grdn:0.146 lr:9.4e-05 updt_s:0.160 data_s:0.097 +INFO 2025-05-06 00:00:55 ts/train.py:232 step:140K smpl:2M ep:12K epch:38.45 loss:0.007 grdn:0.144 lr:9.4e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 00:01:47 ts/train.py:232 step:140K smpl:2M ep:12K epch:38.51 loss:0.008 grdn:0.145 lr:9.4e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 00:02:37 ts/train.py:232 step:140K smpl:2M ep:12K epch:38.56 loss:0.007 grdn:0.138 lr:9.4e-05 updt_s:0.160 data_s:0.093 +INFO 2025-05-06 00:03:30 ts/train.py:232 step:140K smpl:2M ep:12K epch:38.62 loss:0.007 grdn:0.135 lr:9.4e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 00:04:20 ts/train.py:232 step:140K smpl:2M ep:12K epch:38.67 loss:0.007 grdn:0.138 lr:9.4e-05 updt_s:0.160 data_s:0.089 +INFO 2025-05-06 00:05:10 ts/train.py:232 step:141K smpl:2M ep:12K epch:38.73 loss:0.008 grdn:0.142 lr:9.4e-05 updt_s:0.160 data_s:0.087 +INFO 2025-05-06 00:05:59 ts/train.py:232 step:141K smpl:2M ep:12K epch:38.78 loss:0.007 grdn:0.139 lr:9.4e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 00:06:47 ts/train.py:232 step:141K smpl:2M ep:12K epch:38.84 loss:0.008 grdn:0.152 lr:9.4e-05 updt_s:0.161 data_s:0.076 +INFO 2025-05-06 00:07:35 ts/train.py:232 step:141K smpl:2M ep:12K epch:38.89 loss:0.008 grdn:0.143 lr:9.4e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 00:08:25 ts/train.py:232 step:141K smpl:2M ep:12K epch:38.95 loss:0.007 grdn:0.140 lr:9.4e-05 updt_s:0.161 data_s:0.089 +INFO 2025-05-06 00:09:15 ts/train.py:232 step:142K smpl:2M ep:12K epch:39.00 loss:0.007 grdn:0.146 lr:9.4e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-06 00:10:06 ts/train.py:232 step:142K smpl:2M ep:12K epch:39.06 loss:0.007 grdn:0.134 lr:9.4e-05 updt_s:0.160 data_s:0.094 +INFO 2025-05-06 00:10:57 ts/train.py:232 step:142K smpl:2M ep:12K epch:39.11 loss:0.008 grdn:0.146 lr:9.4e-05 updt_s:0.161 data_s:0.095 +INFO 2025-05-06 00:11:48 ts/train.py:232 step:142K smpl:2M ep:12K epch:39.17 loss:0.007 grdn:0.140 lr:9.4e-05 updt_s:0.160 data_s:0.094 +INFO 2025-05-06 00:12:39 ts/train.py:232 step:142K smpl:2M ep:12K epch:39.22 loss:0.007 grdn:0.143 lr:9.4e-05 updt_s:0.160 data_s:0.094 +INFO 2025-05-06 00:13:30 ts/train.py:232 step:143K smpl:2M ep:12K epch:39.28 loss:0.007 grdn:0.142 lr:9.4e-05 updt_s:0.160 data_s:0.095 +INFO 2025-05-06 00:14:22 ts/train.py:232 step:143K smpl:2M ep:12K epch:39.33 loss:0.007 grdn:0.144 lr:9.4e-05 updt_s:0.162 data_s:0.095 +INFO 2025-05-06 00:15:14 ts/train.py:232 step:143K smpl:2M ep:12K epch:39.39 loss:0.007 grdn:0.139 lr:9.4e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 00:16:05 ts/train.py:232 step:143K smpl:2M ep:12K epch:39.44 loss:0.007 grdn:0.142 lr:9.4e-05 updt_s:0.160 data_s:0.096 +INFO 2025-05-06 00:16:56 ts/train.py:232 step:143K smpl:2M ep:12K epch:39.50 loss:0.008 grdn:0.149 lr:9.4e-05 updt_s:0.161 data_s:0.094 +INFO 2025-05-06 00:17:49 ts/train.py:232 step:144K smpl:2M ep:12K epch:39.55 loss:0.007 grdn:0.142 lr:9.4e-05 updt_s:0.159 data_s:0.101 +INFO 2025-05-06 00:18:39 ts/train.py:232 step:144K smpl:2M ep:12K epch:39.61 loss:0.007 grdn:0.139 lr:9.4e-05 updt_s:0.160 data_s:0.091 +INFO 2025-05-06 00:19:30 ts/train.py:232 step:144K smpl:2M ep:12K epch:39.66 loss:0.008 grdn:0.145 lr:9.4e-05 updt_s:0.161 data_s:0.091 +INFO 2025-05-06 00:20:20 ts/train.py:232 step:144K smpl:2M ep:12K epch:39.72 loss:0.007 grdn:0.138 lr:9.4e-05 updt_s:0.160 data_s:0.090 +INFO 2025-05-06 00:21:10 ts/train.py:232 step:144K smpl:2M ep:12K epch:39.77 loss:0.007 grdn:0.133 lr:9.4e-05 updt_s:0.160 data_s:0.090 +INFO 2025-05-06 00:22:00 ts/train.py:232 step:145K smpl:2M ep:12K epch:39.83 loss:0.007 grdn:0.142 lr:9.4e-05 updt_s:0.160 data_s:0.089 +INFO 2025-05-06 00:22:50 ts/train.py:232 step:145K smpl:2M ep:12K epch:39.88 loss:0.007 grdn:0.143 lr:9.4e-05 updt_s:0.160 data_s:0.090 +INFO 2025-05-06 00:23:43 ts/train.py:232 step:145K smpl:2M ep:12K epch:39.94 loss:0.007 grdn:0.141 lr:9.4e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 00:24:35 ts/train.py:232 step:145K smpl:2M ep:12K epch:39.99 loss:0.007 grdn:0.141 lr:9.4e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 00:25:28 ts/train.py:232 step:145K smpl:2M ep:12K epch:40.05 loss:0.007 grdn:0.137 lr:9.4e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 00:26:21 ts/train.py:232 step:146K smpl:2M ep:12K epch:40.10 loss:0.007 grdn:0.141 lr:9.4e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 00:27:13 ts/train.py:232 step:146K smpl:2M ep:12K epch:40.16 loss:0.008 grdn:0.147 lr:9.4e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 00:28:05 ts/train.py:232 step:146K smpl:2M ep:12K epch:40.21 loss:0.007 grdn:0.143 lr:9.4e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 00:28:58 ts/train.py:232 step:146K smpl:2M ep:12K epch:40.27 loss:0.007 grdn:0.133 lr:9.4e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 00:29:50 ts/train.py:232 step:146K smpl:2M ep:12K epch:40.32 loss:0.008 grdn:0.147 lr:9.4e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 00:30:42 ts/train.py:232 step:147K smpl:2M ep:12K epch:40.38 loss:0.007 grdn:0.135 lr:9.4e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 00:31:35 ts/train.py:232 step:147K smpl:2M ep:12K epch:40.43 loss:0.007 grdn:0.146 lr:9.4e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 00:32:27 ts/train.py:232 step:147K smpl:2M ep:12K epch:40.49 loss:0.008 grdn:0.148 lr:9.4e-05 updt_s:0.160 data_s:0.100 +INFO 2025-05-06 00:33:17 ts/train.py:232 step:147K smpl:2M ep:12K epch:40.54 loss:0.008 grdn:0.145 lr:9.4e-05 updt_s:0.160 data_s:0.088 +INFO 2025-05-06 00:34:04 ts/train.py:232 step:147K smpl:2M ep:12K epch:40.60 loss:0.007 grdn:0.135 lr:9.4e-05 updt_s:0.161 data_s:0.074 +INFO 2025-05-06 00:34:51 ts/train.py:232 step:148K smpl:2M ep:12K epch:40.65 loss:0.008 grdn:0.147 lr:9.4e-05 updt_s:0.162 data_s:0.070 +INFO 2025-05-06 00:35:38 ts/train.py:232 step:148K smpl:2M ep:12K epch:40.71 loss:0.007 grdn:0.140 lr:9.4e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 00:36:25 ts/train.py:232 step:148K smpl:2M ep:12K epch:40.76 loss:0.007 grdn:0.135 lr:9.4e-05 updt_s:0.160 data_s:0.074 +INFO 2025-05-06 00:37:12 ts/train.py:232 step:148K smpl:2M ep:12K epch:40.82 loss:0.007 grdn:0.139 lr:9.4e-05 updt_s:0.160 data_s:0.075 +INFO 2025-05-06 00:38:01 ts/train.py:232 step:148K smpl:2M ep:12K epch:40.87 loss:0.007 grdn:0.134 lr:9.3e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-06 00:38:49 ts/train.py:232 step:149K smpl:2M ep:12K epch:40.93 loss:0.008 grdn:0.150 lr:9.3e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 00:39:39 ts/train.py:232 step:149K smpl:2M ep:12K epch:40.98 loss:0.008 grdn:0.138 lr:9.3e-05 updt_s:0.161 data_s:0.086 +INFO 2025-05-06 00:40:28 ts/train.py:232 step:149K smpl:2M ep:12K epch:41.04 loss:0.007 grdn:0.140 lr:9.3e-05 updt_s:0.161 data_s:0.083 +INFO 2025-05-06 00:41:17 ts/train.py:232 step:149K smpl:2M ep:12K epch:41.09 loss:0.008 grdn:0.149 lr:9.3e-05 updt_s:0.161 data_s:0.081 +INFO 2025-05-06 00:42:05 ts/train.py:232 step:149K smpl:2M ep:12K epch:41.15 loss:0.007 grdn:0.142 lr:9.3e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-06 00:42:54 ts/train.py:232 step:150K smpl:2M ep:12K epch:41.21 loss:0.007 grdn:0.137 lr:9.3e-05 updt_s:0.161 data_s:0.081 +INFO 2025-05-06 00:43:43 ts/train.py:232 step:150K smpl:2M ep:12K epch:41.26 loss:0.006 grdn:0.133 lr:9.3e-05 updt_s:0.161 data_s:0.082 +INFO 2025-05-06 00:44:32 ts/train.py:232 step:150K smpl:2M ep:12K epch:41.32 loss:0.007 grdn:0.146 lr:9.3e-05 updt_s:0.160 data_s:0.087 +INFO 2025-05-06 00:44:32 ts/train.py:241 Checkpoint policy after step 150000 +INFO 2025-05-06 00:45:24 ts/train.py:232 step:150K smpl:2M ep:12K epch:41.37 loss:0.008 grdn:0.150 lr:9.3e-05 updt_s:0.162 data_s:0.080 +INFO 2025-05-06 00:46:14 ts/train.py:232 step:150K smpl:2M ep:12K epch:41.43 loss:0.008 grdn:0.147 lr:9.3e-05 updt_s:0.162 data_s:0.089 +INFO 2025-05-06 00:47:09 ts/train.py:232 step:151K smpl:2M ep:12K epch:41.48 loss:0.007 grdn:0.145 lr:9.3e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 00:48:06 ts/train.py:232 step:151K smpl:2M ep:12K epch:41.54 loss:0.007 grdn:0.144 lr:9.3e-05 updt_s:0.161 data_s:0.122 +INFO 2025-05-06 00:49:01 ts/train.py:232 step:151K smpl:2M ep:12K epch:41.59 loss:0.007 grdn:0.143 lr:9.3e-05 updt_s:0.161 data_s:0.115 +INFO 2025-05-06 00:49:57 ts/train.py:232 step:151K smpl:2M ep:12K epch:41.65 loss:0.007 grdn:0.136 lr:9.3e-05 updt_s:0.161 data_s:0.121 +INFO 2025-05-06 00:50:53 ts/train.py:232 step:151K smpl:2M ep:13K epch:41.70 loss:0.007 grdn:0.143 lr:9.3e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-06 00:51:48 ts/train.py:232 step:152K smpl:2M ep:13K epch:41.76 loss:0.007 grdn:0.144 lr:9.3e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-06 00:52:44 ts/train.py:232 step:152K smpl:2M ep:13K epch:41.81 loss:0.007 grdn:0.141 lr:9.3e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-06 00:53:39 ts/train.py:232 step:152K smpl:2M ep:13K epch:41.87 loss:0.007 grdn:0.140 lr:9.3e-05 updt_s:0.160 data_s:0.118 +INFO 2025-05-06 00:54:36 ts/train.py:232 step:152K smpl:2M ep:13K epch:41.92 loss:0.007 grdn:0.143 lr:9.3e-05 updt_s:0.160 data_s:0.123 +INFO 2025-05-06 00:55:33 ts/train.py:232 step:152K smpl:2M ep:13K epch:41.98 loss:0.007 grdn:0.141 lr:9.3e-05 updt_s:0.161 data_s:0.124 +INFO 2025-05-06 00:56:30 ts/train.py:232 step:153K smpl:2M ep:13K epch:42.03 loss:0.007 grdn:0.146 lr:9.3e-05 updt_s:0.161 data_s:0.119 +INFO 2025-05-06 00:57:25 ts/train.py:232 step:153K smpl:2M ep:13K epch:42.09 loss:0.007 grdn:0.145 lr:9.3e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 00:58:22 ts/train.py:232 step:153K smpl:2M ep:13K epch:42.14 loss:0.007 grdn:0.139 lr:9.3e-05 updt_s:0.161 data_s:0.124 +INFO 2025-05-06 00:59:18 ts/train.py:232 step:153K smpl:2M ep:13K epch:42.20 loss:0.007 grdn:0.144 lr:9.3e-05 updt_s:0.161 data_s:0.119 +INFO 2025-05-06 01:00:15 ts/train.py:232 step:153K smpl:2M ep:13K epch:42.25 loss:0.007 grdn:0.138 lr:9.3e-05 updt_s:0.162 data_s:0.118 +INFO 2025-05-06 01:01:11 ts/train.py:232 step:154K smpl:2M ep:13K epch:42.31 loss:0.007 grdn:0.136 lr:9.3e-05 updt_s:0.161 data_s:0.119 +INFO 2025-05-06 01:02:07 ts/train.py:232 step:154K smpl:2M ep:13K epch:42.36 loss:0.008 grdn:0.152 lr:9.3e-05 updt_s:0.161 data_s:0.121 +INFO 2025-05-06 01:03:02 ts/train.py:232 step:154K smpl:2M ep:13K epch:42.42 loss:0.008 grdn:0.148 lr:9.3e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 01:03:57 ts/train.py:232 step:154K smpl:2M ep:13K epch:42.47 loss:0.008 grdn:0.148 lr:9.3e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 01:04:48 ts/train.py:232 step:154K smpl:2M ep:13K epch:42.53 loss:0.008 grdn:0.146 lr:9.3e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 01:05:40 ts/train.py:232 step:155K smpl:2M ep:13K epch:42.58 loss:0.008 grdn:0.144 lr:9.3e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 01:06:33 ts/train.py:232 step:155K smpl:2M ep:13K epch:42.64 loss:0.007 grdn:0.137 lr:9.3e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 01:07:25 ts/train.py:232 step:155K smpl:2M ep:13K epch:42.69 loss:0.007 grdn:0.143 lr:9.3e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 01:08:17 ts/train.py:232 step:155K smpl:2M ep:13K epch:42.75 loss:0.007 grdn:0.142 lr:9.3e-05 updt_s:0.160 data_s:0.100 +INFO 2025-05-06 01:09:10 ts/train.py:232 step:155K smpl:2M ep:13K epch:42.80 loss:0.007 grdn:0.143 lr:9.3e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 01:10:04 ts/train.py:232 step:156K smpl:2M ep:13K epch:42.86 loss:0.007 grdn:0.138 lr:9.3e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 01:10:57 ts/train.py:232 step:156K smpl:2M ep:13K epch:42.91 loss:0.007 grdn:0.139 lr:9.3e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 01:11:49 ts/train.py:232 step:156K smpl:2M ep:13K epch:42.97 loss:0.007 grdn:0.145 lr:9.3e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 01:12:43 ts/train.py:232 step:156K smpl:2M ep:13K epch:43.02 loss:0.007 grdn:0.135 lr:9.3e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-06 01:13:36 ts/train.py:232 step:156K smpl:3M ep:13K epch:43.08 loss:0.007 grdn:0.143 lr:9.3e-05 updt_s:0.163 data_s:0.103 +INFO 2025-05-06 01:14:30 ts/train.py:232 step:157K smpl:3M ep:13K epch:43.13 loss:0.007 grdn:0.141 lr:9.3e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 01:15:23 ts/train.py:232 step:157K smpl:3M ep:13K epch:43.19 loss:0.008 grdn:0.145 lr:9.3e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 01:16:16 ts/train.py:232 step:157K smpl:3M ep:13K epch:43.24 loss:0.007 grdn:0.137 lr:9.3e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-06 01:17:10 ts/train.py:232 step:157K smpl:3M ep:13K epch:43.30 loss:0.007 grdn:0.142 lr:9.3e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 01:18:03 ts/train.py:232 step:157K smpl:3M ep:13K epch:43.35 loss:0.008 grdn:0.147 lr:9.3e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-06 01:18:57 ts/train.py:232 step:158K smpl:3M ep:13K epch:43.41 loss:0.007 grdn:0.139 lr:9.3e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 01:19:45 ts/train.py:232 step:158K smpl:3M ep:13K epch:43.46 loss:0.007 grdn:0.133 lr:9.3e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-06 01:20:34 ts/train.py:232 step:158K smpl:3M ep:13K epch:43.52 loss:0.007 grdn:0.142 lr:9.3e-05 updt_s:0.161 data_s:0.080 +INFO 2025-05-06 01:21:22 ts/train.py:232 step:158K smpl:3M ep:13K epch:43.57 loss:0.007 grdn:0.143 lr:9.3e-05 updt_s:0.161 data_s:0.078 +INFO 2025-05-06 01:22:11 ts/train.py:232 step:158K smpl:3M ep:13K epch:43.63 loss:0.007 grdn:0.147 lr:9.3e-05 updt_s:0.161 data_s:0.083 +INFO 2025-05-06 01:22:59 ts/train.py:232 step:159K smpl:3M ep:13K epch:43.68 loss:0.007 grdn:0.138 lr:9.3e-05 updt_s:0.162 data_s:0.078 +INFO 2025-05-06 01:23:46 ts/train.py:232 step:159K smpl:3M ep:13K epch:43.74 loss:0.008 grdn:0.148 lr:9.3e-05 updt_s:0.161 data_s:0.075 +INFO 2025-05-06 01:24:35 ts/train.py:232 step:159K smpl:3M ep:13K epch:43.79 loss:0.007 grdn:0.139 lr:9.3e-05 updt_s:0.165 data_s:0.081 +INFO 2025-05-06 01:25:25 ts/train.py:232 step:159K smpl:3M ep:13K epch:43.85 loss:0.007 grdn:0.135 lr:9.3e-05 updt_s:0.162 data_s:0.085 +INFO 2025-05-06 01:26:13 ts/train.py:232 step:159K smpl:3M ep:13K epch:43.90 loss:0.007 grdn:0.140 lr:9.3e-05 updt_s:0.162 data_s:0.077 +INFO 2025-05-06 01:27:03 ts/train.py:232 step:160K smpl:3M ep:13K epch:43.96 loss:0.007 grdn:0.144 lr:9.2e-05 updt_s:0.162 data_s:0.090 +INFO 2025-05-06 01:27:51 ts/train.py:232 step:160K smpl:3M ep:13K epch:44.01 loss:0.007 grdn:0.140 lr:9.2e-05 updt_s:0.162 data_s:0.074 +INFO 2025-05-06 01:28:40 ts/train.py:232 step:160K smpl:3M ep:13K epch:44.07 loss:0.007 grdn:0.140 lr:9.2e-05 updt_s:0.161 data_s:0.083 +INFO 2025-05-06 01:29:28 ts/train.py:232 step:160K smpl:3M ep:13K epch:44.12 loss:0.007 grdn:0.137 lr:9.2e-05 updt_s:0.163 data_s:0.079 +INFO 2025-05-06 01:30:17 ts/train.py:232 step:160K smpl:3M ep:13K epch:44.18 loss:0.007 grdn:0.143 lr:9.2e-05 updt_s:0.163 data_s:0.078 +INFO 2025-05-06 01:31:04 ts/train.py:232 step:161K smpl:3M ep:13K epch:44.23 loss:0.007 grdn:0.140 lr:9.2e-05 updt_s:0.162 data_s:0.074 +INFO 2025-05-06 01:31:53 ts/train.py:232 step:161K smpl:3M ep:13K epch:44.29 loss:0.007 grdn:0.138 lr:9.2e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-06 01:32:42 ts/train.py:232 step:161K smpl:3M ep:13K epch:44.34 loss:0.007 grdn:0.136 lr:9.2e-05 updt_s:0.162 data_s:0.084 +INFO 2025-05-06 01:33:37 ts/train.py:232 step:161K smpl:3M ep:13K epch:44.40 loss:0.008 grdn:0.152 lr:9.2e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-06 01:34:30 ts/train.py:232 step:161K smpl:3M ep:13K epch:44.46 loss:0.006 grdn:0.129 lr:9.2e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 01:35:23 ts/train.py:232 step:162K smpl:3M ep:13K epch:44.51 loss:0.008 grdn:0.148 lr:9.2e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-06 01:36:16 ts/train.py:232 step:162K smpl:3M ep:13K epch:44.57 loss:0.008 grdn:0.145 lr:9.2e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 01:37:08 ts/train.py:232 step:162K smpl:3M ep:13K epch:44.62 loss:0.007 grdn:0.144 lr:9.2e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 01:37:59 ts/train.py:232 step:162K smpl:3M ep:13K epch:44.68 loss:0.007 grdn:0.143 lr:9.2e-05 updt_s:0.161 data_s:0.090 +INFO 2025-05-06 01:38:51 ts/train.py:232 step:162K smpl:3M ep:13K epch:44.73 loss:0.008 grdn:0.146 lr:9.2e-05 updt_s:0.162 data_s:0.097 +INFO 2025-05-06 01:39:43 ts/train.py:232 step:163K smpl:3M ep:13K epch:44.79 loss:0.007 grdn:0.133 lr:9.2e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-06 01:40:35 ts/train.py:232 step:163K smpl:3M ep:13K epch:44.84 loss:0.007 grdn:0.145 lr:9.2e-05 updt_s:0.162 data_s:0.100 +INFO 2025-05-06 01:41:28 ts/train.py:232 step:163K smpl:3M ep:13K epch:44.90 loss:0.007 grdn:0.137 lr:9.2e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 01:42:21 ts/train.py:232 step:163K smpl:3M ep:13K epch:44.95 loss:0.007 grdn:0.138 lr:9.2e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 01:43:14 ts/train.py:232 step:163K smpl:3M ep:14K epch:45.01 loss:0.007 grdn:0.141 lr:9.2e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 01:44:06 ts/train.py:232 step:164K smpl:3M ep:14K epch:45.06 loss:0.007 grdn:0.139 lr:9.2e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 01:45:00 ts/train.py:232 step:164K smpl:3M ep:14K epch:45.12 loss:0.007 grdn:0.135 lr:9.2e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-06 01:45:53 ts/train.py:232 step:164K smpl:3M ep:14K epch:45.17 loss:0.007 grdn:0.145 lr:9.2e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 01:46:47 ts/train.py:232 step:164K smpl:3M ep:14K epch:45.23 loss:0.006 grdn:0.135 lr:9.2e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 01:47:40 ts/train.py:232 step:164K smpl:3M ep:14K epch:45.28 loss:0.007 grdn:0.135 lr:9.2e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 01:48:33 ts/train.py:232 step:165K smpl:3M ep:14K epch:45.34 loss:0.007 grdn:0.139 lr:9.2e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 01:49:22 ts/train.py:232 step:165K smpl:3M ep:14K epch:45.39 loss:0.006 grdn:0.135 lr:9.2e-05 updt_s:0.161 data_s:0.080 +INFO 2025-05-06 01:50:11 ts/train.py:232 step:165K smpl:3M ep:14K epch:45.45 loss:0.007 grdn:0.139 lr:9.2e-05 updt_s:0.161 data_s:0.084 +INFO 2025-05-06 01:50:58 ts/train.py:232 step:165K smpl:3M ep:14K epch:45.50 loss:0.007 grdn:0.138 lr:9.2e-05 updt_s:0.161 data_s:0.073 +INFO 2025-05-06 01:51:45 ts/train.py:232 step:165K smpl:3M ep:14K epch:45.56 loss:0.007 grdn:0.138 lr:9.2e-05 updt_s:0.161 data_s:0.075 +INFO 2025-05-06 01:52:33 ts/train.py:232 step:166K smpl:3M ep:14K epch:45.61 loss:0.007 grdn:0.141 lr:9.2e-05 updt_s:0.161 data_s:0.076 +INFO 2025-05-06 01:53:20 ts/train.py:232 step:166K smpl:3M ep:14K epch:45.67 loss:0.006 grdn:0.130 lr:9.2e-05 updt_s:0.161 data_s:0.072 +INFO 2025-05-06 01:54:07 ts/train.py:232 step:166K smpl:3M ep:14K epch:45.72 loss:0.007 grdn:0.143 lr:9.2e-05 updt_s:0.161 data_s:0.074 +INFO 2025-05-06 01:54:55 ts/train.py:232 step:166K smpl:3M ep:14K epch:45.78 loss:0.008 grdn:0.155 lr:9.2e-05 updt_s:0.161 data_s:0.079 +INFO 2025-05-06 01:55:44 ts/train.py:232 step:166K smpl:3M ep:14K epch:45.83 loss:0.007 grdn:0.141 lr:9.2e-05 updt_s:0.161 data_s:0.082 +INFO 2025-05-06 01:56:33 ts/train.py:232 step:167K smpl:3M ep:14K epch:45.89 loss:0.007 grdn:0.138 lr:9.2e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-06 01:57:22 ts/train.py:232 step:167K smpl:3M ep:14K epch:45.94 loss:0.007 grdn:0.149 lr:9.2e-05 updt_s:0.161 data_s:0.081 +INFO 2025-05-06 01:58:10 ts/train.py:232 step:167K smpl:3M ep:14K epch:46.00 loss:0.007 grdn:0.144 lr:9.2e-05 updt_s:0.162 data_s:0.080 +INFO 2025-05-06 01:58:59 ts/train.py:232 step:167K smpl:3M ep:14K epch:46.05 loss:0.007 grdn:0.151 lr:9.2e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-06 01:59:48 ts/train.py:232 step:167K smpl:3M ep:14K epch:46.11 loss:0.007 grdn:0.142 lr:9.2e-05 updt_s:0.162 data_s:0.079 +INFO 2025-05-06 02:00:35 ts/train.py:232 step:168K smpl:3M ep:14K epch:46.16 loss:0.007 grdn:0.143 lr:9.2e-05 updt_s:0.162 data_s:0.075 +INFO 2025-05-06 02:01:23 ts/train.py:232 step:168K smpl:3M ep:14K epch:46.22 loss:0.007 grdn:0.133 lr:9.2e-05 updt_s:0.162 data_s:0.074 +INFO 2025-05-06 02:02:11 ts/train.py:232 step:168K smpl:3M ep:14K epch:46.27 loss:0.006 grdn:0.133 lr:9.2e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 02:03:05 ts/train.py:232 step:168K smpl:3M ep:14K epch:46.33 loss:0.006 grdn:0.131 lr:9.2e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 02:03:57 ts/train.py:232 step:168K smpl:3M ep:14K epch:46.38 loss:0.006 grdn:0.141 lr:9.2e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-06 02:04:49 ts/train.py:232 step:169K smpl:3M ep:14K epch:46.44 loss:0.007 grdn:0.142 lr:9.2e-05 updt_s:0.162 data_s:0.095 +INFO 2025-05-06 02:05:41 ts/train.py:232 step:169K smpl:3M ep:14K epch:46.49 loss:0.007 grdn:0.135 lr:9.2e-05 updt_s:0.162 data_s:0.100 +INFO 2025-05-06 02:06:34 ts/train.py:232 step:169K smpl:3M ep:14K epch:46.55 loss:0.007 grdn:0.143 lr:9.2e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 02:07:27 ts/train.py:232 step:169K smpl:3M ep:14K epch:46.60 loss:0.007 grdn:0.142 lr:9.2e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 02:08:19 ts/train.py:232 step:169K smpl:3M ep:14K epch:46.66 loss:0.007 grdn:0.140 lr:9.2e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 02:09:11 ts/train.py:232 step:170K smpl:3M ep:14K epch:46.71 loss:0.007 grdn:0.135 lr:9.2e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 02:10:04 ts/train.py:232 step:170K smpl:3M ep:14K epch:46.77 loss:0.007 grdn:0.136 lr:9.2e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 02:10:58 ts/train.py:232 step:170K smpl:3M ep:14K epch:46.82 loss:0.007 grdn:0.144 lr:9.2e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 02:11:51 ts/train.py:232 step:170K smpl:3M ep:14K epch:46.88 loss:0.007 grdn:0.142 lr:9.1e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 02:12:44 ts/train.py:232 step:170K smpl:3M ep:14K epch:46.93 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-06 02:13:37 ts/train.py:232 step:171K smpl:3M ep:14K epch:46.99 loss:0.007 grdn:0.144 lr:9.1e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-06 02:14:31 ts/train.py:232 step:171K smpl:3M ep:14K epch:47.04 loss:0.007 grdn:0.142 lr:9.1e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 02:15:24 ts/train.py:232 step:171K smpl:3M ep:14K epch:47.10 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 02:16:16 ts/train.py:232 step:171K smpl:3M ep:14K epch:47.15 loss:0.007 grdn:0.138 lr:9.1e-05 updt_s:0.162 data_s:0.100 +INFO 2025-05-06 02:17:09 ts/train.py:232 step:171K smpl:3M ep:14K epch:47.21 loss:0.007 grdn:0.143 lr:9.1e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 02:18:05 ts/train.py:232 step:172K smpl:3M ep:14K epch:47.26 loss:0.006 grdn:0.134 lr:9.1e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-06 02:18:57 ts/train.py:232 step:172K smpl:3M ep:14K epch:47.32 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 02:19:49 ts/train.py:232 step:172K smpl:3M ep:14K epch:47.37 loss:0.007 grdn:0.136 lr:9.1e-05 updt_s:0.162 data_s:0.097 +INFO 2025-05-06 02:20:43 ts/train.py:232 step:172K smpl:3M ep:14K epch:47.43 loss:0.007 grdn:0.143 lr:9.1e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 02:21:34 ts/train.py:232 step:172K smpl:3M ep:14K epch:47.48 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 02:22:24 ts/train.py:232 step:173K smpl:3M ep:14K epch:47.54 loss:0.007 grdn:0.143 lr:9.1e-05 updt_s:0.162 data_s:0.086 +INFO 2025-05-06 02:23:15 ts/train.py:232 step:173K smpl:3M ep:14K epch:47.60 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.162 data_s:0.092 +INFO 2025-05-06 02:24:04 ts/train.py:232 step:173K smpl:3M ep:14K epch:47.65 loss:0.007 grdn:0.136 lr:9.1e-05 updt_s:0.161 data_s:0.084 +INFO 2025-05-06 02:24:55 ts/train.py:232 step:173K smpl:3M ep:14K epch:47.71 loss:0.007 grdn:0.144 lr:9.1e-05 updt_s:0.162 data_s:0.092 +INFO 2025-05-06 02:25:47 ts/train.py:232 step:173K smpl:3M ep:14K epch:47.76 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.162 data_s:0.096 +INFO 2025-05-06 02:26:38 ts/train.py:232 step:174K smpl:3M ep:14K epch:47.82 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.162 data_s:0.093 +INFO 2025-05-06 02:27:29 ts/train.py:232 step:174K smpl:3M ep:14K epch:47.87 loss:0.007 grdn:0.137 lr:9.1e-05 updt_s:0.162 data_s:0.091 +INFO 2025-05-06 02:28:21 ts/train.py:232 step:174K smpl:3M ep:14K epch:47.93 loss:0.006 grdn:0.133 lr:9.1e-05 updt_s:0.163 data_s:0.095 +INFO 2025-05-06 02:29:13 ts/train.py:232 step:174K smpl:3M ep:14K epch:47.98 loss:0.007 grdn:0.138 lr:9.1e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 02:30:06 ts/train.py:232 step:174K smpl:3M ep:14K epch:48.04 loss:0.007 grdn:0.139 lr:9.1e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 02:30:58 ts/train.py:232 step:175K smpl:3M ep:14K epch:48.09 loss:0.007 grdn:0.144 lr:9.1e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 02:31:51 ts/train.py:232 step:175K smpl:3M ep:14K epch:48.15 loss:0.007 grdn:0.144 lr:9.1e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 02:32:42 ts/train.py:232 step:175K smpl:3M ep:14K epch:48.20 loss:0.007 grdn:0.135 lr:9.1e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 02:33:39 ts/train.py:232 step:175K smpl:3M ep:14K epch:48.26 loss:0.007 grdn:0.145 lr:9.1e-05 updt_s:0.161 data_s:0.122 +INFO 2025-05-06 02:34:34 ts/train.py:232 step:175K smpl:3M ep:14K epch:48.31 loss:0.006 grdn:0.130 lr:9.1e-05 updt_s:0.162 data_s:0.113 +INFO 2025-05-06 02:35:29 ts/train.py:232 step:176K smpl:3M ep:15K epch:48.37 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 02:36:25 ts/train.py:232 step:176K smpl:3M ep:15K epch:48.42 loss:0.008 grdn:0.152 lr:9.1e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-06 02:37:19 ts/train.py:232 step:176K smpl:3M ep:15K epch:48.48 loss:0.007 grdn:0.144 lr:9.1e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 02:38:14 ts/train.py:232 step:176K smpl:3M ep:15K epch:48.53 loss:0.007 grdn:0.137 lr:9.1e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 02:39:07 ts/train.py:232 step:176K smpl:3M ep:15K epch:48.59 loss:0.007 grdn:0.145 lr:9.1e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 02:39:59 ts/train.py:232 step:177K smpl:3M ep:15K epch:48.64 loss:0.007 grdn:0.135 lr:9.1e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 02:40:53 ts/train.py:232 step:177K smpl:3M ep:15K epch:48.70 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.163 data_s:0.107 +INFO 2025-05-06 02:41:48 ts/train.py:232 step:177K smpl:3M ep:15K epch:48.75 loss:0.007 grdn:0.143 lr:9.1e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-06 02:42:42 ts/train.py:232 step:177K smpl:3M ep:15K epch:48.81 loss:0.007 grdn:0.145 lr:9.1e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-06 02:43:36 ts/train.py:232 step:177K smpl:3M ep:15K epch:48.86 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-06 02:44:30 ts/train.py:232 step:178K smpl:3M ep:15K epch:48.92 loss:0.007 grdn:0.136 lr:9.1e-05 updt_s:0.162 data_s:0.110 +INFO 2025-05-06 02:45:25 ts/train.py:232 step:178K smpl:3M ep:15K epch:48.97 loss:0.007 grdn:0.141 lr:9.1e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-06 02:46:19 ts/train.py:232 step:178K smpl:3M ep:15K epch:49.03 loss:0.006 grdn:0.141 lr:9.1e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-06 02:47:14 ts/train.py:232 step:178K smpl:3M ep:15K epch:49.08 loss:0.006 grdn:0.128 lr:9.1e-05 updt_s:0.163 data_s:0.107 +INFO 2025-05-06 02:48:08 ts/train.py:232 step:178K smpl:3M ep:15K epch:49.14 loss:0.007 grdn:0.140 lr:9.1e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-06 02:49:03 ts/train.py:232 step:179K smpl:3M ep:15K epch:49.19 loss:0.006 grdn:0.133 lr:9.1e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 02:49:58 ts/train.py:232 step:179K smpl:3M ep:15K epch:49.25 loss:0.007 grdn:0.139 lr:9.1e-05 updt_s:0.162 data_s:0.111 +INFO 2025-05-06 02:50:52 ts/train.py:232 step:179K smpl:3M ep:15K epch:49.30 loss:0.006 grdn:0.140 lr:9.1e-05 updt_s:0.163 data_s:0.108 +INFO 2025-05-06 02:51:46 ts/train.py:232 step:179K smpl:3M ep:15K epch:49.36 loss:0.007 grdn:0.143 lr:9.1e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 02:52:40 ts/train.py:232 step:179K smpl:3M ep:15K epch:49.41 loss:0.007 grdn:0.148 lr:9.1e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 02:53:34 ts/train.py:232 step:180K smpl:3M ep:15K epch:49.47 loss:0.007 grdn:0.141 lr:9.1e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 02:54:27 ts/train.py:232 step:180K smpl:3M ep:15K epch:49.52 loss:0.007 grdn:0.139 lr:9.1e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-06 02:55:20 ts/train.py:232 step:180K smpl:3M ep:15K epch:49.58 loss:0.007 grdn:0.136 lr:9.1e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 02:55:20 ts/train.py:241 Checkpoint policy after step 180000 +INFO 2025-05-06 02:56:16 ts/train.py:232 step:180K smpl:3M ep:15K epch:49.63 loss:0.007 grdn:0.141 lr:9.0e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 02:57:09 ts/train.py:232 step:180K smpl:3M ep:15K epch:49.69 loss:0.007 grdn:0.142 lr:9.0e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 02:58:01 ts/train.py:232 step:181K smpl:3M ep:15K epch:49.74 loss:0.007 grdn:0.135 lr:9.0e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 02:58:54 ts/train.py:232 step:181K smpl:3M ep:15K epch:49.80 loss:0.006 grdn:0.137 lr:9.0e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 02:59:47 ts/train.py:232 step:181K smpl:3M ep:15K epch:49.85 loss:0.007 grdn:0.143 lr:9.0e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 03:00:40 ts/train.py:232 step:181K smpl:3M ep:15K epch:49.91 loss:0.007 grdn:0.139 lr:9.0e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 03:01:33 ts/train.py:232 step:181K smpl:3M ep:15K epch:49.96 loss:0.007 grdn:0.139 lr:9.0e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 03:02:26 ts/train.py:232 step:182K smpl:3M ep:15K epch:50.02 loss:0.007 grdn:0.139 lr:9.0e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 03:03:20 ts/train.py:232 step:182K smpl:3M ep:15K epch:50.07 loss:0.007 grdn:0.142 lr:9.0e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 03:04:14 ts/train.py:232 step:182K smpl:3M ep:15K epch:50.13 loss:0.006 grdn:0.126 lr:9.0e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 03:05:03 ts/train.py:232 step:182K smpl:3M ep:15K epch:50.18 loss:0.007 grdn:0.148 lr:9.0e-05 updt_s:0.162 data_s:0.084 +INFO 2025-05-06 03:05:50 ts/train.py:232 step:182K smpl:3M ep:15K epch:50.24 loss:0.007 grdn:0.133 lr:9.0e-05 updt_s:0.162 data_s:0.072 +INFO 2025-05-06 03:06:38 ts/train.py:232 step:183K smpl:3M ep:15K epch:50.29 loss:0.007 grdn:0.138 lr:9.0e-05 updt_s:0.161 data_s:0.076 +INFO 2025-05-06 03:07:24 ts/train.py:232 step:183K smpl:3M ep:15K epch:50.35 loss:0.007 grdn:0.136 lr:9.0e-05 updt_s:0.160 data_s:0.072 +INFO 2025-05-06 03:08:12 ts/train.py:232 step:183K smpl:3M ep:15K epch:50.40 loss:0.007 grdn:0.137 lr:9.0e-05 updt_s:0.161 data_s:0.075 +INFO 2025-05-06 03:08:58 ts/train.py:232 step:183K smpl:3M ep:15K epch:50.46 loss:0.007 grdn:0.144 lr:9.0e-05 updt_s:0.161 data_s:0.072 +INFO 2025-05-06 03:09:46 ts/train.py:232 step:183K smpl:3M ep:15K epch:50.51 loss:0.007 grdn:0.137 lr:9.0e-05 updt_s:0.161 data_s:0.074 +INFO 2025-05-06 03:10:33 ts/train.py:232 step:184K smpl:3M ep:15K epch:50.57 loss:0.007 grdn:0.138 lr:9.0e-05 updt_s:0.161 data_s:0.075 +INFO 2025-05-06 03:11:21 ts/train.py:232 step:184K smpl:3M ep:15K epch:50.62 loss:0.007 grdn:0.143 lr:9.0e-05 updt_s:0.162 data_s:0.079 +INFO 2025-05-06 03:12:08 ts/train.py:232 step:184K smpl:3M ep:15K epch:50.68 loss:0.007 grdn:0.143 lr:9.0e-05 updt_s:0.161 data_s:0.074 +INFO 2025-05-06 03:12:56 ts/train.py:232 step:184K smpl:3M ep:15K epch:50.74 loss:0.007 grdn:0.136 lr:9.0e-05 updt_s:0.161 data_s:0.076 +INFO 2025-05-06 03:13:44 ts/train.py:232 step:184K smpl:3M ep:15K epch:50.79 loss:0.007 grdn:0.133 lr:9.0e-05 updt_s:0.161 data_s:0.079 +INFO 2025-05-06 03:14:33 ts/train.py:232 step:185K smpl:3M ep:15K epch:50.85 loss:0.007 grdn:0.136 lr:9.0e-05 updt_s:0.161 data_s:0.083 +INFO 2025-05-06 03:15:20 ts/train.py:232 step:185K smpl:3M ep:15K epch:50.90 loss:0.006 grdn:0.130 lr:9.0e-05 updt_s:0.161 data_s:0.073 +INFO 2025-05-06 03:16:08 ts/train.py:232 step:185K smpl:3M ep:15K epch:50.96 loss:0.006 grdn:0.139 lr:9.0e-05 updt_s:0.161 data_s:0.075 +INFO 2025-05-06 03:16:55 ts/train.py:232 step:185K smpl:3M ep:15K epch:51.01 loss:0.006 grdn:0.135 lr:9.0e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 03:17:43 ts/train.py:232 step:185K smpl:3M ep:15K epch:51.07 loss:0.006 grdn:0.133 lr:9.0e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 03:18:34 ts/train.py:232 step:186K smpl:3M ep:15K epch:51.12 loss:0.007 grdn:0.140 lr:9.0e-05 updt_s:0.160 data_s:0.092 +INFO 2025-05-06 03:19:26 ts/train.py:232 step:186K smpl:3M ep:15K epch:51.18 loss:0.007 grdn:0.139 lr:9.0e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 03:20:19 ts/train.py:232 step:186K smpl:3M ep:15K epch:51.23 loss:0.007 grdn:0.143 lr:9.0e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 03:21:10 ts/train.py:232 step:186K smpl:3M ep:15K epch:51.29 loss:0.006 grdn:0.136 lr:9.0e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 03:22:02 ts/train.py:232 step:186K smpl:3M ep:15K epch:51.34 loss:0.007 grdn:0.139 lr:9.0e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 03:22:54 ts/train.py:232 step:187K smpl:3M ep:15K epch:51.40 loss:0.006 grdn:0.128 lr:9.0e-05 updt_s:0.160 data_s:0.097 +INFO 2025-05-06 03:23:44 ts/train.py:232 step:187K smpl:3M ep:15K epch:51.45 loss:0.006 grdn:0.139 lr:9.0e-05 updt_s:0.161 data_s:0.089 +INFO 2025-05-06 03:24:35 ts/train.py:232 step:187K smpl:3M ep:15K epch:51.51 loss:0.006 grdn:0.127 lr:9.0e-05 updt_s:0.161 data_s:0.092 +INFO 2025-05-06 03:25:26 ts/train.py:232 step:187K smpl:3M ep:15K epch:51.56 loss:0.006 grdn:0.135 lr:9.0e-05 updt_s:0.161 data_s:0.095 +INFO 2025-05-06 03:26:18 ts/train.py:232 step:187K smpl:3M ep:15K epch:51.62 loss:0.007 grdn:0.142 lr:9.0e-05 updt_s:0.162 data_s:0.096 +INFO 2025-05-06 03:27:09 ts/train.py:232 step:188K smpl:3M ep:16K epch:51.67 loss:0.006 grdn:0.143 lr:9.0e-05 updt_s:0.161 data_s:0.095 +INFO 2025-05-06 03:28:00 ts/train.py:232 step:188K smpl:3M ep:16K epch:51.73 loss:0.007 grdn:0.138 lr:9.0e-05 updt_s:0.162 data_s:0.091 +INFO 2025-05-06 03:28:52 ts/train.py:232 step:188K smpl:3M ep:16K epch:51.78 loss:0.006 grdn:0.135 lr:9.0e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 03:29:42 ts/train.py:232 step:188K smpl:3M ep:16K epch:51.84 loss:0.007 grdn:0.142 lr:9.0e-05 updt_s:0.161 data_s:0.090 +INFO 2025-05-06 03:30:32 ts/train.py:232 step:188K smpl:3M ep:16K epch:51.89 loss:0.007 grdn:0.142 lr:9.0e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-06 03:31:22 ts/train.py:232 step:189K smpl:3M ep:16K epch:51.95 loss:0.006 grdn:0.130 lr:9.0e-05 updt_s:0.161 data_s:0.089 +INFO 2025-05-06 03:32:13 ts/train.py:232 step:189K smpl:3M ep:16K epch:52.00 loss:0.007 grdn:0.143 lr:9.0e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-06 03:33:05 ts/train.py:232 step:189K smpl:3M ep:16K epch:52.06 loss:0.007 grdn:0.140 lr:9.0e-05 updt_s:0.162 data_s:0.095 +INFO 2025-05-06 03:34:02 ts/train.py:232 step:189K smpl:3M ep:16K epch:52.11 loss:0.007 grdn:0.140 lr:9.0e-05 updt_s:0.161 data_s:0.125 +INFO 2025-05-06 03:34:59 ts/train.py:232 step:189K smpl:3M ep:16K epch:52.17 loss:0.007 grdn:0.148 lr:9.0e-05 updt_s:0.161 data_s:0.122 +INFO 2025-05-06 03:35:57 ts/train.py:232 step:190K smpl:3M ep:16K epch:52.22 loss:0.006 grdn:0.127 lr:8.9e-05 updt_s:0.160 data_s:0.127 +INFO 2025-05-06 03:36:53 ts/train.py:232 step:190K smpl:3M ep:16K epch:52.28 loss:0.007 grdn:0.147 lr:8.9e-05 updt_s:0.160 data_s:0.120 +INFO 2025-05-06 03:37:50 ts/train.py:232 step:190K smpl:3M ep:16K epch:52.33 loss:0.006 grdn:0.133 lr:8.9e-05 updt_s:0.160 data_s:0.122 +INFO 2025-05-06 03:38:47 ts/train.py:232 step:190K smpl:3M ep:16K epch:52.39 loss:0.007 grdn:0.143 lr:8.9e-05 updt_s:0.161 data_s:0.123 +INFO 2025-05-06 03:39:40 ts/train.py:232 step:190K smpl:3M ep:16K epch:52.44 loss:0.007 grdn:0.147 lr:8.9e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 03:40:34 ts/train.py:232 step:191K smpl:3M ep:16K epch:52.50 loss:0.007 grdn:0.137 lr:8.9e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 03:41:27 ts/train.py:232 step:191K smpl:3M ep:16K epch:52.55 loss:0.006 grdn:0.135 lr:8.9e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 03:42:20 ts/train.py:232 step:191K smpl:3M ep:16K epch:52.61 loss:0.007 grdn:0.140 lr:8.9e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 03:43:13 ts/train.py:232 step:191K smpl:3M ep:16K epch:52.66 loss:0.006 grdn:0.135 lr:8.9e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-06 03:44:06 ts/train.py:232 step:191K smpl:3M ep:16K epch:52.72 loss:0.007 grdn:0.138 lr:8.9e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 03:45:00 ts/train.py:232 step:192K smpl:3M ep:16K epch:52.77 loss:0.008 grdn:0.144 lr:8.9e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 03:45:53 ts/train.py:232 step:192K smpl:3M ep:16K epch:52.83 loss:0.006 grdn:0.132 lr:8.9e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 03:46:47 ts/train.py:232 step:192K smpl:3M ep:16K epch:52.88 loss:0.006 grdn:0.131 lr:8.9e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 03:47:40 ts/train.py:232 step:192K smpl:3M ep:16K epch:52.94 loss:0.007 grdn:0.138 lr:8.9e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 03:48:34 ts/train.py:232 step:192K smpl:3M ep:16K epch:52.99 loss:0.007 grdn:0.141 lr:8.9e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 03:49:29 ts/train.py:232 step:193K smpl:3M ep:16K epch:53.05 loss:0.007 grdn:0.134 lr:8.9e-05 updt_s:0.160 data_s:0.113 +INFO 2025-05-06 03:50:23 ts/train.py:232 step:193K smpl:3M ep:16K epch:53.10 loss:0.007 grdn:0.144 lr:8.9e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 03:51:18 ts/train.py:232 step:193K smpl:3M ep:16K epch:53.16 loss:0.006 grdn:0.134 lr:8.9e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 03:52:12 ts/train.py:232 step:193K smpl:3M ep:16K epch:53.21 loss:0.007 grdn:0.133 lr:8.9e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 03:53:05 ts/train.py:232 step:193K smpl:3M ep:16K epch:53.27 loss:0.007 grdn:0.141 lr:8.9e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 03:54:00 ts/train.py:232 step:194K smpl:3M ep:16K epch:53.32 loss:0.006 grdn:0.133 lr:8.9e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 03:54:54 ts/train.py:232 step:194K smpl:3M ep:16K epch:53.38 loss:0.007 grdn:0.143 lr:8.9e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 03:55:47 ts/train.py:232 step:194K smpl:3M ep:16K epch:53.43 loss:0.007 grdn:0.137 lr:8.9e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 03:56:40 ts/train.py:232 step:194K smpl:3M ep:16K epch:53.49 loss:0.007 grdn:0.145 lr:8.9e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 03:57:34 ts/train.py:232 step:194K smpl:3M ep:16K epch:53.54 loss:0.006 grdn:0.133 lr:8.9e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 03:58:27 ts/train.py:232 step:195K smpl:3M ep:16K epch:53.60 loss:0.007 grdn:0.148 lr:8.9e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 03:59:21 ts/train.py:232 step:195K smpl:3M ep:16K epch:53.65 loss:0.007 grdn:0.133 lr:8.9e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 04:00:15 ts/train.py:232 step:195K smpl:3M ep:16K epch:53.71 loss:0.007 grdn:0.133 lr:8.9e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 04:01:09 ts/train.py:232 step:195K smpl:3M ep:16K epch:53.76 loss:0.007 grdn:0.141 lr:8.9e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 04:02:02 ts/train.py:232 step:195K smpl:3M ep:16K epch:53.82 loss:0.006 grdn:0.131 lr:8.9e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 04:02:56 ts/train.py:232 step:196K smpl:3M ep:16K epch:53.88 loss:0.006 grdn:0.134 lr:8.9e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 04:03:50 ts/train.py:232 step:196K smpl:3M ep:16K epch:53.93 loss:0.006 grdn:0.136 lr:8.9e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 04:04:44 ts/train.py:232 step:196K smpl:3M ep:16K epch:53.99 loss:0.007 grdn:0.145 lr:8.9e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 04:05:41 ts/train.py:232 step:196K smpl:3M ep:16K epch:54.04 loss:0.007 grdn:0.138 lr:8.9e-05 updt_s:0.162 data_s:0.119 +INFO 2025-05-06 04:06:35 ts/train.py:232 step:196K smpl:3M ep:16K epch:54.10 loss:0.007 grdn:0.135 lr:8.9e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 04:07:30 ts/train.py:232 step:197K smpl:3M ep:16K epch:54.15 loss:0.006 grdn:0.134 lr:8.9e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 04:08:25 ts/train.py:232 step:197K smpl:3M ep:16K epch:54.21 loss:0.007 grdn:0.144 lr:8.9e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-06 04:09:20 ts/train.py:232 step:197K smpl:3M ep:16K epch:54.26 loss:0.007 grdn:0.145 lr:8.9e-05 updt_s:0.160 data_s:0.113 +INFO 2025-05-06 04:10:15 ts/train.py:232 step:197K smpl:3M ep:16K epch:54.32 loss:0.006 grdn:0.133 lr:8.9e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 04:11:10 ts/train.py:232 step:197K smpl:3M ep:16K epch:54.37 loss:0.007 grdn:0.136 lr:8.9e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 04:12:03 ts/train.py:232 step:198K smpl:3M ep:16K epch:54.43 loss:0.006 grdn:0.136 lr:8.9e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 04:12:57 ts/train.py:232 step:198K smpl:3M ep:16K epch:54.48 loss:0.006 grdn:0.134 lr:8.9e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 04:13:51 ts/train.py:232 step:198K smpl:3M ep:16K epch:54.54 loss:0.007 grdn:0.139 lr:8.9e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 04:14:45 ts/train.py:232 step:198K smpl:3M ep:16K epch:54.59 loss:0.006 grdn:0.135 lr:8.9e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 04:15:39 ts/train.py:232 step:198K smpl:3M ep:16K epch:54.65 loss:0.006 grdn:0.141 lr:8.9e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 04:16:33 ts/train.py:232 step:199K smpl:3M ep:16K epch:54.70 loss:0.006 grdn:0.136 lr:8.9e-05 updt_s:0.160 data_s:0.113 +INFO 2025-05-06 04:17:28 ts/train.py:232 step:199K smpl:3M ep:16K epch:54.76 loss:0.007 grdn:0.140 lr:8.8e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 04:18:21 ts/train.py:232 step:199K smpl:3M ep:16K epch:54.81 loss:0.006 grdn:0.141 lr:8.8e-05 updt_s:0.160 data_s:0.108 +INFO 2025-05-06 04:19:15 ts/train.py:232 step:199K smpl:3M ep:16K epch:54.87 loss:0.007 grdn:0.138 lr:8.8e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 04:20:10 ts/train.py:232 step:199K smpl:3M ep:16K epch:54.92 loss:0.006 grdn:0.137 lr:8.8e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 04:21:05 ts/train.py:232 step:200K smpl:3M ep:16K epch:54.98 loss:0.006 grdn:0.138 lr:8.8e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-06 04:21:58 ts/train.py:232 step:200K smpl:3M ep:17K epch:55.03 loss:0.007 grdn:0.140 lr:8.8e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 04:22:51 ts/train.py:232 step:200K smpl:3M ep:17K epch:55.09 loss:0.007 grdn:0.140 lr:8.8e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 04:23:44 ts/train.py:232 step:200K smpl:3M ep:17K epch:55.14 loss:0.006 grdn:0.136 lr:8.8e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 04:24:37 ts/train.py:232 step:200K smpl:3M ep:17K epch:55.20 loss:0.006 grdn:0.136 lr:8.8e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-06 04:25:32 ts/train.py:232 step:201K smpl:3M ep:17K epch:55.25 loss:0.006 grdn:0.135 lr:8.8e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 04:26:27 ts/train.py:232 step:201K smpl:3M ep:17K epch:55.31 loss:0.007 grdn:0.139 lr:8.8e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 04:27:22 ts/train.py:232 step:201K smpl:3M ep:17K epch:55.36 loss:0.007 grdn:0.136 lr:8.8e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 04:28:15 ts/train.py:232 step:201K smpl:3M ep:17K epch:55.42 loss:0.006 grdn:0.132 lr:8.8e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 04:29:11 ts/train.py:232 step:201K smpl:3M ep:17K epch:55.47 loss:0.007 grdn:0.139 lr:8.8e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-06 04:30:06 ts/train.py:232 step:202K smpl:3M ep:17K epch:55.53 loss:0.006 grdn:0.135 lr:8.8e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 04:31:00 ts/train.py:232 step:202K smpl:3M ep:17K epch:55.58 loss:0.006 grdn:0.136 lr:8.8e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 04:31:55 ts/train.py:232 step:202K smpl:3M ep:17K epch:55.64 loss:0.006 grdn:0.131 lr:8.8e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 04:32:49 ts/train.py:232 step:202K smpl:3M ep:17K epch:55.69 loss:0.007 grdn:0.136 lr:8.8e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 04:33:44 ts/train.py:232 step:202K smpl:3M ep:17K epch:55.75 loss:0.007 grdn:0.138 lr:8.8e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 04:34:39 ts/train.py:232 step:203K smpl:3M ep:17K epch:55.80 loss:0.007 grdn:0.142 lr:8.8e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 04:35:32 ts/train.py:232 step:203K smpl:3M ep:17K epch:55.86 loss:0.006 grdn:0.130 lr:8.8e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 04:36:25 ts/train.py:232 step:203K smpl:3M ep:17K epch:55.91 loss:0.007 grdn:0.140 lr:8.8e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 04:37:20 ts/train.py:232 step:203K smpl:3M ep:17K epch:55.97 loss:0.007 grdn:0.144 lr:8.8e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-06 04:38:14 ts/train.py:232 step:203K smpl:3M ep:17K epch:56.02 loss:0.006 grdn:0.132 lr:8.8e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 04:39:08 ts/train.py:232 step:204K smpl:3M ep:17K epch:56.08 loss:0.006 grdn:0.131 lr:8.8e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 04:40:02 ts/train.py:232 step:204K smpl:3M ep:17K epch:56.13 loss:0.007 grdn:0.140 lr:8.8e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 04:40:56 ts/train.py:232 step:204K smpl:3M ep:17K epch:56.19 loss:0.006 grdn:0.132 lr:8.8e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 04:41:51 ts/train.py:232 step:204K smpl:3M ep:17K epch:56.24 loss:0.007 grdn:0.137 lr:8.8e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 04:42:44 ts/train.py:232 step:204K smpl:3M ep:17K epch:56.30 loss:0.007 grdn:0.140 lr:8.8e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 04:43:38 ts/train.py:232 step:205K smpl:3M ep:17K epch:56.35 loss:0.007 grdn:0.141 lr:8.8e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 04:44:30 ts/train.py:232 step:205K smpl:3M ep:17K epch:56.41 loss:0.006 grdn:0.140 lr:8.8e-05 updt_s:0.162 data_s:0.100 +INFO 2025-05-06 04:45:24 ts/train.py:232 step:205K smpl:3M ep:17K epch:56.46 loss:0.007 grdn:0.137 lr:8.8e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 04:46:17 ts/train.py:232 step:205K smpl:3M ep:17K epch:56.52 loss:0.006 grdn:0.142 lr:8.8e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 04:47:11 ts/train.py:232 step:205K smpl:3M ep:17K epch:56.57 loss:0.006 grdn:0.137 lr:8.8e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 04:48:04 ts/train.py:232 step:206K smpl:3M ep:17K epch:56.63 loss:0.007 grdn:0.140 lr:8.8e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 04:48:56 ts/train.py:232 step:206K smpl:3M ep:17K epch:56.68 loss:0.007 grdn:0.134 lr:8.8e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 04:49:50 ts/train.py:232 step:206K smpl:3M ep:17K epch:56.74 loss:0.006 grdn:0.133 lr:8.8e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 04:50:42 ts/train.py:232 step:206K smpl:3M ep:17K epch:56.79 loss:0.006 grdn:0.139 lr:8.8e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 04:51:36 ts/train.py:232 step:206K smpl:3M ep:17K epch:56.85 loss:0.007 grdn:0.141 lr:8.8e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 04:52:29 ts/train.py:232 step:207K smpl:3M ep:17K epch:56.90 loss:0.007 grdn:0.138 lr:8.8e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 04:53:17 ts/train.py:232 step:207K smpl:3M ep:17K epch:56.96 loss:0.006 grdn:0.141 lr:8.8e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 04:54:04 ts/train.py:232 step:207K smpl:3M ep:17K epch:57.01 loss:0.007 grdn:0.140 lr:8.8e-05 updt_s:0.162 data_s:0.076 +INFO 2025-05-06 04:54:53 ts/train.py:232 step:207K smpl:3M ep:17K epch:57.07 loss:0.006 grdn:0.136 lr:8.8e-05 updt_s:0.161 data_s:0.083 +INFO 2025-05-06 04:55:41 ts/train.py:232 step:207K smpl:3M ep:17K epch:57.13 loss:0.006 grdn:0.135 lr:8.8e-05 updt_s:0.162 data_s:0.076 +INFO 2025-05-06 04:56:29 ts/train.py:232 step:208K smpl:3M ep:17K epch:57.18 loss:0.006 grdn:0.140 lr:8.7e-05 updt_s:0.162 data_s:0.074 +INFO 2025-05-06 04:57:17 ts/train.py:232 step:208K smpl:3M ep:17K epch:57.24 loss:0.007 grdn:0.137 lr:8.7e-05 updt_s:0.161 data_s:0.078 +INFO 2025-05-06 04:58:04 ts/train.py:232 step:208K smpl:3M ep:17K epch:57.29 loss:0.006 grdn:0.134 lr:8.7e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 04:58:51 ts/train.py:232 step:208K smpl:3M ep:17K epch:57.35 loss:0.006 grdn:0.135 lr:8.7e-05 updt_s:0.161 data_s:0.073 +INFO 2025-05-06 04:59:39 ts/train.py:232 step:208K smpl:3M ep:17K epch:57.40 loss:0.006 grdn:0.136 lr:8.7e-05 updt_s:0.161 data_s:0.074 +INFO 2025-05-06 05:00:26 ts/train.py:232 step:209K smpl:3M ep:17K epch:57.46 loss:0.006 grdn:0.135 lr:8.7e-05 updt_s:0.161 data_s:0.073 +INFO 2025-05-06 05:01:14 ts/train.py:232 step:209K smpl:3M ep:17K epch:57.51 loss:0.007 grdn:0.143 lr:8.7e-05 updt_s:0.162 data_s:0.079 +INFO 2025-05-06 05:02:02 ts/train.py:232 step:209K smpl:3M ep:17K epch:57.57 loss:0.006 grdn:0.131 lr:8.7e-05 updt_s:0.163 data_s:0.074 +INFO 2025-05-06 05:02:49 ts/train.py:232 step:209K smpl:3M ep:17K epch:57.62 loss:0.006 grdn:0.135 lr:8.7e-05 updt_s:0.161 data_s:0.076 +INFO 2025-05-06 05:03:37 ts/train.py:232 step:209K smpl:3M ep:17K epch:57.68 loss:0.007 grdn:0.144 lr:8.7e-05 updt_s:0.162 data_s:0.076 +INFO 2025-05-06 05:04:24 ts/train.py:232 step:210K smpl:3M ep:17K epch:57.73 loss:0.007 grdn:0.137 lr:8.7e-05 updt_s:0.161 data_s:0.075 +INFO 2025-05-06 05:05:12 ts/train.py:232 step:210K smpl:3M ep:17K epch:57.79 loss:0.007 grdn:0.143 lr:8.7e-05 updt_s:0.161 data_s:0.074 +INFO 2025-05-06 05:05:59 ts/train.py:232 step:210K smpl:3M ep:17K epch:57.84 loss:0.007 grdn:0.145 lr:8.7e-05 updt_s:0.161 data_s:0.076 +INFO 2025-05-06 05:05:59 ts/train.py:241 Checkpoint policy after step 210000 +INFO 2025-05-06 05:06:58 ts/train.py:232 step:210K smpl:3M ep:17K epch:57.90 loss:0.006 grdn:0.135 lr:8.7e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-06 05:07:53 ts/train.py:232 step:210K smpl:3M ep:17K epch:57.95 loss:0.006 grdn:0.130 lr:8.7e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 05:08:47 ts/train.py:232 step:211K smpl:3M ep:17K epch:58.01 loss:0.006 grdn:0.142 lr:8.7e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 05:09:41 ts/train.py:232 step:211K smpl:3M ep:17K epch:58.06 loss:0.007 grdn:0.143 lr:8.7e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 05:10:35 ts/train.py:232 step:211K smpl:3M ep:17K epch:58.12 loss:0.007 grdn:0.140 lr:8.7e-05 updt_s:0.162 data_s:0.109 +INFO 2025-05-06 05:11:30 ts/train.py:232 step:211K smpl:3M ep:17K epch:58.17 loss:0.006 grdn:0.134 lr:8.7e-05 updt_s:0.163 data_s:0.109 +INFO 2025-05-06 05:12:24 ts/train.py:232 step:211K smpl:3M ep:17K epch:58.23 loss:0.007 grdn:0.146 lr:8.7e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-06 05:13:17 ts/train.py:232 step:212K smpl:3M ep:17K epch:58.28 loss:0.006 grdn:0.140 lr:8.7e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 05:14:10 ts/train.py:232 step:212K smpl:3M ep:18K epch:58.34 loss:0.007 grdn:0.138 lr:8.7e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-06 05:15:03 ts/train.py:232 step:212K smpl:3M ep:18K epch:58.39 loss:0.007 grdn:0.140 lr:8.7e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 05:15:56 ts/train.py:232 step:212K smpl:3M ep:18K epch:58.45 loss:0.006 grdn:0.141 lr:8.7e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-06 05:16:50 ts/train.py:232 step:212K smpl:3M ep:18K epch:58.50 loss:0.006 grdn:0.138 lr:8.7e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 05:17:43 ts/train.py:232 step:213K smpl:3M ep:18K epch:58.56 loss:0.007 grdn:0.138 lr:8.7e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-06 05:18:37 ts/train.py:232 step:213K smpl:3M ep:18K epch:58.61 loss:0.007 grdn:0.142 lr:8.7e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-06 05:19:30 ts/train.py:232 step:213K smpl:3M ep:18K epch:58.67 loss:0.007 grdn:0.141 lr:8.7e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 05:20:23 ts/train.py:232 step:213K smpl:3M ep:18K epch:58.72 loss:0.007 grdn:0.143 lr:8.7e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 05:21:16 ts/train.py:232 step:213K smpl:3M ep:18K epch:58.78 loss:0.007 grdn:0.146 lr:8.7e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 05:22:12 ts/train.py:232 step:214K smpl:3M ep:18K epch:58.83 loss:0.006 grdn:0.141 lr:8.7e-05 updt_s:0.161 data_s:0.115 +INFO 2025-05-06 05:23:08 ts/train.py:232 step:214K smpl:3M ep:18K epch:58.89 loss:0.006 grdn:0.134 lr:8.7e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-06 05:24:03 ts/train.py:232 step:214K smpl:3M ep:18K epch:58.94 loss:0.007 grdn:0.143 lr:8.7e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 05:24:58 ts/train.py:232 step:214K smpl:3M ep:18K epch:59.00 loss:0.006 grdn:0.135 lr:8.7e-05 updt_s:0.162 data_s:0.114 +INFO 2025-05-06 05:25:54 ts/train.py:232 step:214K smpl:3M ep:18K epch:59.05 loss:0.006 grdn:0.136 lr:8.7e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 05:26:49 ts/train.py:232 step:215K smpl:3M ep:18K epch:59.11 loss:0.007 grdn:0.146 lr:8.7e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 05:27:43 ts/train.py:232 step:215K smpl:3M ep:18K epch:59.16 loss:0.006 grdn:0.135 lr:8.7e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 05:28:37 ts/train.py:232 step:215K smpl:3M ep:18K epch:59.22 loss:0.006 grdn:0.129 lr:8.7e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 05:29:32 ts/train.py:232 step:215K smpl:3M ep:18K epch:59.27 loss:0.006 grdn:0.133 lr:8.7e-05 updt_s:0.161 data_s:0.115 +INFO 2025-05-06 05:30:28 ts/train.py:232 step:215K smpl:3M ep:18K epch:59.33 loss:0.006 grdn:0.136 lr:8.7e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 05:31:23 ts/train.py:232 step:216K smpl:3M ep:18K epch:59.38 loss:0.005 grdn:0.125 lr:8.7e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 05:32:18 ts/train.py:232 step:216K smpl:3M ep:18K epch:59.44 loss:0.006 grdn:0.139 lr:8.7e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 05:33:14 ts/train.py:232 step:216K smpl:3M ep:18K epch:59.49 loss:0.006 grdn:0.133 lr:8.7e-05 updt_s:0.161 data_s:0.121 +INFO 2025-05-06 05:34:10 ts/train.py:232 step:216K smpl:3M ep:18K epch:59.55 loss:0.006 grdn:0.130 lr:8.6e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 05:35:05 ts/train.py:232 step:216K smpl:3M ep:18K epch:59.60 loss:0.006 grdn:0.137 lr:8.6e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-06 05:36:00 ts/train.py:232 step:217K smpl:3M ep:18K epch:59.66 loss:0.006 grdn:0.128 lr:8.6e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 05:36:55 ts/train.py:232 step:217K smpl:3M ep:18K epch:59.71 loss:0.006 grdn:0.139 lr:8.6e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 05:37:49 ts/train.py:232 step:217K smpl:3M ep:18K epch:59.77 loss:0.006 grdn:0.141 lr:8.6e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 05:38:48 ts/train.py:232 step:217K smpl:3M ep:18K epch:59.82 loss:0.006 grdn:0.137 lr:8.6e-05 updt_s:0.161 data_s:0.132 +INFO 2025-05-06 05:39:44 ts/train.py:232 step:217K smpl:3M ep:18K epch:59.88 loss:0.007 grdn:0.142 lr:8.6e-05 updt_s:0.161 data_s:0.120 +INFO 2025-05-06 05:40:42 ts/train.py:232 step:218K smpl:3M ep:18K epch:59.93 loss:0.005 grdn:0.130 lr:8.6e-05 updt_s:0.161 data_s:0.125 +INFO 2025-05-06 05:41:37 ts/train.py:232 step:218K smpl:3M ep:18K epch:59.99 loss:0.006 grdn:0.139 lr:8.6e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-06 05:42:34 ts/train.py:232 step:218K smpl:3M ep:18K epch:60.04 loss:0.006 grdn:0.133 lr:8.6e-05 updt_s:0.161 data_s:0.119 +INFO 2025-05-06 05:43:31 ts/train.py:232 step:218K smpl:3M ep:18K epch:60.10 loss:0.006 grdn:0.132 lr:8.6e-05 updt_s:0.161 data_s:0.125 +INFO 2025-05-06 05:44:27 ts/train.py:232 step:218K smpl:3M ep:18K epch:60.15 loss:0.006 grdn:0.130 lr:8.6e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-06 05:45:23 ts/train.py:232 step:219K smpl:3M ep:18K epch:60.21 loss:0.006 grdn:0.135 lr:8.6e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-06 05:46:19 ts/train.py:232 step:219K smpl:4M ep:18K epch:60.27 loss:0.006 grdn:0.139 lr:8.6e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-06 05:47:15 ts/train.py:232 step:219K smpl:4M ep:18K epch:60.32 loss:0.006 grdn:0.139 lr:8.6e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-06 05:48:10 ts/train.py:232 step:219K smpl:4M ep:18K epch:60.38 loss:0.006 grdn:0.135 lr:8.6e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 05:49:06 ts/train.py:232 step:219K smpl:4M ep:18K epch:60.43 loss:0.006 grdn:0.137 lr:8.6e-05 updt_s:0.162 data_s:0.115 +INFO 2025-05-06 05:50:01 ts/train.py:232 step:220K smpl:4M ep:18K epch:60.49 loss:0.006 grdn:0.133 lr:8.6e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-06 05:50:55 ts/train.py:232 step:220K smpl:4M ep:18K epch:60.54 loss:0.006 grdn:0.137 lr:8.6e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 05:51:49 ts/train.py:232 step:220K smpl:4M ep:18K epch:60.60 loss:0.006 grdn:0.134 lr:8.6e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 05:52:44 ts/train.py:232 step:220K smpl:4M ep:18K epch:60.65 loss:0.006 grdn:0.132 lr:8.6e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 05:53:38 ts/train.py:232 step:220K smpl:4M ep:18K epch:60.71 loss:0.006 grdn:0.138 lr:8.6e-05 updt_s:0.162 data_s:0.109 +INFO 2025-05-06 05:54:35 ts/train.py:232 step:221K smpl:4M ep:18K epch:60.76 loss:0.006 grdn:0.137 lr:8.6e-05 updt_s:0.161 data_s:0.119 +INFO 2025-05-06 05:55:29 ts/train.py:232 step:221K smpl:4M ep:18K epch:60.82 loss:0.006 grdn:0.134 lr:8.6e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-06 05:56:22 ts/train.py:232 step:221K smpl:4M ep:18K epch:60.87 loss:0.006 grdn:0.138 lr:8.6e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 05:57:16 ts/train.py:232 step:221K smpl:4M ep:18K epch:60.93 loss:0.006 grdn:0.135 lr:8.6e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 05:58:09 ts/train.py:232 step:221K smpl:4M ep:18K epch:60.98 loss:0.006 grdn:0.130 lr:8.6e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 05:59:03 ts/train.py:232 step:222K smpl:4M ep:18K epch:61.04 loss:0.006 grdn:0.133 lr:8.6e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 05:59:56 ts/train.py:232 step:222K smpl:4M ep:18K epch:61.09 loss:0.006 grdn:0.135 lr:8.6e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-06 06:00:49 ts/train.py:232 step:222K smpl:4M ep:18K epch:61.15 loss:0.007 grdn:0.140 lr:8.6e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-06 06:01:42 ts/train.py:232 step:222K smpl:4M ep:18K epch:61.20 loss:0.006 grdn:0.134 lr:8.6e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-06 06:02:35 ts/train.py:232 step:222K smpl:4M ep:18K epch:61.26 loss:0.007 grdn:0.135 lr:8.6e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 06:03:27 ts/train.py:232 step:223K smpl:4M ep:18K epch:61.31 loss:0.006 grdn:0.130 lr:8.6e-05 updt_s:0.162 data_s:0.100 +INFO 2025-05-06 06:04:20 ts/train.py:232 step:223K smpl:4M ep:18K epch:61.37 loss:0.006 grdn:0.128 lr:8.6e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-06 06:05:13 ts/train.py:232 step:223K smpl:4M ep:18K epch:61.42 loss:0.006 grdn:0.141 lr:8.6e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 06:06:04 ts/train.py:232 step:223K smpl:4M ep:18K epch:61.48 loss:0.007 grdn:0.143 lr:8.6e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 06:06:57 ts/train.py:232 step:223K smpl:4M ep:18K epch:61.53 loss:0.006 grdn:0.133 lr:8.6e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 06:07:49 ts/train.py:232 step:224K smpl:4M ep:18K epch:61.59 loss:0.006 grdn:0.133 lr:8.6e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 06:08:42 ts/train.py:232 step:224K smpl:4M ep:18K epch:61.64 loss:0.006 grdn:0.136 lr:8.6e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 06:09:34 ts/train.py:232 step:224K smpl:4M ep:19K epch:61.70 loss:0.006 grdn:0.128 lr:8.6e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 06:10:28 ts/train.py:232 step:224K smpl:4M ep:19K epch:61.75 loss:0.006 grdn:0.132 lr:8.6e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 06:11:22 ts/train.py:232 step:224K smpl:4M ep:19K epch:61.81 loss:0.006 grdn:0.137 lr:8.5e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 06:12:16 ts/train.py:232 step:225K smpl:4M ep:19K epch:61.86 loss:0.005 grdn:0.125 lr:8.5e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 06:13:09 ts/train.py:232 step:225K smpl:4M ep:19K epch:61.92 loss:0.006 grdn:0.136 lr:8.5e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 06:14:03 ts/train.py:232 step:225K smpl:4M ep:19K epch:61.97 loss:0.006 grdn:0.139 lr:8.5e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 06:14:56 ts/train.py:232 step:225K smpl:4M ep:19K epch:62.03 loss:0.006 grdn:0.139 lr:8.5e-05 updt_s:0.162 data_s:0.105 +INFO 2025-05-06 06:15:48 ts/train.py:232 step:225K smpl:4M ep:19K epch:62.08 loss:0.006 grdn:0.128 lr:8.5e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-06 06:16:40 ts/train.py:232 step:226K smpl:4M ep:19K epch:62.14 loss:0.006 grdn:0.133 lr:8.5e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 06:17:33 ts/train.py:232 step:226K smpl:4M ep:19K epch:62.19 loss:0.006 grdn:0.124 lr:8.5e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 06:18:24 ts/train.py:232 step:226K smpl:4M ep:19K epch:62.25 loss:0.006 grdn:0.141 lr:8.5e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 06:19:17 ts/train.py:232 step:226K smpl:4M ep:19K epch:62.30 loss:0.006 grdn:0.135 lr:8.5e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 06:20:09 ts/train.py:232 step:226K smpl:4M ep:19K epch:62.36 loss:0.006 grdn:0.132 lr:8.5e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 06:21:01 ts/train.py:232 step:227K smpl:4M ep:19K epch:62.41 loss:0.006 grdn:0.137 lr:8.5e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 06:21:53 ts/train.py:232 step:227K smpl:4M ep:19K epch:62.47 loss:0.006 grdn:0.137 lr:8.5e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 06:22:45 ts/train.py:232 step:227K smpl:4M ep:19K epch:62.52 loss:0.007 grdn:0.145 lr:8.5e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 06:23:37 ts/train.py:232 step:227K smpl:4M ep:19K epch:62.58 loss:0.006 grdn:0.132 lr:8.5e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 06:24:29 ts/train.py:232 step:227K smpl:4M ep:19K epch:62.63 loss:0.007 grdn:0.145 lr:8.5e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 06:25:23 ts/train.py:232 step:228K smpl:4M ep:19K epch:62.69 loss:0.007 grdn:0.140 lr:8.5e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 06:26:17 ts/train.py:232 step:228K smpl:4M ep:19K epch:62.74 loss:0.006 grdn:0.134 lr:8.5e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 06:27:11 ts/train.py:232 step:228K smpl:4M ep:19K epch:62.80 loss:0.006 grdn:0.135 lr:8.5e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 06:28:05 ts/train.py:232 step:228K smpl:4M ep:19K epch:62.85 loss:0.007 grdn:0.138 lr:8.5e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 06:28:59 ts/train.py:232 step:228K smpl:4M ep:19K epch:62.91 loss:0.006 grdn:0.129 lr:8.5e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 06:29:53 ts/train.py:232 step:229K smpl:4M ep:19K epch:62.96 loss:0.007 grdn:0.143 lr:8.5e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 06:30:46 ts/train.py:232 step:229K smpl:4M ep:19K epch:63.02 loss:0.006 grdn:0.140 lr:8.5e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 06:31:39 ts/train.py:232 step:229K smpl:4M ep:19K epch:63.07 loss:0.006 grdn:0.132 lr:8.5e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 06:32:31 ts/train.py:232 step:229K smpl:4M ep:19K epch:63.13 loss:0.007 grdn:0.142 lr:8.5e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 06:33:23 ts/train.py:232 step:229K smpl:4M ep:19K epch:63.18 loss:0.006 grdn:0.136 lr:8.5e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 06:34:16 ts/train.py:232 step:230K smpl:4M ep:19K epch:63.24 loss:0.006 grdn:0.145 lr:8.5e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 06:35:09 ts/train.py:232 step:230K smpl:4M ep:19K epch:63.29 loss:0.006 grdn:0.143 lr:8.5e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 06:36:02 ts/train.py:232 step:230K smpl:4M ep:19K epch:63.35 loss:0.006 grdn:0.132 lr:8.5e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 06:36:55 ts/train.py:232 step:230K smpl:4M ep:19K epch:63.41 loss:0.006 grdn:0.129 lr:8.5e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 06:37:48 ts/train.py:232 step:230K smpl:4M ep:19K epch:63.46 loss:0.006 grdn:0.138 lr:8.5e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 06:38:41 ts/train.py:232 step:231K smpl:4M ep:19K epch:63.52 loss:0.006 grdn:0.139 lr:8.5e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 06:39:33 ts/train.py:232 step:231K smpl:4M ep:19K epch:63.57 loss:0.006 grdn:0.137 lr:8.5e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 06:40:25 ts/train.py:232 step:231K smpl:4M ep:19K epch:63.63 loss:0.006 grdn:0.135 lr:8.5e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 06:41:21 ts/train.py:232 step:231K smpl:4M ep:19K epch:63.68 loss:0.006 grdn:0.135 lr:8.5e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 06:42:14 ts/train.py:232 step:231K smpl:4M ep:19K epch:63.74 loss:0.005 grdn:0.127 lr:8.5e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 06:43:09 ts/train.py:232 step:232K smpl:4M ep:19K epch:63.79 loss:0.006 grdn:0.138 lr:8.5e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 06:44:02 ts/train.py:232 step:232K smpl:4M ep:19K epch:63.85 loss:0.006 grdn:0.132 lr:8.5e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-06 06:44:56 ts/train.py:232 step:232K smpl:4M ep:19K epch:63.90 loss:0.007 grdn:0.141 lr:8.5e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 06:45:50 ts/train.py:232 step:232K smpl:4M ep:19K epch:63.96 loss:0.006 grdn:0.144 lr:8.5e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 06:46:44 ts/train.py:232 step:232K smpl:4M ep:19K epch:64.01 loss:0.006 grdn:0.135 lr:8.4e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 06:47:36 ts/train.py:232 step:233K smpl:4M ep:19K epch:64.07 loss:0.005 grdn:0.124 lr:8.4e-05 updt_s:0.162 data_s:0.100 +INFO 2025-05-06 06:48:28 ts/train.py:232 step:233K smpl:4M ep:19K epch:64.12 loss:0.006 grdn:0.139 lr:8.4e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 06:49:21 ts/train.py:232 step:233K smpl:4M ep:19K epch:64.18 loss:0.006 grdn:0.131 lr:8.4e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 06:50:12 ts/train.py:232 step:233K smpl:4M ep:19K epch:64.23 loss:0.006 grdn:0.138 lr:8.4e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 06:51:05 ts/train.py:232 step:233K smpl:4M ep:19K epch:64.29 loss:0.007 grdn:0.145 lr:8.4e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-06 06:51:57 ts/train.py:232 step:234K smpl:4M ep:19K epch:64.34 loss:0.006 grdn:0.132 lr:8.4e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-06 06:52:49 ts/train.py:232 step:234K smpl:4M ep:19K epch:64.40 loss:0.006 grdn:0.139 lr:8.4e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 06:53:41 ts/train.py:232 step:234K smpl:4M ep:19K epch:64.45 loss:0.005 grdn:0.126 lr:8.4e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 06:54:33 ts/train.py:232 step:234K smpl:4M ep:19K epch:64.51 loss:0.006 grdn:0.138 lr:8.4e-05 updt_s:0.160 data_s:0.099 +INFO 2025-05-06 06:55:26 ts/train.py:232 step:234K smpl:4M ep:19K epch:64.56 loss:0.006 grdn:0.126 lr:8.4e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 06:56:19 ts/train.py:232 step:235K smpl:4M ep:19K epch:64.62 loss:0.006 grdn:0.132 lr:8.4e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 06:57:13 ts/train.py:232 step:235K smpl:4M ep:19K epch:64.67 loss:0.006 grdn:0.142 lr:8.4e-05 updt_s:0.162 data_s:0.107 +INFO 2025-05-06 06:58:06 ts/train.py:232 step:235K smpl:4M ep:19K epch:64.73 loss:0.007 grdn:0.143 lr:8.4e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 06:59:00 ts/train.py:232 step:235K smpl:4M ep:19K epch:64.78 loss:0.006 grdn:0.142 lr:8.4e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 06:59:53 ts/train.py:232 step:235K smpl:4M ep:19K epch:64.84 loss:0.006 grdn:0.136 lr:8.4e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 07:00:46 ts/train.py:232 step:236K smpl:4M ep:19K epch:64.89 loss:0.005 grdn:0.125 lr:8.4e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-06 07:01:39 ts/train.py:232 step:236K smpl:4M ep:19K epch:64.95 loss:0.006 grdn:0.135 lr:8.4e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 07:02:31 ts/train.py:232 step:236K smpl:4M ep:20K epch:65.00 loss:0.006 grdn:0.136 lr:8.4e-05 updt_s:0.162 data_s:0.096 +INFO 2025-05-06 07:03:23 ts/train.py:232 step:236K smpl:4M ep:20K epch:65.06 loss:0.006 grdn:0.139 lr:8.4e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-06 07:04:16 ts/train.py:232 step:236K smpl:4M ep:20K epch:65.11 loss:0.006 grdn:0.136 lr:8.4e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 07:05:08 ts/train.py:232 step:237K smpl:4M ep:20K epch:65.17 loss:0.006 grdn:0.131 lr:8.4e-05 updt_s:0.162 data_s:0.097 +INFO 2025-05-06 07:06:00 ts/train.py:232 step:237K smpl:4M ep:20K epch:65.22 loss:0.006 grdn:0.127 lr:8.4e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 07:06:52 ts/train.py:232 step:237K smpl:4M ep:20K epch:65.28 loss:0.006 grdn:0.134 lr:8.4e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 07:07:44 ts/train.py:232 step:237K smpl:4M ep:20K epch:65.33 loss:0.006 grdn:0.136 lr:8.4e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 07:08:37 ts/train.py:232 step:237K smpl:4M ep:20K epch:65.39 loss:0.006 grdn:0.138 lr:8.4e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 07:09:29 ts/train.py:232 step:238K smpl:4M ep:20K epch:65.44 loss:0.006 grdn:0.137 lr:8.4e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 07:10:21 ts/train.py:232 step:238K smpl:4M ep:20K epch:65.50 loss:0.006 grdn:0.135 lr:8.4e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 07:11:12 ts/train.py:232 step:238K smpl:4M ep:20K epch:65.55 loss:0.006 grdn:0.135 lr:8.4e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 07:12:09 ts/train.py:232 step:238K smpl:4M ep:20K epch:65.61 loss:0.006 grdn:0.131 lr:8.4e-05 updt_s:0.162 data_s:0.119 +INFO 2025-05-06 07:13:02 ts/train.py:232 step:238K smpl:4M ep:20K epch:65.66 loss:0.006 grdn:0.136 lr:8.4e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 07:13:56 ts/train.py:232 step:239K smpl:4M ep:20K epch:65.72 loss:0.006 grdn:0.128 lr:8.4e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 07:14:50 ts/train.py:232 step:239K smpl:4M ep:20K epch:65.77 loss:0.006 grdn:0.134 lr:8.4e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 07:15:44 ts/train.py:232 step:239K smpl:4M ep:20K epch:65.83 loss:0.006 grdn:0.137 lr:8.4e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 07:16:39 ts/train.py:232 step:239K smpl:4M ep:20K epch:65.88 loss:0.006 grdn:0.131 lr:8.4e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-06 07:17:32 ts/train.py:232 step:239K smpl:4M ep:20K epch:65.94 loss:0.006 grdn:0.137 lr:8.4e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 07:18:25 ts/train.py:232 step:240K smpl:4M ep:20K epch:65.99 loss:0.005 grdn:0.125 lr:8.4e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 07:19:17 ts/train.py:232 step:240K smpl:4M ep:20K epch:66.05 loss:0.006 grdn:0.131 lr:8.4e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 07:20:10 ts/train.py:232 step:240K smpl:4M ep:20K epch:66.10 loss:0.006 grdn:0.131 lr:8.4e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 07:20:10 ts/train.py:241 Checkpoint policy after step 240000 +INFO 2025-05-06 07:21:06 ts/train.py:232 step:240K smpl:4M ep:20K epch:66.16 loss:0.006 grdn:0.143 lr:8.3e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 07:21:59 ts/train.py:232 step:240K smpl:4M ep:20K epch:66.21 loss:0.006 grdn:0.146 lr:8.3e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 07:22:51 ts/train.py:232 step:241K smpl:4M ep:20K epch:66.27 loss:0.006 grdn:0.138 lr:8.3e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 07:23:44 ts/train.py:232 step:241K smpl:4M ep:20K epch:66.32 loss:0.007 grdn:0.155 lr:8.3e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 07:24:37 ts/train.py:232 step:241K smpl:4M ep:20K epch:66.38 loss:0.006 grdn:0.133 lr:8.3e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 07:25:30 ts/train.py:232 step:241K smpl:4M ep:20K epch:66.43 loss:0.006 grdn:0.131 lr:8.3e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 07:26:22 ts/train.py:232 step:241K smpl:4M ep:20K epch:66.49 loss:0.005 grdn:0.125 lr:8.3e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 07:27:17 ts/train.py:232 step:242K smpl:4M ep:20K epch:66.55 loss:0.006 grdn:0.138 lr:8.3e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 07:28:11 ts/train.py:232 step:242K smpl:4M ep:20K epch:66.60 loss:0.005 grdn:0.133 lr:8.3e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 07:29:05 ts/train.py:232 step:242K smpl:4M ep:20K epch:66.66 loss:0.006 grdn:0.135 lr:8.3e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 07:29:58 ts/train.py:232 step:242K smpl:4M ep:20K epch:66.71 loss:0.006 grdn:0.138 lr:8.3e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 07:30:50 ts/train.py:232 step:242K smpl:4M ep:20K epch:66.77 loss:0.006 grdn:0.136 lr:8.3e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 07:31:44 ts/train.py:232 step:243K smpl:4M ep:20K epch:66.82 loss:0.007 grdn:0.153 lr:8.3e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 07:32:38 ts/train.py:232 step:243K smpl:4M ep:20K epch:66.88 loss:0.006 grdn:0.145 lr:8.3e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 07:33:30 ts/train.py:232 step:243K smpl:4M ep:20K epch:66.93 loss:0.006 grdn:0.137 lr:8.3e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 07:34:23 ts/train.py:232 step:243K smpl:4M ep:20K epch:66.99 loss:0.006 grdn:0.134 lr:8.3e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-06 07:35:15 ts/train.py:232 step:243K smpl:4M ep:20K epch:67.04 loss:0.006 grdn:0.132 lr:8.3e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 07:36:07 ts/train.py:232 step:244K smpl:4M ep:20K epch:67.10 loss:0.006 grdn:0.139 lr:8.3e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-06 07:37:00 ts/train.py:232 step:244K smpl:4M ep:20K epch:67.15 loss:0.006 grdn:0.131 lr:8.3e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 07:37:52 ts/train.py:232 step:244K smpl:4M ep:20K epch:67.21 loss:0.006 grdn:0.126 lr:8.3e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 07:38:44 ts/train.py:232 step:244K smpl:4M ep:20K epch:67.26 loss:0.006 grdn:0.137 lr:8.3e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 07:39:36 ts/train.py:232 step:244K smpl:4M ep:20K epch:67.32 loss:0.007 grdn:0.143 lr:8.3e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 07:40:29 ts/train.py:232 step:245K smpl:4M ep:20K epch:67.37 loss:0.006 grdn:0.132 lr:8.3e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 07:41:21 ts/train.py:232 step:245K smpl:4M ep:20K epch:67.43 loss:0.007 grdn:0.138 lr:8.3e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 07:42:13 ts/train.py:232 step:245K smpl:4M ep:20K epch:67.48 loss:0.006 grdn:0.130 lr:8.3e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 07:43:10 ts/train.py:232 step:245K smpl:4M ep:20K epch:67.54 loss:0.006 grdn:0.143 lr:8.3e-05 updt_s:0.161 data_s:0.121 +INFO 2025-05-06 07:44:03 ts/train.py:232 step:245K smpl:4M ep:20K epch:67.59 loss:0.006 grdn:0.136 lr:8.3e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 07:44:56 ts/train.py:232 step:246K smpl:4M ep:20K epch:67.65 loss:0.006 grdn:0.139 lr:8.3e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 07:45:50 ts/train.py:232 step:246K smpl:4M ep:20K epch:67.70 loss:0.006 grdn:0.140 lr:8.3e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 07:46:43 ts/train.py:232 step:246K smpl:4M ep:20K epch:67.76 loss:0.006 grdn:0.131 lr:8.3e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 07:47:37 ts/train.py:232 step:246K smpl:4M ep:20K epch:67.81 loss:0.006 grdn:0.135 lr:8.3e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 07:48:30 ts/train.py:232 step:246K smpl:4M ep:20K epch:67.87 loss:0.006 grdn:0.133 lr:8.3e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 07:49:23 ts/train.py:232 step:247K smpl:4M ep:20K epch:67.92 loss:0.006 grdn:0.139 lr:8.3e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 07:50:15 ts/train.py:232 step:247K smpl:4M ep:20K epch:67.98 loss:0.006 grdn:0.135 lr:8.3e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 07:51:07 ts/train.py:232 step:247K smpl:4M ep:20K epch:68.03 loss:0.006 grdn:0.129 lr:8.3e-05 updt_s:0.160 data_s:0.098 +INFO 2025-05-06 07:52:00 ts/train.py:232 step:247K smpl:4M ep:20K epch:68.09 loss:0.006 grdn:0.131 lr:8.3e-05 updt_s:0.160 data_s:0.100 +INFO 2025-05-06 07:52:51 ts/train.py:232 step:247K smpl:4M ep:20K epch:68.14 loss:0.006 grdn:0.128 lr:8.3e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 07:53:45 ts/train.py:232 step:248K smpl:4M ep:20K epch:68.20 loss:0.006 grdn:0.137 lr:8.3e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 07:54:38 ts/train.py:232 step:248K smpl:4M ep:20K epch:68.25 loss:0.006 grdn:0.142 lr:8.2e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 07:55:29 ts/train.py:232 step:248K smpl:4M ep:20K epch:68.31 loss:0.006 grdn:0.134 lr:8.2e-05 updt_s:0.160 data_s:0.096 +INFO 2025-05-06 07:56:22 ts/train.py:232 step:248K smpl:4M ep:21K epch:68.36 loss:0.006 grdn:0.141 lr:8.2e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 07:57:14 ts/train.py:232 step:248K smpl:4M ep:21K epch:68.42 loss:0.006 grdn:0.130 lr:8.2e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 07:58:07 ts/train.py:232 step:249K smpl:4M ep:21K epch:68.47 loss:0.007 grdn:0.141 lr:8.2e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 07:58:56 ts/train.py:232 step:249K smpl:4M ep:21K epch:68.53 loss:0.006 grdn:0.129 lr:8.2e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-06 07:59:45 ts/train.py:232 step:249K smpl:4M ep:21K epch:68.58 loss:0.006 grdn:0.136 lr:8.2e-05 updt_s:0.161 data_s:0.083 +INFO 2025-05-06 08:00:35 ts/train.py:232 step:249K smpl:4M ep:21K epch:68.64 loss:0.006 grdn:0.127 lr:8.2e-05 updt_s:0.161 data_s:0.086 +INFO 2025-05-06 08:01:26 ts/train.py:232 step:249K smpl:4M ep:21K epch:68.69 loss:0.006 grdn:0.134 lr:8.2e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-06 08:02:16 ts/train.py:232 step:250K smpl:4M ep:21K epch:68.75 loss:0.006 grdn:0.136 lr:8.2e-05 updt_s:0.161 data_s:0.089 +INFO 2025-05-06 08:03:07 ts/train.py:232 step:250K smpl:4M ep:21K epch:68.80 loss:0.006 grdn:0.136 lr:8.2e-05 updt_s:0.163 data_s:0.089 +INFO 2025-05-06 08:03:56 ts/train.py:232 step:250K smpl:4M ep:21K epch:68.86 loss:0.006 grdn:0.140 lr:8.2e-05 updt_s:0.162 data_s:0.083 +INFO 2025-05-06 08:04:45 ts/train.py:232 step:250K smpl:4M ep:21K epch:68.91 loss:0.006 grdn:0.131 lr:8.2e-05 updt_s:0.162 data_s:0.083 +INFO 2025-05-06 08:05:34 ts/train.py:232 step:250K smpl:4M ep:21K epch:68.97 loss:0.006 grdn:0.127 lr:8.2e-05 updt_s:0.161 data_s:0.083 +INFO 2025-05-06 08:06:24 ts/train.py:232 step:251K smpl:4M ep:21K epch:69.02 loss:0.006 grdn:0.140 lr:8.2e-05 updt_s:0.162 data_s:0.088 +INFO 2025-05-06 08:07:14 ts/train.py:232 step:251K smpl:4M ep:21K epch:69.08 loss:0.006 grdn:0.133 lr:8.2e-05 updt_s:0.161 data_s:0.089 +INFO 2025-05-06 08:08:04 ts/train.py:232 step:251K smpl:4M ep:21K epch:69.13 loss:0.006 grdn:0.135 lr:8.2e-05 updt_s:0.160 data_s:0.087 +INFO 2025-05-06 08:08:52 ts/train.py:232 step:251K smpl:4M ep:21K epch:69.19 loss:0.006 grdn:0.140 lr:8.2e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 08:09:40 ts/train.py:232 step:251K smpl:4M ep:21K epch:69.24 loss:0.006 grdn:0.139 lr:8.2e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 08:10:29 ts/train.py:232 step:252K smpl:4M ep:21K epch:69.30 loss:0.007 grdn:0.146 lr:8.2e-05 updt_s:0.161 data_s:0.083 +INFO 2025-05-06 08:11:19 ts/train.py:232 step:252K smpl:4M ep:21K epch:69.35 loss:0.006 grdn:0.141 lr:8.2e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-06 08:12:07 ts/train.py:232 step:252K smpl:4M ep:21K epch:69.41 loss:0.005 grdn:0.126 lr:8.2e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 08:13:00 ts/train.py:232 step:252K smpl:4M ep:21K epch:69.46 loss:0.006 grdn:0.137 lr:8.2e-05 updt_s:0.163 data_s:0.099 +INFO 2025-05-06 08:13:49 ts/train.py:232 step:252K smpl:4M ep:21K epch:69.52 loss:0.005 grdn:0.131 lr:8.2e-05 updt_s:0.162 data_s:0.086 +INFO 2025-05-06 08:14:39 ts/train.py:232 step:253K smpl:4M ep:21K epch:69.57 loss:0.006 grdn:0.133 lr:8.2e-05 updt_s:0.162 data_s:0.088 +INFO 2025-05-06 08:15:29 ts/train.py:232 step:253K smpl:4M ep:21K epch:69.63 loss:0.006 grdn:0.129 lr:8.2e-05 updt_s:0.162 data_s:0.085 +INFO 2025-05-06 08:16:19 ts/train.py:232 step:253K smpl:4M ep:21K epch:69.68 loss:0.006 grdn:0.134 lr:8.2e-05 updt_s:0.162 data_s:0.087 +INFO 2025-05-06 08:17:09 ts/train.py:232 step:253K smpl:4M ep:21K epch:69.74 loss:0.006 grdn:0.140 lr:8.2e-05 updt_s:0.162 data_s:0.087 +INFO 2025-05-06 08:17:59 ts/train.py:232 step:253K smpl:4M ep:21K epch:69.80 loss:0.006 grdn:0.134 lr:8.2e-05 updt_s:0.161 data_s:0.089 +INFO 2025-05-06 08:18:50 ts/train.py:232 step:254K smpl:4M ep:21K epch:69.85 loss:0.005 grdn:0.134 lr:8.2e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-06 08:19:41 ts/train.py:232 step:254K smpl:4M ep:21K epch:69.91 loss:0.006 grdn:0.132 lr:8.2e-05 updt_s:0.161 data_s:0.092 +INFO 2025-05-06 08:20:32 ts/train.py:232 step:254K smpl:4M ep:21K epch:69.96 loss:0.005 grdn:0.128 lr:8.2e-05 updt_s:0.162 data_s:0.093 +INFO 2025-05-06 08:21:24 ts/train.py:232 step:254K smpl:4M ep:21K epch:70.02 loss:0.006 grdn:0.140 lr:8.2e-05 updt_s:0.161 data_s:0.095 +INFO 2025-05-06 08:22:14 ts/train.py:232 step:254K smpl:4M ep:21K epch:70.07 loss:0.006 grdn:0.139 lr:8.2e-05 updt_s:0.160 data_s:0.090 +INFO 2025-05-06 08:23:05 ts/train.py:232 step:255K smpl:4M ep:21K epch:70.13 loss:0.006 grdn:0.142 lr:8.2e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-06 08:23:55 ts/train.py:232 step:255K smpl:4M ep:21K epch:70.18 loss:0.006 grdn:0.136 lr:8.2e-05 updt_s:0.161 data_s:0.090 +INFO 2025-05-06 08:24:46 ts/train.py:232 step:255K smpl:4M ep:21K epch:70.24 loss:0.006 grdn:0.134 lr:8.2e-05 updt_s:0.161 data_s:0.092 +INFO 2025-05-06 08:25:37 ts/train.py:232 step:255K smpl:4M ep:21K epch:70.29 loss:0.006 grdn:0.132 lr:8.2e-05 updt_s:0.160 data_s:0.096 +INFO 2025-05-06 08:26:29 ts/train.py:232 step:255K smpl:4M ep:21K epch:70.35 loss:0.006 grdn:0.135 lr:8.1e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 08:27:23 ts/train.py:232 step:256K smpl:4M ep:21K epch:70.40 loss:0.005 grdn:0.132 lr:8.1e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 08:28:17 ts/train.py:232 step:256K smpl:4M ep:21K epch:70.46 loss:0.006 grdn:0.137 lr:8.1e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 08:29:10 ts/train.py:232 step:256K smpl:4M ep:21K epch:70.51 loss:0.006 grdn:0.132 lr:8.1e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 08:30:03 ts/train.py:232 step:256K smpl:4M ep:21K epch:70.57 loss:0.006 grdn:0.135 lr:8.1e-05 updt_s:0.162 data_s:0.102 +INFO 2025-05-06 08:30:56 ts/train.py:232 step:256K smpl:4M ep:21K epch:70.62 loss:0.006 grdn:0.135 lr:8.1e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 08:31:50 ts/train.py:232 step:257K smpl:4M ep:21K epch:70.68 loss:0.006 grdn:0.134 lr:8.1e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 08:32:43 ts/train.py:232 step:257K smpl:4M ep:21K epch:70.73 loss:0.006 grdn:0.137 lr:8.1e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 08:33:35 ts/train.py:232 step:257K smpl:4M ep:21K epch:70.79 loss:0.006 grdn:0.137 lr:8.1e-05 updt_s:0.162 data_s:0.099 +INFO 2025-05-06 08:34:28 ts/train.py:232 step:257K smpl:4M ep:21K epch:70.84 loss:0.006 grdn:0.130 lr:8.1e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 08:35:20 ts/train.py:232 step:257K smpl:4M ep:21K epch:70.90 loss:0.006 grdn:0.137 lr:8.1e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 08:36:13 ts/train.py:232 step:258K smpl:4M ep:21K epch:70.95 loss:0.006 grdn:0.133 lr:8.1e-05 updt_s:0.161 data_s:0.101 +INFO 2025-05-06 08:37:05 ts/train.py:232 step:258K smpl:4M ep:21K epch:71.01 loss:0.005 grdn:0.127 lr:8.1e-05 updt_s:0.162 data_s:0.101 +INFO 2025-05-06 08:37:58 ts/train.py:232 step:258K smpl:4M ep:21K epch:71.06 loss:0.006 grdn:0.134 lr:8.1e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 08:38:50 ts/train.py:232 step:258K smpl:4M ep:21K epch:71.12 loss:0.006 grdn:0.134 lr:8.1e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 08:39:42 ts/train.py:232 step:258K smpl:4M ep:21K epch:71.17 loss:0.006 grdn:0.138 lr:8.1e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 08:40:36 ts/train.py:232 step:259K smpl:4M ep:21K epch:71.23 loss:0.006 grdn:0.135 lr:8.1e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 08:41:28 ts/train.py:232 step:259K smpl:4M ep:21K epch:71.28 loss:0.006 grdn:0.132 lr:8.1e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 08:42:21 ts/train.py:232 step:259K smpl:4M ep:21K epch:71.34 loss:0.006 grdn:0.129 lr:8.1e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 08:43:23 ts/train.py:232 step:259K smpl:4M ep:21K epch:71.39 loss:0.006 grdn:0.135 lr:8.1e-05 updt_s:0.161 data_s:0.145 +INFO 2025-05-06 08:44:16 ts/train.py:232 step:259K smpl:4M ep:21K epch:71.45 loss:0.006 grdn:0.134 lr:8.1e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 08:45:09 ts/train.py:232 step:260K smpl:4M ep:21K epch:71.50 loss:0.005 grdn:0.125 lr:8.1e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 08:46:02 ts/train.py:232 step:260K smpl:4M ep:21K epch:71.56 loss:0.006 grdn:0.142 lr:8.1e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 08:46:55 ts/train.py:232 step:260K smpl:4M ep:21K epch:71.61 loss:0.006 grdn:0.137 lr:8.1e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 08:47:48 ts/train.py:232 step:260K smpl:4M ep:22K epch:71.67 loss:0.006 grdn:0.130 lr:8.1e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 08:48:41 ts/train.py:232 step:260K smpl:4M ep:22K epch:71.72 loss:0.006 grdn:0.134 lr:8.1e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 08:49:35 ts/train.py:232 step:261K smpl:4M ep:22K epch:71.78 loss:0.006 grdn:0.136 lr:8.1e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 08:50:30 ts/train.py:232 step:261K smpl:4M ep:22K epch:71.83 loss:0.006 grdn:0.142 lr:8.1e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 08:51:25 ts/train.py:232 step:261K smpl:4M ep:22K epch:71.89 loss:0.006 grdn:0.135 lr:8.1e-05 updt_s:0.162 data_s:0.112 +INFO 2025-05-06 08:52:19 ts/train.py:232 step:261K smpl:4M ep:22K epch:71.94 loss:0.006 grdn:0.141 lr:8.1e-05 updt_s:0.160 data_s:0.108 +INFO 2025-05-06 08:53:12 ts/train.py:232 step:261K smpl:4M ep:22K epch:72.00 loss:0.006 grdn:0.137 lr:8.1e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 08:54:05 ts/train.py:232 step:262K smpl:4M ep:22K epch:72.05 loss:0.005 grdn:0.128 lr:8.1e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 08:54:59 ts/train.py:232 step:262K smpl:4M ep:22K epch:72.11 loss:0.006 grdn:0.138 lr:8.1e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 08:55:52 ts/train.py:232 step:262K smpl:4M ep:22K epch:72.16 loss:0.006 grdn:0.135 lr:8.1e-05 updt_s:0.160 data_s:0.108 +INFO 2025-05-06 08:56:46 ts/train.py:232 step:262K smpl:4M ep:22K epch:72.22 loss:0.005 grdn:0.133 lr:8.1e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 08:57:39 ts/train.py:232 step:262K smpl:4M ep:22K epch:72.27 loss:0.006 grdn:0.143 lr:8.1e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 08:58:34 ts/train.py:232 step:263K smpl:4M ep:22K epch:72.33 loss:0.006 grdn:0.135 lr:8.0e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 08:59:24 ts/train.py:232 step:263K smpl:4M ep:22K epch:72.38 loss:0.006 grdn:0.135 lr:8.0e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-06 09:00:13 ts/train.py:232 step:263K smpl:4M ep:22K epch:72.44 loss:0.006 grdn:0.140 lr:8.0e-05 updt_s:0.161 data_s:0.087 +INFO 2025-05-06 09:01:05 ts/train.py:232 step:263K smpl:4M ep:22K epch:72.49 loss:0.006 grdn:0.132 lr:8.0e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-06 09:01:56 ts/train.py:232 step:263K smpl:4M ep:22K epch:72.55 loss:0.005 grdn:0.128 lr:8.0e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 09:02:46 ts/train.py:232 step:264K smpl:4M ep:22K epch:72.60 loss:0.006 grdn:0.132 lr:8.0e-05 updt_s:0.163 data_s:0.085 +INFO 2025-05-06 09:03:37 ts/train.py:232 step:264K smpl:4M ep:22K epch:72.66 loss:0.005 grdn:0.125 lr:8.0e-05 updt_s:0.161 data_s:0.092 +INFO 2025-05-06 09:04:29 ts/train.py:232 step:264K smpl:4M ep:22K epch:72.71 loss:0.006 grdn:0.134 lr:8.0e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 09:05:21 ts/train.py:232 step:264K smpl:4M ep:22K epch:72.77 loss:0.006 grdn:0.135 lr:8.0e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 09:06:13 ts/train.py:232 step:264K smpl:4M ep:22K epch:72.82 loss:0.006 grdn:0.133 lr:8.0e-05 updt_s:0.162 data_s:0.098 +INFO 2025-05-06 09:07:04 ts/train.py:232 step:265K smpl:4M ep:22K epch:72.88 loss:0.005 grdn:0.132 lr:8.0e-05 updt_s:0.162 data_s:0.090 +INFO 2025-05-06 09:07:56 ts/train.py:232 step:265K smpl:4M ep:22K epch:72.94 loss:0.005 grdn:0.132 lr:8.0e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 09:08:48 ts/train.py:232 step:265K smpl:4M ep:22K epch:72.99 loss:0.005 grdn:0.127 lr:8.0e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 09:09:39 ts/train.py:232 step:265K smpl:4M ep:22K epch:73.05 loss:0.005 grdn:0.130 lr:8.0e-05 updt_s:0.161 data_s:0.096 +INFO 2025-05-06 09:10:30 ts/train.py:232 step:265K smpl:4M ep:22K epch:73.10 loss:0.005 grdn:0.129 lr:8.0e-05 updt_s:0.160 data_s:0.095 +INFO 2025-05-06 09:11:21 ts/train.py:232 step:266K smpl:4M ep:22K epch:73.16 loss:0.006 grdn:0.135 lr:8.0e-05 updt_s:0.161 data_s:0.089 +INFO 2025-05-06 09:12:13 ts/train.py:232 step:266K smpl:4M ep:22K epch:73.21 loss:0.006 grdn:0.138 lr:8.0e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 09:13:05 ts/train.py:232 step:266K smpl:4M ep:22K epch:73.27 loss:0.005 grdn:0.130 lr:8.0e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 09:14:01 ts/train.py:232 step:266K smpl:4M ep:22K epch:73.32 loss:0.006 grdn:0.133 lr:8.0e-05 updt_s:0.161 data_s:0.121 +INFO 2025-05-06 09:14:57 ts/train.py:232 step:266K smpl:4M ep:22K epch:73.38 loss:0.005 grdn:0.131 lr:8.0e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 09:15:52 ts/train.py:232 step:267K smpl:4M ep:22K epch:73.43 loss:0.006 grdn:0.128 lr:8.0e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-06 09:16:46 ts/train.py:232 step:267K smpl:4M ep:22K epch:73.49 loss:0.006 grdn:0.137 lr:8.0e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 09:17:40 ts/train.py:232 step:267K smpl:4M ep:22K epch:73.54 loss:0.006 grdn:0.141 lr:8.0e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 09:18:34 ts/train.py:232 step:267K smpl:4M ep:22K epch:73.60 loss:0.006 grdn:0.145 lr:8.0e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 09:19:33 ts/train.py:232 step:267K smpl:4M ep:22K epch:73.65 loss:0.005 grdn:0.126 lr:8.0e-05 updt_s:0.160 data_s:0.134 +INFO 2025-05-06 09:20:34 ts/train.py:232 step:268K smpl:4M ep:22K epch:73.71 loss:0.006 grdn:0.141 lr:8.0e-05 updt_s:0.161 data_s:0.146 +INFO 2025-05-06 09:21:36 ts/train.py:232 step:268K smpl:4M ep:22K epch:73.76 loss:0.006 grdn:0.137 lr:8.0e-05 updt_s:0.160 data_s:0.146 +INFO 2025-05-06 09:22:37 ts/train.py:232 step:268K smpl:4M ep:22K epch:73.82 loss:0.006 grdn:0.136 lr:8.0e-05 updt_s:0.160 data_s:0.146 +INFO 2025-05-06 09:23:38 ts/train.py:232 step:268K smpl:4M ep:22K epch:73.87 loss:0.005 grdn:0.131 lr:8.0e-05 updt_s:0.160 data_s:0.140 +INFO 2025-05-06 09:24:39 ts/train.py:232 step:268K smpl:4M ep:22K epch:73.93 loss:0.006 grdn:0.136 lr:8.0e-05 updt_s:0.160 data_s:0.143 +INFO 2025-05-06 09:25:39 ts/train.py:232 step:269K smpl:4M ep:22K epch:73.98 loss:0.005 grdn:0.124 lr:8.0e-05 updt_s:0.160 data_s:0.140 +INFO 2025-05-06 09:26:40 ts/train.py:232 step:269K smpl:4M ep:22K epch:74.04 loss:0.005 grdn:0.132 lr:8.0e-05 updt_s:0.161 data_s:0.143 +INFO 2025-05-06 09:27:41 ts/train.py:232 step:269K smpl:4M ep:22K epch:74.09 loss:0.006 grdn:0.140 lr:8.0e-05 updt_s:0.160 data_s:0.145 +INFO 2025-05-06 09:28:42 ts/train.py:232 step:269K smpl:4M ep:22K epch:74.15 loss:0.006 grdn:0.136 lr:8.0e-05 updt_s:0.161 data_s:0.143 +INFO 2025-05-06 09:29:43 ts/train.py:232 step:269K smpl:4M ep:22K epch:74.20 loss:0.006 grdn:0.133 lr:8.0e-05 updt_s:0.161 data_s:0.142 +INFO 2025-05-06 09:30:40 ts/train.py:232 step:270K smpl:4M ep:22K epch:74.26 loss:0.006 grdn:0.136 lr:8.0e-05 updt_s:0.160 data_s:0.124 +INFO 2025-05-06 09:31:31 ts/train.py:232 step:270K smpl:4M ep:22K epch:74.31 loss:0.006 grdn:0.133 lr:7.9e-05 updt_s:0.161 data_s:0.093 +INFO 2025-05-06 09:32:23 ts/train.py:232 step:270K smpl:4M ep:22K epch:74.37 loss:0.006 grdn:0.134 lr:7.9e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 09:32:23 ts/train.py:241 Checkpoint policy after step 270000 +INFO 2025-05-06 09:33:17 ts/train.py:232 step:270K smpl:4M ep:22K epch:74.42 loss:0.006 grdn:0.140 lr:7.9e-05 updt_s:0.161 data_s:0.092 +INFO 2025-05-06 09:34:09 ts/train.py:232 step:270K smpl:4M ep:22K epch:74.48 loss:0.006 grdn:0.132 lr:7.9e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 09:35:01 ts/train.py:232 step:271K smpl:4M ep:22K epch:74.53 loss:0.005 grdn:0.127 lr:7.9e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 09:35:54 ts/train.py:232 step:271K smpl:4M ep:22K epch:74.59 loss:0.006 grdn:0.128 lr:7.9e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 09:36:45 ts/train.py:232 step:271K smpl:4M ep:22K epch:74.64 loss:0.006 grdn:0.138 lr:7.9e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 09:37:38 ts/train.py:232 step:271K smpl:4M ep:22K epch:74.70 loss:0.006 grdn:0.139 lr:7.9e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 09:38:29 ts/train.py:232 step:271K smpl:4M ep:22K epch:74.75 loss:0.005 grdn:0.127 lr:7.9e-05 updt_s:0.161 data_s:0.095 +INFO 2025-05-06 09:39:21 ts/train.py:232 step:272K smpl:4M ep:22K epch:74.81 loss:0.006 grdn:0.138 lr:7.9e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 09:40:13 ts/train.py:232 step:272K smpl:4M ep:22K epch:74.86 loss:0.005 grdn:0.127 lr:7.9e-05 updt_s:0.160 data_s:0.098 +INFO 2025-05-06 09:41:05 ts/train.py:232 step:272K smpl:4M ep:22K epch:74.92 loss:0.006 grdn:0.138 lr:7.9e-05 updt_s:0.160 data_s:0.098 +INFO 2025-05-06 09:41:56 ts/train.py:232 step:272K smpl:4M ep:22K epch:74.97 loss:0.006 grdn:0.134 lr:7.9e-05 updt_s:0.161 data_s:0.094 +INFO 2025-05-06 09:42:48 ts/train.py:232 step:272K smpl:4M ep:23K epch:75.03 loss:0.006 grdn:0.142 lr:7.9e-05 updt_s:0.162 data_s:0.095 +INFO 2025-05-06 09:43:40 ts/train.py:232 step:273K smpl:4M ep:23K epch:75.08 loss:0.005 grdn:0.125 lr:7.9e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 09:44:33 ts/train.py:232 step:273K smpl:4M ep:23K epch:75.14 loss:0.005 grdn:0.129 lr:7.9e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 09:45:26 ts/train.py:232 step:273K smpl:4M ep:23K epch:75.19 loss:0.006 grdn:0.135 lr:7.9e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 09:46:19 ts/train.py:232 step:273K smpl:4M ep:23K epch:75.25 loss:0.005 grdn:0.125 lr:7.9e-05 updt_s:0.161 data_s:0.104 +INFO 2025-05-06 09:47:12 ts/train.py:232 step:273K smpl:4M ep:23K epch:75.30 loss:0.006 grdn:0.138 lr:7.9e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 09:48:05 ts/train.py:232 step:274K smpl:4M ep:23K epch:75.36 loss:0.006 grdn:0.135 lr:7.9e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 09:48:55 ts/train.py:232 step:274K smpl:4M ep:23K epch:75.41 loss:0.006 grdn:0.140 lr:7.9e-05 updt_s:0.162 data_s:0.090 +INFO 2025-05-06 09:49:46 ts/train.py:232 step:274K smpl:4M ep:23K epch:75.47 loss:0.006 grdn:0.132 lr:7.9e-05 updt_s:0.161 data_s:0.094 +INFO 2025-05-06 09:50:38 ts/train.py:232 step:274K smpl:4M ep:23K epch:75.52 loss:0.005 grdn:0.128 lr:7.9e-05 updt_s:0.160 data_s:0.098 +INFO 2025-05-06 09:51:31 ts/train.py:232 step:274K smpl:4M ep:23K epch:75.58 loss:0.006 grdn:0.132 lr:7.9e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 09:52:24 ts/train.py:232 step:275K smpl:4M ep:23K epch:75.63 loss:0.006 grdn:0.129 lr:7.9e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 09:53:16 ts/train.py:232 step:275K smpl:4M ep:23K epch:75.69 loss:0.006 grdn:0.136 lr:7.9e-05 updt_s:0.161 data_s:0.099 +INFO 2025-05-06 09:54:09 ts/train.py:232 step:275K smpl:4M ep:23K epch:75.74 loss:0.006 grdn:0.142 lr:7.9e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 09:55:02 ts/train.py:232 step:275K smpl:4M ep:23K epch:75.80 loss:0.005 grdn:0.133 lr:7.9e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 09:55:55 ts/train.py:232 step:275K smpl:4M ep:23K epch:75.85 loss:0.006 grdn:0.136 lr:7.9e-05 updt_s:0.162 data_s:0.104 +INFO 2025-05-06 09:56:48 ts/train.py:232 step:276K smpl:4M ep:23K epch:75.91 loss:0.005 grdn:0.129 lr:7.9e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 09:57:40 ts/train.py:232 step:276K smpl:4M ep:23K epch:75.96 loss:0.006 grdn:0.128 lr:7.9e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 09:58:30 ts/train.py:232 step:276K smpl:4M ep:23K epch:76.02 loss:0.006 grdn:0.136 lr:7.9e-05 updt_s:0.160 data_s:0.090 +INFO 2025-05-06 09:59:22 ts/train.py:232 step:276K smpl:4M ep:23K epch:76.08 loss:0.006 grdn:0.132 lr:7.9e-05 updt_s:0.161 data_s:0.095 +INFO 2025-05-06 10:00:14 ts/train.py:232 step:276K smpl:4M ep:23K epch:76.13 loss:0.005 grdn:0.130 lr:7.9e-05 updt_s:0.160 data_s:0.099 +INFO 2025-05-06 10:01:08 ts/train.py:232 step:277K smpl:4M ep:23K epch:76.19 loss:0.006 grdn:0.138 lr:7.9e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 10:02:03 ts/train.py:232 step:277K smpl:4M ep:23K epch:76.24 loss:0.006 grdn:0.141 lr:7.8e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 10:02:56 ts/train.py:232 step:277K smpl:4M ep:23K epch:76.30 loss:0.006 grdn:0.133 lr:7.8e-05 updt_s:0.161 data_s:0.107 +INFO 2025-05-06 10:03:51 ts/train.py:232 step:277K smpl:4M ep:23K epch:76.35 loss:0.006 grdn:0.134 lr:7.8e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 10:04:44 ts/train.py:232 step:277K smpl:4M ep:23K epch:76.41 loss:0.006 grdn:0.134 lr:7.8e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 10:05:38 ts/train.py:232 step:278K smpl:4M ep:23K epch:76.46 loss:0.006 grdn:0.137 lr:7.8e-05 updt_s:0.159 data_s:0.109 +INFO 2025-05-06 10:06:31 ts/train.py:232 step:278K smpl:4M ep:23K epch:76.52 loss:0.006 grdn:0.136 lr:7.8e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 10:07:23 ts/train.py:232 step:278K smpl:4M ep:23K epch:76.57 loss:0.006 grdn:0.140 lr:7.8e-05 updt_s:0.159 data_s:0.103 +INFO 2025-05-06 10:08:17 ts/train.py:232 step:278K smpl:4M ep:23K epch:76.63 loss:0.005 grdn:0.126 lr:7.8e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 10:09:09 ts/train.py:232 step:278K smpl:4M ep:23K epch:76.68 loss:0.006 grdn:0.141 lr:7.8e-05 updt_s:0.159 data_s:0.102 +INFO 2025-05-06 10:10:02 ts/train.py:232 step:279K smpl:4M ep:23K epch:76.74 loss:0.005 grdn:0.132 lr:7.8e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 10:10:55 ts/train.py:232 step:279K smpl:4M ep:23K epch:76.79 loss:0.006 grdn:0.129 lr:7.8e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 10:11:49 ts/train.py:232 step:279K smpl:4M ep:23K epch:76.85 loss:0.006 grdn:0.130 lr:7.8e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 10:12:43 ts/train.py:232 step:279K smpl:4M ep:23K epch:76.90 loss:0.005 grdn:0.133 lr:7.8e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 10:13:38 ts/train.py:232 step:279K smpl:4M ep:23K epch:76.96 loss:0.006 grdn:0.140 lr:7.8e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-06 10:14:35 ts/train.py:232 step:280K smpl:4M ep:23K epch:77.01 loss:0.006 grdn:0.136 lr:7.8e-05 updt_s:0.160 data_s:0.124 +INFO 2025-05-06 10:15:31 ts/train.py:232 step:280K smpl:4M ep:23K epch:77.07 loss:0.005 grdn:0.130 lr:7.8e-05 updt_s:0.160 data_s:0.118 +INFO 2025-05-06 10:16:27 ts/train.py:232 step:280K smpl:4M ep:23K epch:77.12 loss:0.005 grdn:0.126 lr:7.8e-05 updt_s:0.159 data_s:0.120 +INFO 2025-05-06 10:17:22 ts/train.py:232 step:280K smpl:4M ep:23K epch:77.18 loss:0.005 grdn:0.134 lr:7.8e-05 updt_s:0.160 data_s:0.113 +INFO 2025-05-06 10:18:15 ts/train.py:232 step:280K smpl:4M ep:23K epch:77.23 loss:0.005 grdn:0.129 lr:7.8e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 10:19:07 ts/train.py:232 step:281K smpl:4M ep:23K epch:77.29 loss:0.005 grdn:0.127 lr:7.8e-05 updt_s:0.161 data_s:0.100 +INFO 2025-05-06 10:20:00 ts/train.py:232 step:281K smpl:4M ep:23K epch:77.34 loss:0.006 grdn:0.139 lr:7.8e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 10:20:53 ts/train.py:232 step:281K smpl:4M ep:23K epch:77.40 loss:0.006 grdn:0.136 lr:7.8e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 10:21:46 ts/train.py:232 step:281K smpl:4M ep:23K epch:77.45 loss:0.006 grdn:0.131 lr:7.8e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 10:22:38 ts/train.py:232 step:281K smpl:5M ep:23K epch:77.51 loss:0.005 grdn:0.125 lr:7.8e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 10:23:31 ts/train.py:232 step:282K smpl:5M ep:23K epch:77.56 loss:0.005 grdn:0.128 lr:7.8e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 10:24:24 ts/train.py:232 step:282K smpl:5M ep:23K epch:77.62 loss:0.006 grdn:0.133 lr:7.8e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 10:25:17 ts/train.py:232 step:282K smpl:5M ep:23K epch:77.67 loss:0.005 grdn:0.132 lr:7.8e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 10:26:09 ts/train.py:232 step:282K smpl:5M ep:23K epch:77.73 loss:0.006 grdn:0.144 lr:7.8e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 10:27:02 ts/train.py:232 step:282K smpl:5M ep:23K epch:77.78 loss:0.006 grdn:0.135 lr:7.8e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 10:27:54 ts/train.py:232 step:283K smpl:5M ep:23K epch:77.84 loss:0.005 grdn:0.134 lr:7.8e-05 updt_s:0.161 data_s:0.098 +INFO 2025-05-06 10:28:46 ts/train.py:232 step:283K smpl:5M ep:23K epch:77.89 loss:0.005 grdn:0.127 lr:7.8e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 10:29:38 ts/train.py:232 step:283K smpl:5M ep:23K epch:77.95 loss:0.005 grdn:0.130 lr:7.8e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 10:30:29 ts/train.py:232 step:283K smpl:5M ep:23K epch:78.00 loss:0.005 grdn:0.124 lr:7.8e-05 updt_s:0.160 data_s:0.097 +INFO 2025-05-06 10:31:21 ts/train.py:232 step:283K smpl:5M ep:23K epch:78.06 loss:0.005 grdn:0.125 lr:7.8e-05 updt_s:0.160 data_s:0.099 +INFO 2025-05-06 10:32:16 ts/train.py:232 step:284K smpl:5M ep:23K epch:78.11 loss:0.005 grdn:0.125 lr:7.8e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 10:33:09 ts/train.py:232 step:284K smpl:5M ep:23K epch:78.17 loss:0.005 grdn:0.131 lr:7.7e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 10:34:02 ts/train.py:232 step:284K smpl:5M ep:23K epch:78.22 loss:0.005 grdn:0.134 lr:7.7e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 10:34:55 ts/train.py:232 step:284K smpl:5M ep:23K epch:78.28 loss:0.006 grdn:0.139 lr:7.7e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 10:35:48 ts/train.py:232 step:284K smpl:5M ep:24K epch:78.33 loss:0.006 grdn:0.140 lr:7.7e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 10:36:40 ts/train.py:232 step:285K smpl:5M ep:24K epch:78.39 loss:0.006 grdn:0.132 lr:7.7e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 10:37:33 ts/train.py:232 step:285K smpl:5M ep:24K epch:78.44 loss:0.005 grdn:0.130 lr:7.7e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-06 10:38:26 ts/train.py:232 step:285K smpl:5M ep:24K epch:78.50 loss:0.005 grdn:0.126 lr:7.7e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 10:39:18 ts/train.py:232 step:285K smpl:5M ep:24K epch:78.55 loss:0.006 grdn:0.135 lr:7.7e-05 updt_s:0.160 data_s:0.100 +INFO 2025-05-06 10:40:10 ts/train.py:232 step:285K smpl:5M ep:24K epch:78.61 loss:0.005 grdn:0.131 lr:7.7e-05 updt_s:0.160 data_s:0.098 +INFO 2025-05-06 10:41:02 ts/train.py:232 step:286K smpl:5M ep:24K epch:78.66 loss:0.005 grdn:0.131 lr:7.7e-05 updt_s:0.160 data_s:0.096 +INFO 2025-05-06 10:41:55 ts/train.py:232 step:286K smpl:5M ep:24K epch:78.72 loss:0.005 grdn:0.128 lr:7.7e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 10:42:48 ts/train.py:232 step:286K smpl:5M ep:24K epch:78.77 loss:0.006 grdn:0.137 lr:7.7e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 10:43:41 ts/train.py:232 step:286K smpl:5M ep:24K epch:78.83 loss:0.005 grdn:0.131 lr:7.7e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 10:44:34 ts/train.py:232 step:286K smpl:5M ep:24K epch:78.88 loss:0.006 grdn:0.134 lr:7.7e-05 updt_s:0.159 data_s:0.102 +INFO 2025-05-06 10:45:27 ts/train.py:232 step:287K smpl:5M ep:24K epch:78.94 loss:0.006 grdn:0.135 lr:7.7e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 10:46:22 ts/train.py:232 step:287K smpl:5M ep:24K epch:78.99 loss:0.006 grdn:0.130 lr:7.7e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 10:47:15 ts/train.py:232 step:287K smpl:5M ep:24K epch:79.05 loss:0.006 grdn:0.138 lr:7.7e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 10:48:13 ts/train.py:232 step:287K smpl:5M ep:24K epch:79.10 loss:0.005 grdn:0.124 lr:7.7e-05 updt_s:0.160 data_s:0.125 +INFO 2025-05-06 10:49:07 ts/train.py:232 step:287K smpl:5M ep:24K epch:79.16 loss:0.006 grdn:0.134 lr:7.7e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 10:50:02 ts/train.py:232 step:288K smpl:5M ep:24K epch:79.22 loss:0.005 grdn:0.132 lr:7.7e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-06 10:50:57 ts/train.py:232 step:288K smpl:5M ep:24K epch:79.27 loss:0.006 grdn:0.134 lr:7.7e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 10:51:52 ts/train.py:232 step:288K smpl:5M ep:24K epch:79.33 loss:0.005 grdn:0.134 lr:7.7e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-06 10:52:48 ts/train.py:232 step:288K smpl:5M ep:24K epch:79.38 loss:0.005 grdn:0.130 lr:7.7e-05 updt_s:0.160 data_s:0.118 +INFO 2025-05-06 10:53:42 ts/train.py:232 step:288K smpl:5M ep:24K epch:79.44 loss:0.005 grdn:0.133 lr:7.7e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 10:54:36 ts/train.py:232 step:289K smpl:5M ep:24K epch:79.49 loss:0.006 grdn:0.131 lr:7.7e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 10:55:31 ts/train.py:232 step:289K smpl:5M ep:24K epch:79.55 loss:0.006 grdn:0.135 lr:7.7e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-06 10:56:26 ts/train.py:232 step:289K smpl:5M ep:24K epch:79.60 loss:0.006 grdn:0.138 lr:7.7e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-06 10:57:21 ts/train.py:232 step:289K smpl:5M ep:24K epch:79.66 loss:0.006 grdn:0.133 lr:7.7e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 10:58:16 ts/train.py:232 step:289K smpl:5M ep:24K epch:79.71 loss:0.005 grdn:0.130 lr:7.7e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 10:59:12 ts/train.py:232 step:290K smpl:5M ep:24K epch:79.77 loss:0.006 grdn:0.135 lr:7.7e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-06 11:00:07 ts/train.py:232 step:290K smpl:5M ep:24K epch:79.82 loss:0.006 grdn:0.142 lr:7.7e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-06 11:01:03 ts/train.py:232 step:290K smpl:5M ep:24K epch:79.88 loss:0.005 grdn:0.126 lr:7.7e-05 updt_s:0.161 data_s:0.117 +INFO 2025-05-06 11:01:58 ts/train.py:232 step:290K smpl:5M ep:24K epch:79.93 loss:0.006 grdn:0.134 lr:7.7e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 11:02:52 ts/train.py:232 step:290K smpl:5M ep:24K epch:79.99 loss:0.005 grdn:0.133 lr:7.7e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 11:03:45 ts/train.py:232 step:291K smpl:5M ep:24K epch:80.04 loss:0.006 grdn:0.136 lr:7.6e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 11:04:34 ts/train.py:232 step:291K smpl:5M ep:24K epch:80.10 loss:0.006 grdn:0.134 lr:7.6e-05 updt_s:0.161 data_s:0.080 +INFO 2025-05-06 11:05:21 ts/train.py:232 step:291K smpl:5M ep:24K epch:80.15 loss:0.005 grdn:0.129 lr:7.6e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 11:06:09 ts/train.py:232 step:291K smpl:5M ep:24K epch:80.21 loss:0.005 grdn:0.125 lr:7.6e-05 updt_s:0.161 data_s:0.075 +INFO 2025-05-06 11:06:58 ts/train.py:232 step:291K smpl:5M ep:24K epch:80.26 loss:0.005 grdn:0.135 lr:7.6e-05 updt_s:0.161 data_s:0.086 +INFO 2025-05-06 11:07:47 ts/train.py:232 step:292K smpl:5M ep:24K epch:80.32 loss:0.006 grdn:0.138 lr:7.6e-05 updt_s:0.161 data_s:0.082 +INFO 2025-05-06 11:08:35 ts/train.py:232 step:292K smpl:5M ep:24K epch:80.37 loss:0.006 grdn:0.141 lr:7.6e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 11:09:24 ts/train.py:232 step:292K smpl:5M ep:24K epch:80.43 loss:0.006 grdn:0.142 lr:7.6e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 11:10:12 ts/train.py:232 step:292K smpl:5M ep:24K epch:80.48 loss:0.005 grdn:0.129 lr:7.6e-05 updt_s:0.160 data_s:0.078 +INFO 2025-05-06 11:10:59 ts/train.py:232 step:292K smpl:5M ep:24K epch:80.54 loss:0.006 grdn:0.139 lr:7.6e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 11:11:47 ts/train.py:232 step:293K smpl:5M ep:24K epch:80.59 loss:0.005 grdn:0.133 lr:7.6e-05 updt_s:0.160 data_s:0.076 +INFO 2025-05-06 11:12:35 ts/train.py:232 step:293K smpl:5M ep:24K epch:80.65 loss:0.005 grdn:0.133 lr:7.6e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 11:13:22 ts/train.py:232 step:293K smpl:5M ep:24K epch:80.70 loss:0.005 grdn:0.130 lr:7.6e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 11:14:10 ts/train.py:232 step:293K smpl:5M ep:24K epch:80.76 loss:0.005 grdn:0.127 lr:7.6e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 11:14:57 ts/train.py:232 step:293K smpl:5M ep:24K epch:80.81 loss:0.006 grdn:0.135 lr:7.6e-05 updt_s:0.162 data_s:0.075 +INFO 2025-05-06 11:15:45 ts/train.py:232 step:294K smpl:5M ep:24K epch:80.87 loss:0.006 grdn:0.138 lr:7.6e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 11:16:33 ts/train.py:232 step:294K smpl:5M ep:24K epch:80.92 loss:0.005 grdn:0.129 lr:7.6e-05 updt_s:0.160 data_s:0.077 +INFO 2025-05-06 11:17:20 ts/train.py:232 step:294K smpl:5M ep:24K epch:80.98 loss:0.006 grdn:0.140 lr:7.6e-05 updt_s:0.160 data_s:0.077 +INFO 2025-05-06 11:18:18 ts/train.py:232 step:294K smpl:5M ep:24K epch:81.03 loss:0.005 grdn:0.130 lr:7.6e-05 updt_s:0.160 data_s:0.126 +INFO 2025-05-06 11:19:13 ts/train.py:232 step:294K smpl:5M ep:24K epch:81.09 loss:0.005 grdn:0.129 lr:7.6e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 11:20:09 ts/train.py:232 step:295K smpl:5M ep:24K epch:81.14 loss:0.005 grdn:0.132 lr:7.6e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-06 11:21:04 ts/train.py:232 step:295K smpl:5M ep:24K epch:81.20 loss:0.005 grdn:0.134 lr:7.6e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 11:21:59 ts/train.py:232 step:295K smpl:5M ep:24K epch:81.25 loss:0.006 grdn:0.137 lr:7.6e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-06 11:22:54 ts/train.py:232 step:295K smpl:5M ep:24K epch:81.31 loss:0.005 grdn:0.127 lr:7.6e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-06 11:23:49 ts/train.py:232 step:295K smpl:5M ep:24K epch:81.36 loss:0.005 grdn:0.128 lr:7.6e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 11:24:42 ts/train.py:232 step:296K smpl:5M ep:24K epch:81.42 loss:0.006 grdn:0.132 lr:7.6e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 11:25:35 ts/train.py:232 step:296K smpl:5M ep:24K epch:81.47 loss:0.005 grdn:0.128 lr:7.6e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 11:26:28 ts/train.py:232 step:296K smpl:5M ep:24K epch:81.53 loss:0.006 grdn:0.144 lr:7.6e-05 updt_s:0.162 data_s:0.103 +INFO 2025-05-06 11:27:24 ts/train.py:232 step:296K smpl:5M ep:24K epch:81.58 loss:0.005 grdn:0.131 lr:7.6e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 11:28:20 ts/train.py:232 step:296K smpl:5M ep:24K epch:81.64 loss:0.005 grdn:0.129 lr:7.6e-05 updt_s:0.161 data_s:0.118 +INFO 2025-05-06 11:29:16 ts/train.py:232 step:297K smpl:5M ep:25K epch:81.69 loss:0.005 grdn:0.137 lr:7.6e-05 updt_s:0.160 data_s:0.120 +INFO 2025-05-06 11:30:12 ts/train.py:232 step:297K smpl:5M ep:25K epch:81.75 loss:0.006 grdn:0.136 lr:7.6e-05 updt_s:0.160 data_s:0.118 +INFO 2025-05-06 11:31:08 ts/train.py:232 step:297K smpl:5M ep:25K epch:81.80 loss:0.006 grdn:0.134 lr:7.6e-05 updt_s:0.160 data_s:0.121 +INFO 2025-05-06 11:32:04 ts/train.py:232 step:297K smpl:5M ep:25K epch:81.86 loss:0.006 grdn:0.130 lr:7.5e-05 updt_s:0.160 data_s:0.118 +INFO 2025-05-06 11:32:59 ts/train.py:232 step:297K smpl:5M ep:25K epch:81.91 loss:0.005 grdn:0.128 lr:7.5e-05 updt_s:0.159 data_s:0.115 +INFO 2025-05-06 11:33:56 ts/train.py:232 step:298K smpl:5M ep:25K epch:81.97 loss:0.005 grdn:0.130 lr:7.5e-05 updt_s:0.159 data_s:0.122 +INFO 2025-05-06 11:34:50 ts/train.py:232 step:298K smpl:5M ep:25K epch:82.02 loss:0.006 grdn:0.137 lr:7.5e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 11:35:45 ts/train.py:232 step:298K smpl:5M ep:25K epch:82.08 loss:0.005 grdn:0.130 lr:7.5e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 11:36:40 ts/train.py:232 step:298K smpl:5M ep:25K epch:82.13 loss:0.005 grdn:0.134 lr:7.5e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-06 11:37:35 ts/train.py:232 step:298K smpl:5M ep:25K epch:82.19 loss:0.006 grdn:0.137 lr:7.5e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 11:38:30 ts/train.py:232 step:299K smpl:5M ep:25K epch:82.24 loss:0.005 grdn:0.133 lr:7.5e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-06 11:39:25 ts/train.py:232 step:299K smpl:5M ep:25K epch:82.30 loss:0.005 grdn:0.137 lr:7.5e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-06 11:40:19 ts/train.py:232 step:299K smpl:5M ep:25K epch:82.35 loss:0.005 grdn:0.131 lr:7.5e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 11:41:14 ts/train.py:232 step:299K smpl:5M ep:25K epch:82.41 loss:0.006 grdn:0.139 lr:7.5e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 11:42:08 ts/train.py:232 step:299K smpl:5M ep:25K epch:82.47 loss:0.005 grdn:0.125 lr:7.5e-05 updt_s:0.161 data_s:0.109 +INFO 2025-05-06 11:43:03 ts/train.py:232 step:300K smpl:5M ep:25K epch:82.52 loss:0.005 grdn:0.134 lr:7.5e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-06 11:43:58 ts/train.py:232 step:300K smpl:5M ep:25K epch:82.58 loss:0.005 grdn:0.130 lr:7.5e-05 updt_s:0.161 data_s:0.115 +INFO 2025-05-06 11:44:53 ts/train.py:232 step:300K smpl:5M ep:25K epch:82.63 loss:0.005 grdn:0.132 lr:7.5e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-06 11:44:53 ts/train.py:241 Checkpoint policy after step 300000 +INFO 2025-05-06 11:45:49 ts/train.py:232 step:300K smpl:5M ep:25K epch:82.69 loss:0.005 grdn:0.125 lr:7.5e-05 updt_s:0.161 data_s:0.106 +INFO 2025-05-06 11:46:44 ts/train.py:232 step:300K smpl:5M ep:25K epch:82.74 loss:0.005 grdn:0.134 lr:7.5e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 11:47:39 ts/train.py:232 step:301K smpl:5M ep:25K epch:82.80 loss:0.006 grdn:0.138 lr:7.5e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 11:48:34 ts/train.py:232 step:301K smpl:5M ep:25K epch:82.85 loss:0.005 grdn:0.127 lr:7.5e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 11:49:29 ts/train.py:232 step:301K smpl:5M ep:25K epch:82.91 loss:0.006 grdn:0.143 lr:7.5e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-06 11:50:22 ts/train.py:232 step:301K smpl:5M ep:25K epch:82.96 loss:0.005 grdn:0.137 lr:7.5e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 11:51:12 ts/train.py:232 step:301K smpl:5M ep:25K epch:83.02 loss:0.005 grdn:0.128 lr:7.5e-05 updt_s:0.160 data_s:0.089 +INFO 2025-05-06 11:52:03 ts/train.py:232 step:302K smpl:5M ep:25K epch:83.07 loss:0.005 grdn:0.127 lr:7.5e-05 updt_s:0.160 data_s:0.093 +INFO 2025-05-06 11:52:52 ts/train.py:232 step:302K smpl:5M ep:25K epch:83.13 loss:0.006 grdn:0.141 lr:7.5e-05 updt_s:0.160 data_s:0.087 +INFO 2025-05-06 11:53:44 ts/train.py:232 step:302K smpl:5M ep:25K epch:83.18 loss:0.005 grdn:0.128 lr:7.5e-05 updt_s:0.160 data_s:0.097 +INFO 2025-05-06 11:54:34 ts/train.py:232 step:302K smpl:5M ep:25K epch:83.24 loss:0.005 grdn:0.131 lr:7.5e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-06 11:55:25 ts/train.py:232 step:302K smpl:5M ep:25K epch:83.29 loss:0.005 grdn:0.133 lr:7.5e-05 updt_s:0.160 data_s:0.095 +INFO 2025-05-06 11:56:15 ts/train.py:232 step:303K smpl:5M ep:25K epch:83.35 loss:0.006 grdn:0.141 lr:7.5e-05 updt_s:0.160 data_s:0.089 +INFO 2025-05-06 11:57:05 ts/train.py:232 step:303K smpl:5M ep:25K epch:83.40 loss:0.006 grdn:0.135 lr:7.5e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-06 11:57:54 ts/train.py:232 step:303K smpl:5M ep:25K epch:83.46 loss:0.005 grdn:0.129 lr:7.5e-05 updt_s:0.161 data_s:0.085 +INFO 2025-05-06 11:58:45 ts/train.py:232 step:303K smpl:5M ep:25K epch:83.51 loss:0.005 grdn:0.128 lr:7.5e-05 updt_s:0.161 data_s:0.091 +INFO 2025-05-06 11:59:35 ts/train.py:232 step:303K smpl:5M ep:25K epch:83.57 loss:0.005 grdn:0.129 lr:7.5e-05 updt_s:0.161 data_s:0.092 +INFO 2025-05-06 12:00:26 ts/train.py:232 step:304K smpl:5M ep:25K epch:83.62 loss:0.006 grdn:0.133 lr:7.5e-05 updt_s:0.161 data_s:0.089 +INFO 2025-05-06 12:01:14 ts/train.py:232 step:304K smpl:5M ep:25K epch:83.68 loss:0.006 grdn:0.141 lr:7.4e-05 updt_s:0.162 data_s:0.080 +INFO 2025-05-06 12:02:03 ts/train.py:232 step:304K smpl:5M ep:25K epch:83.73 loss:0.005 grdn:0.134 lr:7.4e-05 updt_s:0.160 data_s:0.086 +INFO 2025-05-06 12:02:53 ts/train.py:232 step:304K smpl:5M ep:25K epch:83.79 loss:0.005 grdn:0.131 lr:7.4e-05 updt_s:0.161 data_s:0.088 +INFO 2025-05-06 12:03:43 ts/train.py:232 step:304K smpl:5M ep:25K epch:83.84 loss:0.005 grdn:0.131 lr:7.4e-05 updt_s:0.161 data_s:0.087 +INFO 2025-05-06 12:04:36 ts/train.py:232 step:305K smpl:5M ep:25K epch:83.90 loss:0.005 grdn:0.125 lr:7.4e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 12:05:26 ts/train.py:232 step:305K smpl:5M ep:25K epch:83.95 loss:0.005 grdn:0.131 lr:7.4e-05 updt_s:0.160 data_s:0.087 +INFO 2025-05-06 12:06:18 ts/train.py:232 step:305K smpl:5M ep:25K epch:84.01 loss:0.005 grdn:0.126 lr:7.4e-05 updt_s:0.159 data_s:0.101 +INFO 2025-05-06 12:07:11 ts/train.py:232 step:305K smpl:5M ep:25K epch:84.06 loss:0.005 grdn:0.134 lr:7.4e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-06 12:08:04 ts/train.py:232 step:305K smpl:5M ep:25K epch:84.12 loss:0.006 grdn:0.136 lr:7.4e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 12:08:55 ts/train.py:232 step:306K smpl:5M ep:25K epch:84.17 loss:0.005 grdn:0.135 lr:7.4e-05 updt_s:0.160 data_s:0.094 +INFO 2025-05-06 12:09:47 ts/train.py:232 step:306K smpl:5M ep:25K epch:84.23 loss:0.006 grdn:0.132 lr:7.4e-05 updt_s:0.160 data_s:0.099 +INFO 2025-05-06 12:10:39 ts/train.py:232 step:306K smpl:5M ep:25K epch:84.28 loss:0.006 grdn:0.139 lr:7.4e-05 updt_s:0.160 data_s:0.100 +INFO 2025-05-06 12:11:30 ts/train.py:232 step:306K smpl:5M ep:25K epch:84.34 loss:0.005 grdn:0.128 lr:7.4e-05 updt_s:0.160 data_s:0.096 +INFO 2025-05-06 12:12:23 ts/train.py:232 step:306K smpl:5M ep:25K epch:84.39 loss:0.005 grdn:0.130 lr:7.4e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 12:13:16 ts/train.py:232 step:307K smpl:5M ep:25K epch:84.45 loss:0.005 grdn:0.130 lr:7.4e-05 updt_s:0.160 data_s:0.102 +INFO 2025-05-06 12:14:08 ts/train.py:232 step:307K smpl:5M ep:25K epch:84.50 loss:0.006 grdn:0.138 lr:7.4e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 12:15:02 ts/train.py:232 step:307K smpl:5M ep:25K epch:84.56 loss:0.005 grdn:0.128 lr:7.4e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-06 12:15:55 ts/train.py:232 step:307K smpl:5M ep:25K epch:84.61 loss:0.005 grdn:0.137 lr:7.4e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 12:16:48 ts/train.py:232 step:307K smpl:5M ep:25K epch:84.67 loss:0.005 grdn:0.130 lr:7.4e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 12:17:42 ts/train.py:232 step:308K smpl:5M ep:25K epch:84.72 loss:0.005 grdn:0.125 lr:7.4e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-06 12:18:36 ts/train.py:232 step:308K smpl:5M ep:25K epch:84.78 loss:0.006 grdn:0.135 lr:7.4e-05 updt_s:0.159 data_s:0.107 +INFO 2025-05-06 12:19:28 ts/train.py:232 step:308K smpl:5M ep:25K epch:84.83 loss:0.005 grdn:0.134 lr:7.4e-05 updt_s:0.159 data_s:0.104 +INFO 2025-05-06 12:20:25 ts/train.py:232 step:308K smpl:5M ep:25K epch:84.89 loss:0.005 grdn:0.132 lr:7.4e-05 updt_s:0.159 data_s:0.124 +INFO 2025-05-06 12:21:21 ts/train.py:232 step:308K smpl:5M ep:25K epch:84.94 loss:0.005 grdn:0.132 lr:7.4e-05 updt_s:0.160 data_s:0.119 +INFO 2025-05-06 12:22:16 ts/train.py:232 step:309K smpl:5M ep:25K epch:85.00 loss:0.005 grdn:0.128 lr:7.4e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-06 12:23:12 ts/train.py:232 step:309K smpl:5M ep:26K epch:85.05 loss:0.006 grdn:0.143 lr:7.4e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-06 12:24:07 ts/train.py:232 step:309K smpl:5M ep:26K epch:85.11 loss:0.005 grdn:0.135 lr:7.4e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-06 12:25:03 ts/train.py:232 step:309K smpl:5M ep:26K epch:85.16 loss:0.006 grdn:0.140 lr:7.4e-05 updt_s:0.160 data_s:0.119 +INFO 2025-05-06 12:25:59 ts/train.py:232 step:309K smpl:5M ep:26K epch:85.22 loss:0.006 grdn:0.137 lr:7.4e-05 updt_s:0.160 data_s:0.117 +INFO 2025-05-06 12:26:55 ts/train.py:232 step:310K smpl:5M ep:26K epch:85.27 loss:0.005 grdn:0.132 lr:7.4e-05 updt_s:0.160 data_s:0.118 +INFO 2025-05-06 12:27:50 ts/train.py:232 step:310K smpl:5M ep:26K epch:85.33 loss:0.005 grdn:0.125 lr:7.4e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-06 12:28:45 ts/train.py:232 step:310K smpl:5M ep:26K epch:85.38 loss:0.005 grdn:0.122 lr:7.4e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 12:29:40 ts/train.py:232 step:310K smpl:5M ep:26K epch:85.44 loss:0.006 grdn:0.138 lr:7.4e-05 updt_s:0.160 data_s:0.113 +INFO 2025-05-06 12:30:34 ts/train.py:232 step:310K smpl:5M ep:26K epch:85.49 loss:0.006 grdn:0.137 lr:7.3e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 12:31:29 ts/train.py:232 step:311K smpl:5M ep:26K epch:85.55 loss:0.005 grdn:0.136 lr:7.3e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-06 12:32:24 ts/train.py:232 step:311K smpl:5M ep:26K epch:85.61 loss:0.005 grdn:0.133 lr:7.3e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 12:33:19 ts/train.py:232 step:311K smpl:5M ep:26K epch:85.66 loss:0.005 grdn:0.127 lr:7.3e-05 updt_s:0.159 data_s:0.114 +INFO 2025-05-06 12:34:13 ts/train.py:232 step:311K smpl:5M ep:26K epch:85.72 loss:0.005 grdn:0.137 lr:7.3e-05 updt_s:0.159 data_s:0.110 +INFO 2025-05-06 12:35:06 ts/train.py:232 step:311K smpl:5M ep:26K epch:85.77 loss:0.005 grdn:0.130 lr:7.3e-05 updt_s:0.160 data_s:0.104 +INFO 2025-05-06 12:35:58 ts/train.py:232 step:312K smpl:5M ep:26K epch:85.83 loss:0.005 grdn:0.138 lr:7.3e-05 updt_s:0.159 data_s:0.103 +INFO 2025-05-06 12:36:46 ts/train.py:232 step:312K smpl:5M ep:26K epch:85.88 loss:0.006 grdn:0.142 lr:7.3e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 12:37:34 ts/train.py:232 step:312K smpl:5M ep:26K epch:85.94 loss:0.005 grdn:0.124 lr:7.3e-05 updt_s:0.160 data_s:0.078 +INFO 2025-05-06 12:38:22 ts/train.py:232 step:312K smpl:5M ep:26K epch:85.99 loss:0.005 grdn:0.121 lr:7.3e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 12:39:08 ts/train.py:232 step:312K smpl:5M ep:26K epch:86.05 loss:0.005 grdn:0.130 lr:7.3e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 12:39:56 ts/train.py:232 step:313K smpl:5M ep:26K epch:86.10 loss:0.005 grdn:0.133 lr:7.3e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 12:40:44 ts/train.py:232 step:313K smpl:5M ep:26K epch:86.16 loss:0.006 grdn:0.146 lr:7.3e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 12:41:31 ts/train.py:232 step:313K smpl:5M ep:26K epch:86.21 loss:0.005 grdn:0.129 lr:7.3e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-06 12:42:20 ts/train.py:232 step:313K smpl:5M ep:26K epch:86.27 loss:0.005 grdn:0.140 lr:7.3e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 12:43:07 ts/train.py:232 step:313K smpl:5M ep:26K epch:86.32 loss:0.005 grdn:0.135 lr:7.3e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 12:43:56 ts/train.py:232 step:314K smpl:5M ep:26K epch:86.38 loss:0.005 grdn:0.131 lr:7.3e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 12:44:43 ts/train.py:232 step:314K smpl:5M ep:26K epch:86.43 loss:0.005 grdn:0.133 lr:7.3e-05 updt_s:0.160 data_s:0.077 +INFO 2025-05-06 12:45:30 ts/train.py:232 step:314K smpl:5M ep:26K epch:86.49 loss:0.005 grdn:0.135 lr:7.3e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-06 12:46:18 ts/train.py:232 step:314K smpl:5M ep:26K epch:86.54 loss:0.005 grdn:0.129 lr:7.3e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 12:47:05 ts/train.py:232 step:314K smpl:5M ep:26K epch:86.60 loss:0.005 grdn:0.130 lr:7.3e-05 updt_s:0.160 data_s:0.075 +INFO 2025-05-06 12:47:52 ts/train.py:232 step:315K smpl:5M ep:26K epch:86.65 loss:0.005 grdn:0.137 lr:7.3e-05 updt_s:0.160 data_s:0.075 +INFO 2025-05-06 12:48:40 ts/train.py:232 step:315K smpl:5M ep:26K epch:86.71 loss:0.005 grdn:0.134 lr:7.3e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 12:49:28 ts/train.py:232 step:315K smpl:5M ep:26K epch:86.76 loss:0.005 grdn:0.127 lr:7.3e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 12:50:23 ts/train.py:232 step:315K smpl:5M ep:26K epch:86.82 loss:0.005 grdn:0.127 lr:7.3e-05 updt_s:0.161 data_s:0.115 +INFO 2025-05-06 12:51:17 ts/train.py:232 step:315K smpl:5M ep:26K epch:86.87 loss:0.005 grdn:0.134 lr:7.3e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 12:52:11 ts/train.py:232 step:316K smpl:5M ep:26K epch:86.93 loss:0.006 grdn:0.136 lr:7.3e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 12:53:04 ts/train.py:232 step:316K smpl:5M ep:26K epch:86.98 loss:0.005 grdn:0.132 lr:7.3e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 12:53:58 ts/train.py:232 step:316K smpl:5M ep:26K epch:87.04 loss:0.005 grdn:0.123 lr:7.3e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 12:54:52 ts/train.py:232 step:316K smpl:5M ep:26K epch:87.09 loss:0.005 grdn:0.126 lr:7.3e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 12:55:47 ts/train.py:232 step:316K smpl:5M ep:26K epch:87.15 loss:0.004 grdn:0.123 lr:7.3e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 12:56:41 ts/train.py:232 step:317K smpl:5M ep:26K epch:87.20 loss:0.005 grdn:0.132 lr:7.3e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 12:57:34 ts/train.py:232 step:317K smpl:5M ep:26K epch:87.26 loss:0.005 grdn:0.133 lr:7.2e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 12:58:28 ts/train.py:232 step:317K smpl:5M ep:26K epch:87.31 loss:0.005 grdn:0.130 lr:7.2e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 12:59:20 ts/train.py:232 step:317K smpl:5M ep:26K epch:87.37 loss:0.005 grdn:0.133 lr:7.2e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 13:00:14 ts/train.py:232 step:317K smpl:5M ep:26K epch:87.42 loss:0.005 grdn:0.130 lr:7.2e-05 updt_s:0.160 data_s:0.105 +INFO 2025-05-06 13:01:06 ts/train.py:232 step:318K smpl:5M ep:26K epch:87.48 loss:0.005 grdn:0.132 lr:7.2e-05 updt_s:0.161 data_s:0.102 +INFO 2025-05-06 13:02:00 ts/train.py:232 step:318K smpl:5M ep:26K epch:87.53 loss:0.006 grdn:0.136 lr:7.2e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 13:02:54 ts/train.py:232 step:318K smpl:5M ep:26K epch:87.59 loss:0.005 grdn:0.126 lr:7.2e-05 updt_s:0.161 data_s:0.108 +INFO 2025-05-06 13:03:48 ts/train.py:232 step:318K smpl:5M ep:26K epch:87.64 loss:0.005 grdn:0.127 lr:7.2e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 13:04:42 ts/train.py:232 step:318K smpl:5M ep:26K epch:87.70 loss:0.005 grdn:0.132 lr:7.2e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 13:05:38 ts/train.py:232 step:319K smpl:5M ep:26K epch:87.75 loss:0.005 grdn:0.127 lr:7.2e-05 updt_s:0.160 data_s:0.119 +INFO 2025-05-06 13:06:35 ts/train.py:232 step:319K smpl:5M ep:26K epch:87.81 loss:0.005 grdn:0.132 lr:7.2e-05 updt_s:0.161 data_s:0.120 +INFO 2025-05-06 13:07:31 ts/train.py:232 step:319K smpl:5M ep:26K epch:87.86 loss:0.005 grdn:0.135 lr:7.2e-05 updt_s:0.160 data_s:0.119 +INFO 2025-05-06 13:08:27 ts/train.py:232 step:319K smpl:5M ep:26K epch:87.92 loss:0.005 grdn:0.136 lr:7.2e-05 updt_s:0.161 data_s:0.119 +INFO 2025-05-06 13:09:22 ts/train.py:232 step:319K smpl:5M ep:26K epch:87.97 loss:0.005 grdn:0.129 lr:7.2e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 13:10:18 ts/train.py:232 step:320K smpl:5M ep:26K epch:88.03 loss:0.005 grdn:0.136 lr:7.2e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 13:11:13 ts/train.py:232 step:320K smpl:5M ep:26K epch:88.08 loss:0.006 grdn:0.138 lr:7.2e-05 updt_s:0.160 data_s:0.114 +INFO 2025-05-06 13:12:07 ts/train.py:232 step:320K smpl:5M ep:26K epch:88.14 loss:0.005 grdn:0.131 lr:7.2e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 13:13:02 ts/train.py:232 step:320K smpl:5M ep:26K epch:88.19 loss:0.005 grdn:0.124 lr:7.2e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 13:13:57 ts/train.py:232 step:320K smpl:5M ep:26K epch:88.25 loss:0.005 grdn:0.134 lr:7.2e-05 updt_s:0.160 data_s:0.115 +INFO 2025-05-06 13:14:53 ts/train.py:232 step:321K smpl:5M ep:26K epch:88.30 loss:0.005 grdn:0.127 lr:7.2e-05 updt_s:0.161 data_s:0.116 +INFO 2025-05-06 13:15:47 ts/train.py:232 step:321K smpl:5M ep:27K epch:88.36 loss:0.005 grdn:0.124 lr:7.2e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 13:16:42 ts/train.py:232 step:321K smpl:5M ep:27K epch:88.41 loss:0.005 grdn:0.132 lr:7.2e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 13:17:36 ts/train.py:232 step:321K smpl:5M ep:27K epch:88.47 loss:0.005 grdn:0.131 lr:7.2e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 13:18:30 ts/train.py:232 step:321K smpl:5M ep:27K epch:88.52 loss:0.005 grdn:0.138 lr:7.2e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 13:19:25 ts/train.py:232 step:322K smpl:5M ep:27K epch:88.58 loss:0.005 grdn:0.125 lr:7.2e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 13:20:19 ts/train.py:232 step:322K smpl:5M ep:27K epch:88.63 loss:0.005 grdn:0.126 lr:7.2e-05 updt_s:0.161 data_s:0.110 +INFO 2025-05-06 13:21:15 ts/train.py:232 step:322K smpl:5M ep:27K epch:88.69 loss:0.006 grdn:0.133 lr:7.2e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-06 13:22:11 ts/train.py:232 step:322K smpl:5M ep:27K epch:88.75 loss:0.005 grdn:0.126 lr:7.2e-05 updt_s:0.160 data_s:0.120 +INFO 2025-05-06 13:23:05 ts/train.py:232 step:322K smpl:5M ep:27K epch:88.80 loss:0.005 grdn:0.136 lr:7.2e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 13:24:00 ts/train.py:232 step:323K smpl:5M ep:27K epch:88.86 loss:0.005 grdn:0.130 lr:7.2e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 13:24:54 ts/train.py:232 step:323K smpl:5M ep:27K epch:88.91 loss:0.005 grdn:0.132 lr:7.2e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 13:25:48 ts/train.py:232 step:323K smpl:5M ep:27K epch:88.97 loss:0.005 grdn:0.127 lr:7.2e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 13:26:43 ts/train.py:232 step:323K smpl:5M ep:27K epch:89.02 loss:0.005 grdn:0.131 lr:7.1e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 13:27:35 ts/train.py:232 step:323K smpl:5M ep:27K epch:89.08 loss:0.005 grdn:0.133 lr:7.1e-05 updt_s:0.161 data_s:0.103 +INFO 2025-05-06 13:28:29 ts/train.py:232 step:324K smpl:5M ep:27K epch:89.13 loss:0.005 grdn:0.127 lr:7.1e-05 updt_s:0.161 data_s:0.105 +INFO 2025-05-06 13:29:23 ts/train.py:232 step:324K smpl:5M ep:27K epch:89.19 loss:0.005 grdn:0.134 lr:7.1e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 13:30:17 ts/train.py:232 step:324K smpl:5M ep:27K epch:89.24 loss:0.005 grdn:0.125 lr:7.1e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 13:31:11 ts/train.py:232 step:324K smpl:5M ep:27K epch:89.30 loss:0.005 grdn:0.138 lr:7.1e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 13:32:05 ts/train.py:232 step:324K smpl:5M ep:27K epch:89.35 loss:0.005 grdn:0.134 lr:7.1e-05 updt_s:0.162 data_s:0.108 +INFO 2025-05-06 13:32:59 ts/train.py:232 step:325K smpl:5M ep:27K epch:89.41 loss:0.006 grdn:0.138 lr:7.1e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 13:33:53 ts/train.py:232 step:325K smpl:5M ep:27K epch:89.46 loss:0.005 grdn:0.129 lr:7.1e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 13:34:48 ts/train.py:232 step:325K smpl:5M ep:27K epch:89.52 loss:0.005 grdn:0.126 lr:7.1e-05 updt_s:0.161 data_s:0.111 +INFO 2025-05-06 13:35:43 ts/train.py:232 step:325K smpl:5M ep:27K epch:89.57 loss:0.005 grdn:0.136 lr:7.1e-05 updt_s:0.160 data_s:0.112 +INFO 2025-05-06 13:36:36 ts/train.py:232 step:325K smpl:5M ep:27K epch:89.63 loss:0.005 grdn:0.125 lr:7.1e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 13:37:29 ts/train.py:232 step:326K smpl:5M ep:27K epch:89.68 loss:0.005 grdn:0.140 lr:7.1e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 13:38:17 ts/train.py:232 step:326K smpl:5M ep:27K epch:89.74 loss:0.004 grdn:0.119 lr:7.1e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 13:39:07 ts/train.py:232 step:326K smpl:5M ep:27K epch:89.79 loss:0.005 grdn:0.132 lr:7.1e-05 updt_s:0.160 data_s:0.090 +INFO 2025-05-06 13:39:56 ts/train.py:232 step:326K smpl:5M ep:27K epch:89.85 loss:0.005 grdn:0.136 lr:7.1e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 13:40:44 ts/train.py:232 step:326K smpl:5M ep:27K epch:89.90 loss:0.005 grdn:0.131 lr:7.1e-05 updt_s:0.161 data_s:0.081 +INFO 2025-05-06 13:41:34 ts/train.py:232 step:327K smpl:5M ep:27K epch:89.96 loss:0.005 grdn:0.136 lr:7.1e-05 updt_s:0.161 data_s:0.087 +INFO 2025-05-06 13:42:26 ts/train.py:232 step:327K smpl:5M ep:27K epch:90.01 loss:0.005 grdn:0.130 lr:7.1e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 13:43:14 ts/train.py:232 step:327K smpl:5M ep:27K epch:90.07 loss:0.006 grdn:0.140 lr:7.1e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 13:44:04 ts/train.py:232 step:327K smpl:5M ep:27K epch:90.12 loss:0.005 grdn:0.130 lr:7.1e-05 updt_s:0.162 data_s:0.086 +INFO 2025-05-06 13:44:53 ts/train.py:232 step:327K smpl:5M ep:27K epch:90.18 loss:0.005 grdn:0.124 lr:7.1e-05 updt_s:0.162 data_s:0.083 +INFO 2025-05-06 13:45:40 ts/train.py:232 step:328K smpl:5M ep:27K epch:90.23 loss:0.005 grdn:0.131 lr:7.1e-05 updt_s:0.163 data_s:0.073 +INFO 2025-05-06 13:46:28 ts/train.py:232 step:328K smpl:5M ep:27K epch:90.29 loss:0.005 grdn:0.132 lr:7.1e-05 updt_s:0.162 data_s:0.077 +INFO 2025-05-06 13:47:17 ts/train.py:232 step:328K smpl:5M ep:27K epch:90.34 loss:0.005 grdn:0.126 lr:7.1e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-06 13:48:05 ts/train.py:232 step:328K smpl:5M ep:27K epch:90.40 loss:0.005 grdn:0.130 lr:7.1e-05 updt_s:0.162 data_s:0.078 +INFO 2025-05-06 13:48:54 ts/train.py:232 step:328K smpl:5M ep:27K epch:90.45 loss:0.005 grdn:0.136 lr:7.1e-05 updt_s:0.162 data_s:0.082 +INFO 2025-05-06 13:49:43 ts/train.py:232 step:329K smpl:5M ep:27K epch:90.51 loss:0.005 grdn:0.133 lr:7.1e-05 updt_s:0.162 data_s:0.079 +INFO 2025-05-06 13:50:31 ts/train.py:232 step:329K smpl:5M ep:27K epch:90.56 loss:0.005 grdn:0.131 lr:7.1e-05 updt_s:0.162 data_s:0.080 +INFO 2025-05-06 13:51:20 ts/train.py:232 step:329K smpl:5M ep:27K epch:90.62 loss:0.005 grdn:0.138 lr:7.1e-05 updt_s:0.163 data_s:0.080 +INFO 2025-05-06 13:52:17 ts/train.py:232 step:329K smpl:5M ep:27K epch:90.67 loss:0.005 grdn:0.131 lr:7.1e-05 updt_s:0.161 data_s:0.122 +INFO 2025-05-06 13:53:11 ts/train.py:232 step:329K smpl:5M ep:27K epch:90.73 loss:0.005 grdn:0.136 lr:7.0e-05 updt_s:0.162 data_s:0.106 +INFO 2025-05-06 13:54:06 ts/train.py:232 step:330K smpl:5M ep:27K epch:90.78 loss:0.005 grdn:0.132 lr:7.0e-05 updt_s:0.161 data_s:0.114 +INFO 2025-05-06 13:55:01 ts/train.py:232 step:330K smpl:5M ep:27K epch:90.84 loss:0.006 grdn:0.142 lr:7.0e-05 updt_s:0.161 data_s:0.112 +INFO 2025-05-06 13:55:56 ts/train.py:232 step:330K smpl:5M ep:27K epch:90.89 loss:0.005 grdn:0.128 lr:7.0e-05 updt_s:0.161 data_s:0.113 +INFO 2025-05-06 13:55:56 ts/train.py:241 Checkpoint policy after step 330000 +INFO 2025-05-06 13:56:51 ts/train.py:232 step:330K smpl:5M ep:27K epch:90.95 loss:0.005 grdn:0.134 lr:7.0e-05 updt_s:0.160 data_s:0.101 +INFO 2025-05-06 13:57:46 ts/train.py:232 step:330K smpl:5M ep:27K epch:91.00 loss:0.005 grdn:0.130 lr:7.0e-05 updt_s:0.159 data_s:0.114 +INFO 2025-05-06 13:58:40 ts/train.py:232 step:331K smpl:5M ep:27K epch:91.06 loss:0.005 grdn:0.131 lr:7.0e-05 updt_s:0.159 data_s:0.112 +INFO 2025-05-06 13:59:35 ts/train.py:232 step:331K smpl:5M ep:27K epch:91.11 loss:0.005 grdn:0.132 lr:7.0e-05 updt_s:0.159 data_s:0.113 +INFO 2025-05-06 14:00:29 ts/train.py:232 step:331K smpl:5M ep:27K epch:91.17 loss:0.005 grdn:0.139 lr:7.0e-05 updt_s:0.159 data_s:0.111 +INFO 2025-05-06 14:01:22 ts/train.py:232 step:331K smpl:5M ep:27K epch:91.22 loss:0.006 grdn:0.141 lr:7.0e-05 updt_s:0.159 data_s:0.105 +INFO 2025-05-06 14:02:15 ts/train.py:232 step:331K smpl:5M ep:27K epch:91.28 loss:0.005 grdn:0.133 lr:7.0e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-06 14:03:08 ts/train.py:232 step:332K smpl:5M ep:27K epch:91.33 loss:0.005 grdn:0.128 lr:7.0e-05 updt_s:0.159 data_s:0.107 +INFO 2025-05-06 14:04:02 ts/train.py:232 step:332K smpl:5M ep:27K epch:91.39 loss:0.005 grdn:0.133 lr:7.0e-05 updt_s:0.159 data_s:0.110 +INFO 2025-05-06 14:04:56 ts/train.py:232 step:332K smpl:5M ep:27K epch:91.44 loss:0.006 grdn:0.136 lr:7.0e-05 updt_s:0.160 data_s:0.106 +INFO 2025-05-06 14:05:50 ts/train.py:232 step:332K smpl:5M ep:27K epch:91.50 loss:0.005 grdn:0.132 lr:7.0e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 14:06:43 ts/train.py:232 step:332K smpl:5M ep:27K epch:91.55 loss:0.005 grdn:0.121 lr:7.0e-05 updt_s:0.159 data_s:0.107 +INFO 2025-05-06 14:07:38 ts/train.py:232 step:333K smpl:5M ep:27K epch:91.61 loss:0.005 grdn:0.133 lr:7.0e-05 updt_s:0.159 data_s:0.115 +INFO 2025-05-06 14:08:31 ts/train.py:232 step:333K smpl:5M ep:27K epch:91.66 loss:0.005 grdn:0.131 lr:7.0e-05 updt_s:0.159 data_s:0.108 +INFO 2025-05-06 14:09:22 ts/train.py:232 step:333K smpl:5M ep:28K epch:91.72 loss:0.005 grdn:0.130 lr:7.0e-05 updt_s:0.160 data_s:0.093 +INFO 2025-05-06 14:10:13 ts/train.py:232 step:333K smpl:5M ep:28K epch:91.77 loss:0.005 grdn:0.128 lr:7.0e-05 updt_s:0.160 data_s:0.096 +INFO 2025-05-06 14:11:07 ts/train.py:232 step:333K smpl:5M ep:28K epch:91.83 loss:0.005 grdn:0.131 lr:7.0e-05 updt_s:0.159 data_s:0.107 +INFO 2025-05-06 14:12:01 ts/train.py:232 step:334K smpl:5M ep:28K epch:91.89 loss:0.005 grdn:0.131 lr:7.0e-05 updt_s:0.159 data_s:0.110 +INFO 2025-05-06 14:12:54 ts/train.py:232 step:334K smpl:5M ep:28K epch:91.94 loss:0.005 grdn:0.140 lr:7.0e-05 updt_s:0.160 data_s:0.103 +INFO 2025-05-06 14:13:47 ts/train.py:232 step:334K smpl:5M ep:28K epch:92.00 loss:0.006 grdn:0.145 lr:7.0e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 14:14:41 ts/train.py:232 step:334K smpl:5M ep:28K epch:92.05 loss:0.005 grdn:0.126 lr:7.0e-05 updt_s:0.160 data_s:0.109 +INFO 2025-05-06 14:15:36 ts/train.py:232 step:334K smpl:5M ep:28K epch:92.11 loss:0.005 grdn:0.127 lr:7.0e-05 updt_s:0.159 data_s:0.113 +INFO 2025-05-06 14:16:30 ts/train.py:232 step:335K smpl:5M ep:28K epch:92.16 loss:0.005 grdn:0.136 lr:7.0e-05 updt_s:0.160 data_s:0.111 +INFO 2025-05-06 14:17:26 ts/train.py:232 step:335K smpl:5M ep:28K epch:92.22 loss:0.005 grdn:0.124 lr:7.0e-05 updt_s:0.159 data_s:0.116 +INFO 2025-05-06 14:18:19 ts/train.py:232 step:335K smpl:5M ep:28K epch:92.27 loss:0.005 grdn:0.133 lr:7.0e-05 updt_s:0.159 data_s:0.106 +INFO 2025-05-06 14:19:12 ts/train.py:232 step:335K smpl:5M ep:28K epch:92.33 loss:0.005 grdn:0.124 lr:7.0e-05 updt_s:0.160 data_s:0.107 +INFO 2025-05-06 14:20:06 ts/train.py:232 step:335K smpl:5M ep:28K epch:92.38 loss:0.005 grdn:0.136 lr:7.0e-05 updt_s:0.159 data_s:0.111 +INFO 2025-05-06 14:21:02 ts/train.py:232 step:336K smpl:5M ep:28K epch:92.44 loss:0.006 grdn:0.146 lr:7.0e-05 updt_s:0.160 data_s:0.116 +INFO 2025-05-06 14:21:56 ts/train.py:232 step:336K smpl:5M ep:28K epch:92.49 loss:0.005 grdn:0.139 lr:6.9e-05 updt_s:0.160 data_s:0.110 +INFO 2025-05-06 14:22:50 ts/train.py:232 step:336K smpl:5M ep:28K epch:92.55 loss:0.005 grdn:0.135 lr:6.9e-05 updt_s:0.159 data_s:0.112 +INFO 2025-05-06 14:23:42 ts/train.py:232 step:336K smpl:5M ep:28K epch:92.60 loss:0.005 grdn:0.132 lr:6.9e-05 updt_s:0.160 data_s:0.100 +INFO 2025-05-06 14:24:33 ts/train.py:232 step:336K smpl:5M ep:28K epch:92.66 loss:0.005 grdn:0.129 lr:6.9e-05 updt_s:0.160 data_s:0.094 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/requirements.txt b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8fc771efb7757d31b751b471861a827f2ce353fc --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/requirements.txt @@ -0,0 +1,146 @@ +sympy==1.13.1 +torch==2.6.0 +nvidia-cuda-cupti-cu12==12.4.127 +gitdb==4.0.12 +nvidia-cusolver-cu12==11.6.1.9 +packaging==24.2 +numba==0.61.2 +safetensors==0.5.3 +nvidia-cusparselt-cu12==0.6.2 +pynput==1.8.1 +fsspec==2024.12.0 +filelock==3.18.0 +pydantic==2.11.3 +cloudpickle==3.1.1 +pillow==11.2.1 +lerobot==0.1.0 +inquirerpy==0.3.4 +charset-normalizer==3.4.1 +PySocks==1.7.1 +pycparser==2.22 +nvidia-cublas-cu12==12.4.5.8 +propcache==0.3.1 +GitPython==3.1.44 +llvmlite==0.44.0 +protobuf==5.29.4 +async-timeout==5.0.1 +draccus==0.10.0 +soupsieve==2.6 +Werkzeug==3.1.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +av==14.3.0 +huggingface-hub==0.30.2 +jsonlines==4.0.0 +setproctitle==1.3.5 +imageio==2.37.0 +aiohttp==3.11.16 +einops==0.8.1 +imageio-ffmpeg==0.6.0 +pytz==2025.2 +pymunk==6.11.1 +typing-inspect==0.9.0 +PyYAML==6.0.2 +cffi==1.17.1 +feetech-servo-sdk==1.0.0 +pyserial==3.5 +mypy-extensions==1.0.0 +MarkupSafe==3.0.2 +zipp==3.21.0 +typing_extensions==4.13.2 +setuptools==78.1.0 +python-xlib==0.33 +tzdata==2025.2 +requests==2.32.3 +pip==25.0.1 +mpmath==1.3.0 +annotated-types==0.7.0 +nvidia-cuda-runtime-cu12==12.4.127 +prompt_toolkit==3.0.51 +evdev==1.9.1 +dill==0.3.8 +nvidia-cufft-cu12==11.2.1.3 +python-dateutil==2.9.0.post0 +nvidia-nvjitlink-cu12==12.4.127 +attrs==25.3.0 +orderly-set==5.4.0 +regex==2024.11.6 +tqdm==4.67.1 +pyzmq==26.4.0 +numpy==2.2.4 +nvidia-cudnn-cu12==9.1.0.70 +toml==0.10.2 +fasteners==0.19 +wheel==0.45.1 +xxhash==3.5.0 +platformdirs==4.3.7 +nvidia-curand-cu12==10.3.5.147 +sentry-sdk==2.26.1 +yarl==1.20.0 +Flask==3.1.0 +smmap==5.0.2 +nvidia-cusparse-cu12==12.3.1.170 +docker-pycreds==0.4.0 +importlib_metadata==8.6.1 +Farama-Notifications==0.0.4 +pyyaml-include==1.4.1 +termcolor==3.0.1 +zarr==2.18.3 +mergedeep==1.3.4 +wandb==0.19.9 +urllib3==2.4.0 +pandas==2.2.3 +beautifulsoup4==4.13.4 +six==1.17.0 +hf_transfer==0.1.9 +antlr4-python3-runtime==4.9.3 +click==8.1.8 +psutil==7.0.0 +omegaconf==2.3.0 +wcwidth==0.2.13 +certifi==2025.1.31 +typing-inspection==0.4.0 +itsdangerous==2.2.0 +diffusers==0.33.1 +opencv-python-headless==4.11.0.86 +torchvision==0.21.0 +pydantic_core==2.33.1 +networkx==3.4.2 +rerun-sdk==0.22.1 +cmake==4.0.0 +numcodecs==0.13.1 +gymnasium==0.29.1 +multiprocess==0.70.16 +pfzy==0.3.4 +idna==3.10 +asciitree==0.3.3 +triton==3.2.0 +multidict==6.4.3 +frozenlist==1.5.0 +gdown==5.2.0 +deepdiff==8.4.2 +aiohappyeyeballs==2.6.1 +blinker==1.9.0 +Jinja2==3.1.6 +aiosignal==1.3.2 +pyarrow==19.0.1 +nvidia-nccl-cu12==2.21.5 +h5py==3.13.0 +TorchCodec==0.2.1 +datasets==3.5.0 +nvidia-nvtx-cu12==12.4.127 +packaging==24.2 +importlib_metadata==8.0.0 +autocommand==2.2.2 +typing_extensions==4.12.2 +zipp==3.19.2 +platformdirs==4.2.2 +jaraco.collections==5.1.0 +jaraco.text==3.12.1 +jaraco.context==5.3.0 +wheel==0.45.1 +typeguard==4.3.0 +tomli==2.0.1 +more-itertools==10.3.0 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +inflect==7.3.1 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/wandb-metadata.json b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..67fb993059789bbd28ca363f928fcccc3f384f14 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/files/wandb-metadata.json @@ -0,0 +1,110 @@ +{ + "os": "Linux-6.8.0-58-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.17", + "startedAt": "2025-05-05T13:46:33.428335Z", + "args": [ + "--dataset.repo_id=shylee/so100_pengripA", + "--policy.type=diffusion", + "--policy.device=cuda", + "--dataset.root=/SSD/LSY/lerobot_dataset/pengripA", + "--wandb.enable=true", + "--save_freq=30000", + "--steps=900000", + "--policy.noise_scheduler_type=DDIM", + "--policy.num_inference_steps=10", + "--batch_size=16", + "--policy.down_dims=[256,512,1024]", + "--policy.vision_backbone=resnet50", + "--policy.pretrained_backbone_weights=ResNet50_Weights.IMAGENET1K_V1", + "--policy.use_group_norm=false", + "--policy.pre_resize_shape=[224,224]", + "--policy.crop_shape=[224,224]", + "--policy.crop_is_random=false", + "--policy.freeze_vision_backbone=false", + "--policy.horizon=32", + "--policy.n_action_steps=32", + "--policy.optimizer_lr=0.0001", + "--policy.use_ema=true", + "--policy.ema_lambda=0.5", + "--output_dir=/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "--job_name=DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4" + ], + "program": "/NAS/LSY/lerobot/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "git": { + "remote": "https://github.com/huggingface/lerobot.git", + "commit": "42bf1e8b9df3d38b3898b24e46b5e0386910c466" + }, + "root": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4", + "host": "8b1bacebd36d", + "executable": "/opt/conda/envs/lerobot/bin/python", + "codePathLocal": "lerobot/scripts/train.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA RTX 6000 Ada Generation", + "gpu_count": 8, + "disk": { + "/": { + "total": "3778772082688", + "used": "3371102212096" + } + }, + "memory": { + "total": "540891144192" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-core.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..61f33a90b98d30c6e1144d950565f4aee06256aa --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-core.log @@ -0,0 +1,6 @@ +{"time":"2025-05-05T13:46:33.231495093Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp8y52wo1b/port-102711.txt","pid":102711,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-05-05T13:46:33.232523102Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":102711} +{"time":"2025-05-05T13:46:33.232500882Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":39127,"Zone":""}} +{"time":"2025-05-05T13:46:33.42106851Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:53992"} +{"time":"2025-05-05T13:46:33.42881454Z","level":"INFO","msg":"handleInformInit: received","streamId":"or9d7oy0","id":"127.0.0.1:53992"} +{"time":"2025-05-05T13:46:34.472503481Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"or9d7oy0","id":"127.0.0.1:53992"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-internal.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..95577922fe177a6276ba796a7fd52ba26c00c0aa --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-internal.log @@ -0,0 +1,13 @@ +{"time":"2025-05-05T13:46:33.42896044Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-core.log"} +{"time":"2025-05-05T13:46:34.472474241Z","level":"INFO","msg":"created new stream","id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.472498521Z","level":"INFO","msg":"stream: started","id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.472695701Z","level":"INFO","msg":"writer: Do: started","stream_id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.472704801Z","level":"INFO","msg":"sender: started","stream_id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.47285267Z","level":"INFO","msg":"handler: started","stream_id":"or9d7oy0"} +{"time":"2025-05-05T13:46:34.815364301Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-05-05T18:28:10.438000676Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49720->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T09:33:25.436758862Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:41668->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T11:02:10.437470095Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:58472->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:15:55.437088156Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:45422->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:30:55.436340164Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:51617->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:42:40.435841749Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/or9d7oy0/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:46107->141.223.7.51:53: i/o timeout"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug.log b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c9380a323f5aecefbc6b4434c56996a5a5a9d062 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug.log @@ -0,0 +1,22 @@ +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Configure stats pid to 102711 +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug.log +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/logs/debug-internal.log +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:init():781] calling init triggers +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'shylee/so100_pengripA', 'root': '/SSD/LSY/lerobot_dataset/pengripA', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'horizon': 32, 'n_action_steps': 32, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ema': True, 'ema_lambda': 0.5}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4', 'job_name': 'DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 900000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 30000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:init():809] starting backend +2025-05-05 13:46:33,423 INFO MainThread:102711 [wandb_init.py:init():813] sending inform_init request +2025-05-05 13:46:33,427 INFO MainThread:102711 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-05-05 13:46:33,427 INFO MainThread:102711 [wandb_init.py:init():823] backend started and connected +2025-05-05 13:46:33,429 INFO MainThread:102711 [wandb_init.py:init():915] updated telemetry +2025-05-05 13:46:33,587 INFO MainThread:102711 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-05-05 13:46:34,771 INFO MainThread:102711 [wandb_init.py:init():1014] starting run threads in backend +2025-05-05 13:46:34,900 INFO MainThread:102711 [wandb_run.py:_console_start():2454] atexit reg +2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-05-05 13:46:34,901 INFO MainThread:102711 [wandb_run.py:_redirect():2394] Redirects installed. +2025-05-05 13:46:34,902 INFO MainThread:102711 [wandb_init.py:init():1056] run started, returning control to user process diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/run-or9d7oy0.wandb b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/run-or9d7oy0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..200e9823b0e1eaa259c8a5552fbd83c6cdfdeef8 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/run-or9d7oy0.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a45d882bcdf2ea648c5e90d4990ea6a8f662753594cbc1b700d9e772982527a0 +size 20185088