diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6e5589b43ee52bd280c23605c010a87b76c015fc --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e4ed48f842701dd5b003adddc562bb9e223d22fdbd0aad55ae71d64d1905dd1 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1c33646195ba601cca271c8543025c3f774bc81a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.044397670300381e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b60a7d01929fc9d9907d2b908c8d12fb7c3e07e2 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e75c93f97e3ad3326c2f7568777c953420f97588cd8cdfbc1be623bddca50a59 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1d8497bd7cfd26d25fab7a8f79002527c5348872 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed0f1916edd21be6796437f2a790c792bd59e150a2d2227f83d081f8b36021a2 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a15f677a8636acb8dc9409520ae800be927a2679 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 390000, + "verbose": false, + "_step_count": 390001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.044397670300381e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..9b8e02f046c68b25df949961c99b3b5917065d80 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/390000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 390000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..46150e69e86309a4c6406671c66a9e98ec23c9a5 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c997ea88f4a586b826851c95b3d66a9bc9d9668ca187164cc883cf2794999473 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1d40674d60d7b8af26656472e6e042f9db106ad6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 5.527273377010439e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..855cd6e71fbf5718b09d2372f594795eaada13c9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d089a3829642a23d8b359ef01e5d195f6874dd3eb0076e55824fc6675a2ec92 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d5ee523bdd0bfb7952abcae7cc190f5169200a9b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c89e6c3fff8a41da8758f97c3402a8ee9961d6b627f37db6d3734be79862f91 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..56b95070343527bb3d50e47b311c1b9e0f69978f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 420000, + "verbose": false, + "_step_count": 420001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.527273377010439e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..bb0ce907623865596f71441ca75ab257971c1b55 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/420000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 420000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..400db26e42a455b789bb37d0c45646b029f7baa4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cd718463305a43c8fa9c957bd78a025b7969f609a8542d5ef9dd1bc929c8774 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..2f02db3040a7a6bdfad0336ac978af3b3bf8432f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 5.004365747991111e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b91bc9a81549b85e5d40cdb6007d1bbf58a94fd4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36aebb6f77647ed5f48450ffed969d9919c229026f61b17ff59a1fa681bc49a1 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e86b39f16b6421a17fb2e4c9b891aac384f498ec --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8695df75c38130eb2397f2fe4582099932f0c804c05e31514b67b0adcb032040 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ee8adcba04bf026c6be79d4168d85cdce285b910 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 450000, + "verbose": false, + "_step_count": 450001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.004365747991111e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..07e97e165f1e6245e7d2e431c0587c506b887c3e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/450000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 450000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0b0c0b78d4065a0376cd16ec50cf1298fd2a7ee0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:650f8db101b005c0c853fd6f8618098939733fc36e853109044888fcce8bce05 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..185ed4617482b40deb15b33a188fe7f97e23f4c9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 4.4814102337808293e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d33bf846f07166a66827663ab84a5a072dd56411 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6e65693ee4b0a7b9e332391c47aeee10c659b73592cd26db9fb895f53a8f6b9 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2de10c0e06c115dee1c137041f1d0a7ff666a0f1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5f56ee12c004c8ee21929e5e4c0389e987887fc82a6c1a40dc1d416d53c3388 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a0b5034bbc249bb90d3558fd2d0584df557319e4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 480000, + "verbose": false, + "_step_count": 480001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 4.4814102337808293e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..a54500e6a42a0151bd2a06726edce1d9ecf0a2e2 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/480000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 480000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a0b8dcd0d91ff71b57c443fa83c4f9a3bc48a3e4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e4b1e2dee95fc284dcf8bd8b6528c9bcf9af460845429bccb49c74509cc4978 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..088e05678fc18c310620072b05f782dd216ba833 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 3.9641428101410884e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8add993edadbc1033a71c69e9c39fdc86a5c249e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0fb40c92cdfc56da88121b993ba3f6312a08307d5eb202a288b21a90d1cc82c +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3bf6010629d7dbca2e9b6781c72d44666045c0ad --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18b1e20c06b1e1a53d4e58fa2aebcde9ef3e86003cbec12ee4a41409637e6890 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..60573d75e75dda18fcda9031c35db214eaf9bbd6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 510000, + "verbose": false, + "_step_count": 510001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 3.9641428101410884e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8c7035b86e69cd307e77133d7211848fb02474a9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/510000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 510000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..65641d250f35b55c86a8617658a8700705cf57b6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19ca431ac552429e25311a77997fb0dc8330dd2882ca252243ee9636dbff54c7 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..88b4b9ccc1b52457955627c808b873afd081d43e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 3.4582370636841024e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9c56874b98ff908cf2cce2a6c83642ea3ce3d8b2 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:872dc65b3e8648653f141fe2dcc74ded8893c09a41c71f53e2b3feca0194744a +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5b8d6524b53671babfa6cddba904b64de312deb8 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d172c79ac214b8841d2aa4f4ac2c78e1f97493833acc6ab31ee3fa20593e8058 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6ba6b261244956225f1e9c33461cb499b6610fff --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 540000, + "verbose": false, + "_step_count": 540001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 3.4582370636841024e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..3c2f6e37b4db10fefdc1e301eb1523863fc3a230 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/540000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 540000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..495512e1efa2e42b2e393fc66062c5cfb410fb69 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4437a16476a24672e0973bae9754a756178b9a2bcd8ac2e2d9a8f1bd632be79 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..d7f1d9d7949b75022e1f956d21929a89f2c30d46 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.969241961808482e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5d523a172df8bbe62a85970c0736cdee7cc377a8 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2393221b6e458ff23a0221aa31234bff8bb976d2eef8f9c60d58e54756e9b55d +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cf539efeaae1d1672b796e1360ab2f71ce56c2b0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:470564e771f64a10b5627e4dfd4341a2692d0089ab9d03f613c032a3ec73e635 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..241852cc90a19937813ad01d9b852a9b5acc0c75 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 570000, + "verbose": false, + "_step_count": 570001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.969241961808482e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..25cf656c471a27bc70b42dcccf3b85bb11e144cf --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/570000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 570000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d25ebe3e2c6a384288beabc894e6d03069f9ae85 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:072f58f17f3412bb82191867616bddef7e7326ce42efa4317723cf1433d3700f +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1e06b28115903bca73e927bd34166cbe105f9008 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.5025209895060332e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5878d16ff92720141e3e3340fadf94c7ed361ee6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf488f7c055c790473322b5ff89833dafbeeee8be19339c4a21b7d9980ec037c +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f6d70d3ecd02e5a2c514c5b32e67f7be8ab9eb4e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12550d26f7d721ebe29f7ed1bf9b094fb97389fcff03e24fd3d93f9c466a5898 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9ce41c0f5550f876c36888584be6ca09bdc67fa6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 600000, + "verbose": false, + "_step_count": 600001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.5025209895060332e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..13ce6152ad24f61b4c05746d5cd0bf93c31d7c0f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/600000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 600000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4e4d9e2d8621edda2f0b332efd2a85cd890d6444 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad9542f4edeabfff9066dfe20d25de20194fb55ed34509757abb44d93ae568ee +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..85d863715dc241fd41b24c5316e09855e12972da --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.0631933206103992e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..31d5f813424652965ed824acdd9e03e931283e2c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e7fb14fe800023308a65d3161339b930ae0e09373264c1d135f7e209913be64 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7cc3274816e6ac755ed120147c5a84d800b1c24a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5321735fc180c092b787f670d66e3efd956e7afdbd952b4d4d33411ed05c197 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a7d20ad7dd005e2ca9bb3b3b833686c191654091 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 630000, + "verbose": false, + "_step_count": 630001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.0631933206103992e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..71be4f013a155caed9df1f4d0513d9218f407ac3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/630000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 630000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cc2c5218b54cf0330065c7804ba9be7164eccdf1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f39c9a10cb3734c6ee340837304bf71169bb9fc0ca91bbba5a17d57fe70157c +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..5e685a39dc53152fbe671658b7b94efc36625ab5 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 1.6560776687438e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f8c88080d8acc2cb3bab62a0d20e129a7a0da69b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f27707068bdb02922c8ee44c02e9ed77cb10a1545a6929dff87475c8354aebfb +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d757d098dbd40b5d2fdfeefc8fecf2acbbd1729b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23f468c77a80b9c12a98f1c6b85dbdafb764c3095e293c2dc1b343a901e00cbd +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b278d58035ccdcfa8cd70128a420b21e10878aa9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 660000, + "verbose": false, + "_step_count": 660001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.6560776687438e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..1946a814a39401d13bc968ec845f3081adf9bc13 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/660000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 660000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log index eb106f8bffbb1de9a45650186cca3cf6bd2790c4..0a006fd387e91b2e6238bbb58a5e80b464887fc8 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log @@ -13,3 +13,4 @@ {"time":"2025-05-06T12:44:48.040371676Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:43741->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T16:50:48.039369614Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:38891->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T18:02:28.241678384Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream","body":"\n
\n\nPlease try again in 30 seconds.\n
\n\n"} +{"time":"2025-05-07T16:25:48.040273802Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:42598->141.223.7.51:53: i/o timeout"} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log index 76782edb640ecc6b31d76fc22e0eacd4ac2ecd30..3e712f8fcefd2033fa72eb332c1970951cfd22f4 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log @@ -1950,3 +1950,1437 @@ INFO 2025-05-07 11:29:14 ts/train.py:232 step:385K smpl:3M ep:16K epch:52.99 los INFO 2025-05-07 11:30:40 ts/train.py:232 step:385K smpl:3M ep:16K epch:53.02 loss:0.006 grdn:0.180 lr:6.1e-05 updt_s:0.423 data_s:0.008 INFO 2025-05-07 11:32:05 ts/train.py:232 step:385K smpl:3M ep:16K epch:53.05 loss:0.006 grdn:0.179 lr:6.1e-05 updt_s:0.423 data_s:0.001 INFO 2025-05-07 11:33:30 ts/train.py:232 step:385K smpl:3M ep:16K epch:53.08 loss:0.007 grdn:0.183 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:34:55 ts/train.py:232 step:386K smpl:3M ep:16K epch:53.10 loss:0.006 grdn:0.179 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:36:20 ts/train.py:232 step:386K smpl:3M ep:16K epch:53.13 loss:0.006 grdn:0.176 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:37:45 ts/train.py:232 step:386K smpl:3M ep:16K epch:53.16 loss:0.005 grdn:0.175 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:39:10 ts/train.py:232 step:386K smpl:3M ep:16K epch:53.19 loss:0.006 grdn:0.175 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:40:35 ts/train.py:232 step:386K smpl:3M ep:16K epch:53.21 loss:0.005 grdn:0.164 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:42:00 ts/train.py:232 step:387K smpl:3M ep:16K epch:53.24 loss:0.006 grdn:0.192 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:43:25 ts/train.py:232 step:387K smpl:3M ep:16K epch:53.27 loss:0.006 grdn:0.170 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:44:50 ts/train.py:232 step:387K smpl:3M ep:16K epch:53.30 loss:0.006 grdn:0.186 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:46:14 ts/train.py:232 step:387K smpl:3M ep:16K epch:53.32 loss:0.006 grdn:0.183 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:47:39 ts/train.py:232 step:387K smpl:3M ep:16K epch:53.35 loss:0.006 grdn:0.172 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:49:04 ts/train.py:232 step:388K smpl:3M ep:16K epch:53.38 loss:0.006 grdn:0.169 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:50:29 ts/train.py:232 step:388K smpl:3M ep:16K epch:53.41 loss:0.005 grdn:0.168 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:51:54 ts/train.py:232 step:388K smpl:3M ep:16K epch:53.43 loss:0.006 grdn:0.167 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:53:20 ts/train.py:232 step:388K smpl:3M ep:16K epch:53.46 loss:0.006 grdn:0.175 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:54:45 ts/train.py:232 step:388K smpl:3M ep:16K epch:53.49 loss:0.006 grdn:0.169 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:56:10 ts/train.py:232 step:389K smpl:3M ep:16K epch:53.52 loss:0.006 grdn:0.168 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:57:35 ts/train.py:232 step:389K smpl:3M ep:16K epch:53.54 loss:0.006 grdn:0.173 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:59:00 ts/train.py:232 step:389K smpl:3M ep:16K epch:53.57 loss:0.006 grdn:0.180 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:00:25 ts/train.py:232 step:389K smpl:3M ep:16K epch:53.60 loss:0.006 grdn:0.180 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:01:50 ts/train.py:232 step:389K smpl:3M ep:16K epch:53.63 loss:0.006 grdn:0.180 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:03:15 ts/train.py:232 step:390K smpl:3M ep:16K epch:53.65 loss:0.006 grdn:0.179 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:04:40 ts/train.py:232 step:390K smpl:3M ep:16K epch:53.68 loss:0.006 grdn:0.180 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:06:05 ts/train.py:232 step:390K smpl:3M ep:16K epch:53.71 loss:0.006 grdn:0.178 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:06:05 ts/train.py:241 Checkpoint policy after step 390000 +INFO 2025-05-07 12:07:33 ts/train.py:232 step:390K smpl:3M ep:16K epch:53.74 loss:0.007 grdn:0.190 lr:6.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 12:08:58 ts/train.py:232 step:390K smpl:3M ep:16K epch:53.76 loss:0.005 grdn:0.169 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:10:23 ts/train.py:232 step:391K smpl:3M ep:16K epch:53.79 loss:0.005 grdn:0.163 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:11:48 ts/train.py:232 step:391K smpl:3M ep:16K epch:53.82 loss:0.006 grdn:0.177 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:13:13 ts/train.py:232 step:391K smpl:3M ep:16K epch:53.85 loss:0.006 grdn:0.177 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:14:38 ts/train.py:232 step:391K smpl:3M ep:16K epch:53.88 loss:0.006 grdn:0.185 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:16:03 ts/train.py:232 step:391K smpl:3M ep:16K epch:53.90 loss:0.006 grdn:0.170 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:17:28 ts/train.py:232 step:392K smpl:3M ep:16K epch:53.93 loss:0.006 grdn:0.186 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:18:53 ts/train.py:232 step:392K smpl:3M ep:16K epch:53.96 loss:0.006 grdn:0.174 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:20:20 ts/train.py:232 step:392K smpl:3M ep:16K epch:53.99 loss:0.006 grdn:0.178 lr:6.0e-05 updt_s:0.423 data_s:0.010 +INFO 2025-05-07 12:21:45 ts/train.py:232 step:392K smpl:3M ep:16K epch:54.01 loss:0.006 grdn:0.181 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:23:10 ts/train.py:232 step:392K smpl:3M ep:16K epch:54.04 loss:0.007 grdn:0.203 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:24:35 ts/train.py:232 step:393K smpl:3M ep:16K epch:54.07 loss:0.006 grdn:0.186 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:26:00 ts/train.py:232 step:393K smpl:3M ep:16K epch:54.10 loss:0.006 grdn:0.167 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:27:25 ts/train.py:232 step:393K smpl:3M ep:16K epch:54.12 loss:0.005 grdn:0.170 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:28:50 ts/train.py:232 step:393K smpl:3M ep:16K epch:54.15 loss:0.006 grdn:0.187 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:30:15 ts/train.py:232 step:393K smpl:3M ep:16K epch:54.18 loss:0.006 grdn:0.183 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:31:41 ts/train.py:232 step:394K smpl:3M ep:16K epch:54.21 loss:0.006 grdn:0.172 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:33:06 ts/train.py:232 step:394K smpl:3M ep:16K epch:54.23 loss:0.006 grdn:0.171 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:34:31 ts/train.py:232 step:394K smpl:3M ep:16K epch:54.26 loss:0.005 grdn:0.166 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:35:56 ts/train.py:232 step:394K smpl:3M ep:16K epch:54.29 loss:0.006 grdn:0.170 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:37:20 ts/train.py:232 step:394K smpl:3M ep:16K epch:54.32 loss:0.006 grdn:0.183 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:38:45 ts/train.py:232 step:395K smpl:3M ep:16K epch:54.34 loss:0.006 grdn:0.174 lr:6.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 12:40:10 ts/train.py:232 step:395K smpl:3M ep:16K epch:54.37 loss:0.006 grdn:0.181 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:41:36 ts/train.py:232 step:395K smpl:3M ep:16K epch:54.40 loss:0.007 grdn:0.184 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:43:01 ts/train.py:232 step:395K smpl:3M ep:16K epch:54.43 loss:0.006 grdn:0.179 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:44:26 ts/train.py:232 step:395K smpl:3M ep:16K epch:54.45 loss:0.006 grdn:0.179 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:45:51 ts/train.py:232 step:396K smpl:3M ep:16K epch:54.48 loss:0.006 grdn:0.174 lr:6.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:47:16 ts/train.py:232 step:396K smpl:3M ep:16K epch:54.51 loss:0.006 grdn:0.188 lr:5.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:48:41 ts/train.py:232 step:396K smpl:3M ep:16K epch:54.54 loss:0.007 grdn:0.188 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 12:50:07 ts/train.py:232 step:396K smpl:3M ep:16K epch:54.56 loss:0.005 grdn:0.163 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 12:51:32 ts/train.py:232 step:396K smpl:3M ep:16K epch:54.59 loss:0.006 grdn:0.174 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 12:52:57 ts/train.py:232 step:397K smpl:3M ep:16K epch:54.62 loss:0.006 grdn:0.169 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 12:54:22 ts/train.py:232 step:397K smpl:3M ep:16K epch:54.65 loss:0.005 grdn:0.167 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 12:55:48 ts/train.py:232 step:397K smpl:3M ep:16K epch:54.67 loss:0.007 grdn:0.187 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 12:57:13 ts/train.py:232 step:397K smpl:3M ep:16K epch:54.70 loss:0.006 grdn:0.172 lr:5.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 12:58:38 ts/train.py:232 step:397K smpl:3M ep:16K epch:54.73 loss:0.006 grdn:0.173 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:00:03 ts/train.py:232 step:398K smpl:3M ep:16K epch:54.76 loss:0.006 grdn:0.170 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:01:29 ts/train.py:232 step:398K smpl:3M ep:16K epch:54.78 loss:0.006 grdn:0.172 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:02:54 ts/train.py:232 step:398K smpl:3M ep:16K epch:54.81 loss:0.005 grdn:0.170 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:04:19 ts/train.py:232 step:398K smpl:3M ep:16K epch:54.84 loss:0.006 grdn:0.181 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:05:45 ts/train.py:232 step:398K smpl:3M ep:16K epch:54.87 loss:0.006 grdn:0.182 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:07:10 ts/train.py:232 step:399K smpl:3M ep:16K epch:54.89 loss:0.006 grdn:0.176 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:08:35 ts/train.py:232 step:399K smpl:3M ep:16K epch:54.92 loss:0.006 grdn:0.181 lr:5.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:10:02 ts/train.py:232 step:399K smpl:3M ep:16K epch:54.95 loss:0.005 grdn:0.165 lr:5.9e-05 updt_s:0.424 data_s:0.009 +INFO 2025-05-07 13:11:27 ts/train.py:232 step:399K smpl:3M ep:16K epch:54.98 loss:0.006 grdn:0.184 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:12:53 ts/train.py:232 step:399K smpl:3M ep:17K epch:55.00 loss:0.006 grdn:0.188 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:14:18 ts/train.py:232 step:400K smpl:3M ep:17K epch:55.03 loss:0.006 grdn:0.177 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:15:43 ts/train.py:232 step:400K smpl:3M ep:17K epch:55.06 loss:0.005 grdn:0.171 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:17:09 ts/train.py:232 step:400K smpl:3M ep:17K epch:55.09 loss:0.005 grdn:0.168 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:18:34 ts/train.py:232 step:400K smpl:3M ep:17K epch:55.11 loss:0.005 grdn:0.175 lr:5.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:19:59 ts/train.py:232 step:400K smpl:3M ep:17K epch:55.14 loss:0.006 grdn:0.181 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:21:24 ts/train.py:232 step:401K smpl:3M ep:17K epch:55.17 loss:0.006 grdn:0.185 lr:5.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:22:50 ts/train.py:232 step:401K smpl:3M ep:17K epch:55.20 loss:0.006 grdn:0.182 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:24:15 ts/train.py:232 step:401K smpl:3M ep:17K epch:55.22 loss:0.006 grdn:0.190 lr:5.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:25:40 ts/train.py:232 step:401K smpl:3M ep:17K epch:55.25 loss:0.006 grdn:0.174 lr:5.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:27:05 ts/train.py:232 step:401K smpl:3M ep:17K epch:55.28 loss:0.006 grdn:0.187 lr:5.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:28:30 ts/train.py:232 step:402K smpl:3M ep:17K epch:55.31 loss:0.006 grdn:0.171 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:29:56 ts/train.py:232 step:402K smpl:3M ep:17K epch:55.33 loss:0.006 grdn:0.178 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:31:21 ts/train.py:232 step:402K smpl:3M ep:17K epch:55.36 loss:0.006 grdn:0.181 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:32:46 ts/train.py:232 step:402K smpl:3M ep:17K epch:55.39 loss:0.006 grdn:0.180 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:34:12 ts/train.py:232 step:402K smpl:3M ep:17K epch:55.42 loss:0.005 grdn:0.176 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:35:37 ts/train.py:232 step:403K smpl:3M ep:17K epch:55.44 loss:0.005 grdn:0.173 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:37:02 ts/train.py:232 step:403K smpl:3M ep:17K epch:55.47 loss:0.005 grdn:0.162 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:38:28 ts/train.py:232 step:403K smpl:3M ep:17K epch:55.50 loss:0.006 grdn:0.177 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:39:53 ts/train.py:232 step:403K smpl:3M ep:17K epch:55.53 loss:0.006 grdn:0.175 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:41:18 ts/train.py:232 step:403K smpl:3M ep:17K epch:55.56 loss:0.007 grdn:0.194 lr:5.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:42:43 ts/train.py:232 step:404K smpl:3M ep:17K epch:55.58 loss:0.006 grdn:0.180 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:44:09 ts/train.py:232 step:404K smpl:3M ep:17K epch:55.61 loss:0.006 grdn:0.186 lr:5.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:45:34 ts/train.py:232 step:404K smpl:3M ep:17K epch:55.64 loss:0.005 grdn:0.160 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:46:59 ts/train.py:232 step:404K smpl:3M ep:17K epch:55.67 loss:0.005 grdn:0.166 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:48:24 ts/train.py:232 step:404K smpl:3M ep:17K epch:55.69 loss:0.005 grdn:0.177 lr:5.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:49:50 ts/train.py:232 step:405K smpl:3M ep:17K epch:55.72 loss:0.005 grdn:0.173 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:51:15 ts/train.py:232 step:405K smpl:3M ep:17K epch:55.75 loss:0.006 grdn:0.183 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:52:40 ts/train.py:232 step:405K smpl:3M ep:17K epch:55.78 loss:0.005 grdn:0.175 lr:5.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:54:05 ts/train.py:232 step:405K smpl:3M ep:17K epch:55.80 loss:0.007 grdn:0.197 lr:5.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:55:30 ts/train.py:232 step:405K smpl:3M ep:17K epch:55.83 loss:0.006 grdn:0.185 lr:5.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:56:56 ts/train.py:232 step:406K smpl:3M ep:17K epch:55.86 loss:0.005 grdn:0.167 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 13:58:21 ts/train.py:232 step:406K smpl:3M ep:17K epch:55.89 loss:0.006 grdn:0.176 lr:5.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 13:59:48 ts/train.py:232 step:406K smpl:3M ep:17K epch:55.91 loss:0.006 grdn:0.189 lr:5.8e-05 updt_s:0.424 data_s:0.009 +INFO 2025-05-07 14:01:13 ts/train.py:232 step:406K smpl:3M ep:17K epch:55.94 loss:0.006 grdn:0.180 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:02:38 ts/train.py:232 step:406K smpl:3M ep:17K epch:55.97 loss:0.006 grdn:0.180 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:04:04 ts/train.py:232 step:407K smpl:3M ep:17K epch:56.00 loss:0.007 grdn:0.191 lr:5.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 14:05:29 ts/train.py:232 step:407K smpl:3M ep:17K epch:56.02 loss:0.006 grdn:0.175 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:06:55 ts/train.py:232 step:407K smpl:3M ep:17K epch:56.05 loss:0.006 grdn:0.186 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:08:20 ts/train.py:232 step:407K smpl:3M ep:17K epch:56.08 loss:0.006 grdn:0.176 lr:5.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:09:45 ts/train.py:232 step:407K smpl:3M ep:17K epch:56.11 loss:0.005 grdn:0.169 lr:5.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:11:11 ts/train.py:232 step:408K smpl:3M ep:17K epch:56.13 loss:0.005 grdn:0.167 lr:5.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:12:36 ts/train.py:232 step:408K smpl:3M ep:17K epch:56.16 loss:0.006 grdn:0.180 lr:5.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:14:01 ts/train.py:232 step:408K smpl:3M ep:17K epch:56.19 loss:0.005 grdn:0.167 lr:5.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:15:26 ts/train.py:232 step:408K smpl:3M ep:17K epch:56.22 loss:0.006 grdn:0.183 lr:5.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:16:52 ts/train.py:232 step:408K smpl:3M ep:17K epch:56.24 loss:0.006 grdn:0.177 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:18:17 ts/train.py:232 step:409K smpl:3M ep:17K epch:56.27 loss:0.006 grdn:0.179 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:19:42 ts/train.py:232 step:409K smpl:3M ep:17K epch:56.30 loss:0.006 grdn:0.182 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:21:07 ts/train.py:232 step:409K smpl:3M ep:17K epch:56.33 loss:0.006 grdn:0.184 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:22:33 ts/train.py:232 step:409K smpl:3M ep:17K epch:56.35 loss:0.006 grdn:0.188 lr:5.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:23:58 ts/train.py:232 step:409K smpl:3M ep:17K epch:56.38 loss:0.006 grdn:0.175 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:25:23 ts/train.py:232 step:410K smpl:3M ep:17K epch:56.41 loss:0.005 grdn:0.159 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:26:48 ts/train.py:232 step:410K smpl:3M ep:17K epch:56.44 loss:0.006 grdn:0.184 lr:5.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:28:13 ts/train.py:232 step:410K smpl:3M ep:17K epch:56.46 loss:0.006 grdn:0.173 lr:5.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 14:29:39 ts/train.py:232 step:410K smpl:3M ep:17K epch:56.49 loss:0.006 grdn:0.177 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:31:04 ts/train.py:232 step:410K smpl:3M ep:17K epch:56.52 loss:0.007 grdn:0.186 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:32:29 ts/train.py:232 step:411K smpl:3M ep:17K epch:56.55 loss:0.006 grdn:0.183 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:33:54 ts/train.py:232 step:411K smpl:3M ep:17K epch:56.57 loss:0.006 grdn:0.183 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:35:19 ts/train.py:232 step:411K smpl:3M ep:17K epch:56.60 loss:0.006 grdn:0.188 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:36:44 ts/train.py:232 step:411K smpl:3M ep:17K epch:56.63 loss:0.005 grdn:0.168 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:38:09 ts/train.py:232 step:411K smpl:3M ep:17K epch:56.66 loss:0.006 grdn:0.180 lr:5.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:39:34 ts/train.py:232 step:412K smpl:3M ep:17K epch:56.68 loss:0.006 grdn:0.179 lr:5.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:40:59 ts/train.py:232 step:412K smpl:3M ep:17K epch:56.71 loss:0.006 grdn:0.184 lr:5.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:42:24 ts/train.py:232 step:412K smpl:3M ep:17K epch:56.74 loss:0.005 grdn:0.168 lr:5.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:43:49 ts/train.py:232 step:412K smpl:3M ep:17K epch:56.77 loss:0.005 grdn:0.180 lr:5.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:45:14 ts/train.py:232 step:412K smpl:3M ep:17K epch:56.79 loss:0.006 grdn:0.189 lr:5.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 14:46:39 ts/train.py:232 step:413K smpl:3M ep:17K epch:56.82 loss:0.005 grdn:0.171 lr:5.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:48:04 ts/train.py:232 step:413K smpl:3M ep:17K epch:56.85 loss:0.006 grdn:0.183 lr:5.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:49:30 ts/train.py:232 step:413K smpl:3M ep:17K epch:56.88 loss:0.005 grdn:0.169 lr:5.7e-05 updt_s:0.423 data_s:0.007 +INFO 2025-05-07 14:50:55 ts/train.py:232 step:413K smpl:3M ep:17K epch:56.90 loss:0.005 grdn:0.167 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:52:20 ts/train.py:232 step:413K smpl:3M ep:17K epch:56.93 loss:0.006 grdn:0.173 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:53:45 ts/train.py:232 step:414K smpl:3M ep:17K epch:56.96 loss:0.005 grdn:0.164 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:55:09 ts/train.py:232 step:414K smpl:3M ep:17K epch:56.99 loss:0.006 grdn:0.189 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 14:56:34 ts/train.py:232 step:414K smpl:3M ep:17K epch:57.01 loss:0.005 grdn:0.170 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:57:59 ts/train.py:232 step:414K smpl:3M ep:17K epch:57.04 loss:0.006 grdn:0.185 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 14:59:24 ts/train.py:232 step:414K smpl:3M ep:17K epch:57.07 loss:0.006 grdn:0.188 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:00:49 ts/train.py:232 step:415K smpl:3M ep:17K epch:57.10 loss:0.006 grdn:0.175 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:02:13 ts/train.py:232 step:415K smpl:3M ep:17K epch:57.13 loss:0.006 grdn:0.186 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:03:38 ts/train.py:232 step:415K smpl:3M ep:17K epch:57.15 loss:0.005 grdn:0.169 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:05:03 ts/train.py:232 step:415K smpl:3M ep:17K epch:57.18 loss:0.005 grdn:0.171 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:06:28 ts/train.py:232 step:415K smpl:3M ep:17K epch:57.21 loss:0.006 grdn:0.184 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:07:53 ts/train.py:232 step:416K smpl:3M ep:17K epch:57.24 loss:0.007 grdn:0.194 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:09:18 ts/train.py:232 step:416K smpl:3M ep:17K epch:57.26 loss:0.006 grdn:0.182 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:10:42 ts/train.py:232 step:416K smpl:3M ep:17K epch:57.29 loss:0.006 grdn:0.184 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 15:12:07 ts/train.py:232 step:416K smpl:3M ep:17K epch:57.32 loss:0.006 grdn:0.183 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:13:32 ts/train.py:232 step:416K smpl:3M ep:17K epch:57.35 loss:0.006 grdn:0.183 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 15:14:57 ts/train.py:232 step:417K smpl:3M ep:17K epch:57.37 loss:0.006 grdn:0.183 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 15:16:21 ts/train.py:232 step:417K smpl:3M ep:17K epch:57.40 loss:0.006 grdn:0.185 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 15:17:46 ts/train.py:232 step:417K smpl:3M ep:17K epch:57.43 loss:0.006 grdn:0.174 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 15:19:11 ts/train.py:232 step:417K smpl:3M ep:17K epch:57.46 loss:0.006 grdn:0.179 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 15:20:36 ts/train.py:232 step:417K smpl:3M ep:17K epch:57.48 loss:0.005 grdn:0.167 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 15:22:01 ts/train.py:232 step:418K smpl:3M ep:17K epch:57.51 loss:0.006 grdn:0.172 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 15:23:25 ts/train.py:232 step:418K smpl:3M ep:17K epch:57.54 loss:0.006 grdn:0.184 lr:5.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 15:24:50 ts/train.py:232 step:418K smpl:3M ep:17K epch:57.57 loss:0.006 grdn:0.177 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:26:15 ts/train.py:232 step:418K smpl:3M ep:17K epch:57.59 loss:0.005 grdn:0.179 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:27:40 ts/train.py:232 step:418K smpl:3M ep:17K epch:57.62 loss:0.005 grdn:0.173 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:29:05 ts/train.py:232 step:419K smpl:3M ep:17K epch:57.65 loss:0.006 grdn:0.180 lr:5.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:30:30 ts/train.py:232 step:419K smpl:3M ep:17K epch:57.68 loss:0.006 grdn:0.182 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:31:54 ts/train.py:232 step:419K smpl:3M ep:17K epch:57.70 loss:0.006 grdn:0.182 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:33:19 ts/train.py:232 step:419K smpl:3M ep:17K epch:57.73 loss:0.006 grdn:0.192 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:34:44 ts/train.py:232 step:419K smpl:3M ep:17K epch:57.76 loss:0.005 grdn:0.167 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:36:09 ts/train.py:232 step:420K smpl:3M ep:17K epch:57.79 loss:0.006 grdn:0.178 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:37:34 ts/train.py:232 step:420K smpl:3M ep:17K epch:57.81 loss:0.006 grdn:0.189 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:39:00 ts/train.py:232 step:420K smpl:3M ep:17K epch:57.84 loss:0.006 grdn:0.190 lr:5.5e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-07 15:39:00 ts/train.py:241 Checkpoint policy after step 420000 +INFO 2025-05-07 15:40:29 ts/train.py:232 step:420K smpl:3M ep:17K epch:57.87 loss:0.005 grdn:0.176 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:41:54 ts/train.py:232 step:420K smpl:3M ep:17K epch:57.90 loss:0.005 grdn:0.175 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:43:19 ts/train.py:232 step:421K smpl:3M ep:17K epch:57.92 loss:0.006 grdn:0.178 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:44:44 ts/train.py:232 step:421K smpl:3M ep:17K epch:57.95 loss:0.006 grdn:0.179 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:46:09 ts/train.py:232 step:421K smpl:3M ep:17K epch:57.98 loss:0.006 grdn:0.187 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:47:33 ts/train.py:232 step:421K smpl:3M ep:17K epch:58.01 loss:0.005 grdn:0.185 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:48:58 ts/train.py:232 step:421K smpl:3M ep:17K epch:58.03 loss:0.005 grdn:0.170 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:50:23 ts/train.py:232 step:422K smpl:3M ep:17K epch:58.06 loss:0.005 grdn:0.163 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:51:48 ts/train.py:232 step:422K smpl:3M ep:17K epch:58.09 loss:0.005 grdn:0.170 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:53:13 ts/train.py:232 step:422K smpl:3M ep:17K epch:58.12 loss:0.006 grdn:0.179 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:54:38 ts/train.py:232 step:422K smpl:3M ep:17K epch:58.14 loss:0.006 grdn:0.191 lr:5.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 15:56:03 ts/train.py:232 step:422K smpl:3M ep:17K epch:58.17 loss:0.006 grdn:0.175 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:57:28 ts/train.py:232 step:423K smpl:3M ep:17K epch:58.20 loss:0.006 grdn:0.183 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 15:58:53 ts/train.py:232 step:423K smpl:3M ep:17K epch:58.23 loss:0.005 grdn:0.178 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:00:18 ts/train.py:232 step:423K smpl:3M ep:17K epch:58.25 loss:0.006 grdn:0.186 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:01:43 ts/train.py:232 step:423K smpl:3M ep:17K epch:58.28 loss:0.005 grdn:0.171 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:03:07 ts/train.py:232 step:423K smpl:3M ep:17K epch:58.31 loss:0.006 grdn:0.182 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:04:32 ts/train.py:232 step:424K smpl:3M ep:18K epch:58.34 loss:0.006 grdn:0.189 lr:5.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 16:05:57 ts/train.py:232 step:424K smpl:3M ep:18K epch:58.36 loss:0.006 grdn:0.179 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:07:22 ts/train.py:232 step:424K smpl:3M ep:18K epch:58.39 loss:0.006 grdn:0.195 lr:5.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 16:08:47 ts/train.py:232 step:424K smpl:3M ep:18K epch:58.42 loss:0.005 grdn:0.175 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:10:12 ts/train.py:232 step:424K smpl:3M ep:18K epch:58.45 loss:0.005 grdn:0.172 lr:5.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:11:37 ts/train.py:232 step:425K smpl:3M ep:18K epch:58.47 loss:0.006 grdn:0.192 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:13:02 ts/train.py:232 step:425K smpl:3M ep:18K epch:58.50 loss:0.006 grdn:0.183 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:14:27 ts/train.py:232 step:425K smpl:3M ep:18K epch:58.53 loss:0.005 grdn:0.172 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:15:52 ts/train.py:232 step:425K smpl:3M ep:18K epch:58.56 loss:0.005 grdn:0.174 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:17:17 ts/train.py:232 step:425K smpl:3M ep:18K epch:58.58 loss:0.005 grdn:0.183 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:18:42 ts/train.py:232 step:426K smpl:3M ep:18K epch:58.61 loss:0.006 grdn:0.189 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:20:07 ts/train.py:232 step:426K smpl:3M ep:18K epch:58.64 loss:0.006 grdn:0.171 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:21:32 ts/train.py:232 step:426K smpl:3M ep:18K epch:58.67 loss:0.006 grdn:0.184 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:22:56 ts/train.py:232 step:426K smpl:3M ep:18K epch:58.70 loss:0.006 grdn:0.191 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:24:21 ts/train.py:232 step:426K smpl:3M ep:18K epch:58.72 loss:0.006 grdn:0.177 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:25:46 ts/train.py:232 step:427K smpl:3M ep:18K epch:58.75 loss:0.006 grdn:0.179 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:27:11 ts/train.py:232 step:427K smpl:3M ep:18K epch:58.78 loss:0.005 grdn:0.175 lr:5.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 16:29:54 ts/train.py:232 step:427K smpl:3M ep:18K epch:58.81 loss:0.006 grdn:0.175 lr:5.4e-05 updt_s:0.422 data_s:0.390 +INFO 2025-05-07 16:31:19 ts/train.py:232 step:427K smpl:3M ep:18K epch:58.83 loss:0.006 grdn:0.177 lr:5.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 16:32:43 ts/train.py:232 step:427K smpl:3M ep:18K epch:58.86 loss:0.006 grdn:0.184 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:34:08 ts/train.py:232 step:428K smpl:3M ep:18K epch:58.89 loss:0.005 grdn:0.172 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:35:33 ts/train.py:232 step:428K smpl:3M ep:18K epch:58.92 loss:0.006 grdn:0.193 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:36:58 ts/train.py:232 step:428K smpl:3M ep:18K epch:58.94 loss:0.005 grdn:0.175 lr:5.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 16:38:23 ts/train.py:232 step:428K smpl:3M ep:18K epch:58.97 loss:0.005 grdn:0.166 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:39:48 ts/train.py:232 step:428K smpl:3M ep:18K epch:59.00 loss:0.005 grdn:0.171 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:41:13 ts/train.py:232 step:429K smpl:3M ep:18K epch:59.03 loss:0.006 grdn:0.180 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:42:38 ts/train.py:232 step:429K smpl:3M ep:18K epch:59.05 loss:0.005 grdn:0.174 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:44:03 ts/train.py:232 step:429K smpl:3M ep:18K epch:59.08 loss:0.005 grdn:0.182 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:45:28 ts/train.py:232 step:429K smpl:3M ep:18K epch:59.11 loss:0.005 grdn:0.167 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:46:53 ts/train.py:232 step:429K smpl:3M ep:18K epch:59.14 loss:0.005 grdn:0.174 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:48:18 ts/train.py:232 step:430K smpl:3M ep:18K epch:59.16 loss:0.005 grdn:0.177 lr:5.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 16:49:43 ts/train.py:232 step:430K smpl:3M ep:18K epch:59.19 loss:0.006 grdn:0.185 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:51:08 ts/train.py:232 step:430K smpl:3M ep:18K epch:59.22 loss:0.006 grdn:0.184 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:52:33 ts/train.py:232 step:430K smpl:3M ep:18K epch:59.25 loss:0.006 grdn:0.179 lr:5.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:53:58 ts/train.py:232 step:430K smpl:3M ep:18K epch:59.27 loss:0.006 grdn:0.182 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 16:55:23 ts/train.py:232 step:431K smpl:3M ep:18K epch:59.30 loss:0.005 grdn:0.180 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 16:56:48 ts/train.py:232 step:431K smpl:3M ep:18K epch:59.33 loss:0.005 grdn:0.171 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 16:58:13 ts/train.py:232 step:431K smpl:3M ep:18K epch:59.36 loss:0.005 grdn:0.175 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 16:59:38 ts/train.py:232 step:431K smpl:3M ep:18K epch:59.38 loss:0.005 grdn:0.170 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:01:03 ts/train.py:232 step:431K smpl:3M ep:18K epch:59.41 loss:0.006 grdn:0.189 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:02:28 ts/train.py:232 step:432K smpl:3M ep:18K epch:59.44 loss:0.005 grdn:0.166 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:03:53 ts/train.py:232 step:432K smpl:3M ep:18K epch:59.47 loss:0.005 grdn:0.179 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:05:18 ts/train.py:232 step:432K smpl:3M ep:18K epch:59.49 loss:0.005 grdn:0.162 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:06:43 ts/train.py:232 step:432K smpl:3M ep:18K epch:59.52 loss:0.006 grdn:0.183 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:08:08 ts/train.py:232 step:432K smpl:3M ep:18K epch:59.55 loss:0.004 grdn:0.160 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:09:33 ts/train.py:232 step:433K smpl:3M ep:18K epch:59.58 loss:0.005 grdn:0.177 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:10:58 ts/train.py:232 step:433K smpl:3M ep:18K epch:59.60 loss:0.005 grdn:0.174 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:12:23 ts/train.py:232 step:433K smpl:3M ep:18K epch:59.63 loss:0.007 grdn:0.194 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:13:48 ts/train.py:232 step:433K smpl:3M ep:18K epch:59.66 loss:0.005 grdn:0.162 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:15:13 ts/train.py:232 step:433K smpl:3M ep:18K epch:59.69 loss:0.006 grdn:0.174 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:16:38 ts/train.py:232 step:434K smpl:3M ep:18K epch:59.71 loss:0.006 grdn:0.191 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:18:03 ts/train.py:232 step:434K smpl:3M ep:18K epch:59.74 loss:0.006 grdn:0.181 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:19:29 ts/train.py:232 step:434K smpl:3M ep:18K epch:59.77 loss:0.005 grdn:0.172 lr:5.3e-05 updt_s:0.423 data_s:0.007 +INFO 2025-05-07 17:20:53 ts/train.py:232 step:434K smpl:3M ep:18K epch:59.80 loss:0.005 grdn:0.168 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:22:18 ts/train.py:232 step:434K smpl:3M ep:18K epch:59.82 loss:0.006 grdn:0.188 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:23:43 ts/train.py:232 step:435K smpl:3M ep:18K epch:59.85 loss:0.005 grdn:0.179 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:25:08 ts/train.py:232 step:435K smpl:3M ep:18K epch:59.88 loss:0.005 grdn:0.175 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:26:33 ts/train.py:232 step:435K smpl:3M ep:18K epch:59.91 loss:0.006 grdn:0.180 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:27:59 ts/train.py:232 step:435K smpl:3M ep:18K epch:59.93 loss:0.006 grdn:0.196 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:29:24 ts/train.py:232 step:435K smpl:3M ep:18K epch:59.96 loss:0.005 grdn:0.170 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:30:49 ts/train.py:232 step:436K smpl:3M ep:18K epch:59.99 loss:0.005 grdn:0.173 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:32:14 ts/train.py:232 step:436K smpl:3M ep:18K epch:60.02 loss:0.005 grdn:0.178 lr:5.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:33:39 ts/train.py:232 step:436K smpl:3M ep:18K epch:60.04 loss:0.005 grdn:0.180 lr:5.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:35:04 ts/train.py:232 step:436K smpl:3M ep:18K epch:60.07 loss:0.006 grdn:0.183 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:36:29 ts/train.py:232 step:436K smpl:3M ep:18K epch:60.10 loss:0.005 grdn:0.179 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:37:54 ts/train.py:232 step:437K smpl:3M ep:18K epch:60.13 loss:0.006 grdn:0.170 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:39:19 ts/train.py:232 step:437K smpl:3M ep:18K epch:60.15 loss:0.005 grdn:0.177 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:40:44 ts/train.py:232 step:437K smpl:3M ep:18K epch:60.18 loss:0.006 grdn:0.185 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:42:09 ts/train.py:232 step:437K smpl:3M ep:18K epch:60.21 loss:0.005 grdn:0.179 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:43:34 ts/train.py:232 step:437K smpl:3M ep:18K epch:60.24 loss:0.006 grdn:0.179 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:44:59 ts/train.py:232 step:438K smpl:4M ep:18K epch:60.27 loss:0.005 grdn:0.174 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:46:24 ts/train.py:232 step:438K smpl:4M ep:18K epch:60.29 loss:0.006 grdn:0.197 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:47:49 ts/train.py:232 step:438K smpl:4M ep:18K epch:60.32 loss:0.005 grdn:0.172 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:49:14 ts/train.py:232 step:438K smpl:4M ep:18K epch:60.35 loss:0.006 grdn:0.187 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:50:39 ts/train.py:232 step:438K smpl:4M ep:18K epch:60.38 loss:0.006 grdn:0.184 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:52:04 ts/train.py:232 step:439K smpl:4M ep:18K epch:60.40 loss:0.006 grdn:0.189 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:53:29 ts/train.py:232 step:439K smpl:4M ep:18K epch:60.43 loss:0.006 grdn:0.190 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:54:54 ts/train.py:232 step:439K smpl:4M ep:18K epch:60.46 loss:0.006 grdn:0.181 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 17:56:19 ts/train.py:232 step:439K smpl:4M ep:18K epch:60.49 loss:0.005 grdn:0.167 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:57:44 ts/train.py:232 step:439K smpl:4M ep:18K epch:60.51 loss:0.006 grdn:0.182 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 17:59:09 ts/train.py:232 step:440K smpl:4M ep:18K epch:60.54 loss:0.005 grdn:0.168 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:00:34 ts/train.py:232 step:440K smpl:4M ep:18K epch:60.57 loss:0.005 grdn:0.177 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:01:59 ts/train.py:232 step:440K smpl:4M ep:18K epch:60.60 loss:0.005 grdn:0.192 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:03:24 ts/train.py:232 step:440K smpl:4M ep:18K epch:60.62 loss:0.005 grdn:0.180 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:04:49 ts/train.py:232 step:440K smpl:4M ep:18K epch:60.65 loss:0.005 grdn:0.162 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:06:13 ts/train.py:232 step:441K smpl:4M ep:18K epch:60.68 loss:0.006 grdn:0.184 lr:5.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:07:39 ts/train.py:232 step:441K smpl:4M ep:18K epch:60.71 loss:0.005 grdn:0.186 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:09:05 ts/train.py:232 step:441K smpl:4M ep:18K epch:60.73 loss:0.005 grdn:0.176 lr:5.2e-05 updt_s:0.423 data_s:0.009 +INFO 2025-05-07 18:10:30 ts/train.py:232 step:441K smpl:4M ep:18K epch:60.76 loss:0.005 grdn:0.182 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:11:55 ts/train.py:232 step:441K smpl:4M ep:18K epch:60.79 loss:0.005 grdn:0.187 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:13:20 ts/train.py:232 step:442K smpl:4M ep:18K epch:60.82 loss:0.005 grdn:0.175 lr:5.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:14:45 ts/train.py:232 step:442K smpl:4M ep:18K epch:60.84 loss:0.005 grdn:0.179 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:16:10 ts/train.py:232 step:442K smpl:4M ep:18K epch:60.87 loss:0.006 grdn:0.190 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:17:35 ts/train.py:232 step:442K smpl:4M ep:18K epch:60.90 loss:0.005 grdn:0.181 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:19:00 ts/train.py:232 step:442K smpl:4M ep:18K epch:60.93 loss:0.005 grdn:0.173 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:20:25 ts/train.py:232 step:443K smpl:4M ep:18K epch:60.95 loss:0.005 grdn:0.175 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:21:50 ts/train.py:232 step:443K smpl:4M ep:18K epch:60.98 loss:0.005 grdn:0.175 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:23:15 ts/train.py:232 step:443K smpl:4M ep:18K epch:61.01 loss:0.006 grdn:0.185 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:24:40 ts/train.py:232 step:443K smpl:4M ep:18K epch:61.04 loss:0.005 grdn:0.178 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:26:06 ts/train.py:232 step:443K smpl:4M ep:18K epch:61.06 loss:0.006 grdn:0.183 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:27:31 ts/train.py:232 step:444K smpl:4M ep:18K epch:61.09 loss:0.005 grdn:0.180 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:28:56 ts/train.py:232 step:444K smpl:4M ep:18K epch:61.12 loss:0.006 grdn:0.181 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:30:21 ts/train.py:232 step:444K smpl:4M ep:18K epch:61.15 loss:0.005 grdn:0.183 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:31:46 ts/train.py:232 step:444K smpl:4M ep:18K epch:61.17 loss:0.005 grdn:0.179 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:33:11 ts/train.py:232 step:444K smpl:4M ep:18K epch:61.20 loss:0.006 grdn:0.181 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:34:36 ts/train.py:232 step:445K smpl:4M ep:18K epch:61.23 loss:0.005 grdn:0.175 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:36:01 ts/train.py:232 step:445K smpl:4M ep:18K epch:61.26 loss:0.005 grdn:0.181 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:37:26 ts/train.py:232 step:445K smpl:4M ep:18K epch:61.28 loss:0.005 grdn:0.186 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:38:51 ts/train.py:232 step:445K smpl:4M ep:18K epch:61.31 loss:0.005 grdn:0.172 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:40:16 ts/train.py:232 step:445K smpl:4M ep:18K epch:61.34 loss:0.006 grdn:0.184 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:41:41 ts/train.py:232 step:446K smpl:4M ep:18K epch:61.37 loss:0.005 grdn:0.165 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:43:06 ts/train.py:232 step:446K smpl:4M ep:18K epch:61.39 loss:0.005 grdn:0.166 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:44:31 ts/train.py:232 step:446K smpl:4M ep:18K epch:61.42 loss:0.006 grdn:0.181 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:45:56 ts/train.py:232 step:446K smpl:4M ep:18K epch:61.45 loss:0.005 grdn:0.176 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:47:21 ts/train.py:232 step:446K smpl:4M ep:18K epch:61.48 loss:0.005 grdn:0.183 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:48:45 ts/train.py:232 step:447K smpl:4M ep:18K epch:61.50 loss:0.005 grdn:0.171 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:50:10 ts/train.py:232 step:447K smpl:4M ep:18K epch:61.53 loss:0.004 grdn:0.156 lr:5.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:51:35 ts/train.py:232 step:447K smpl:4M ep:18K epch:61.56 loss:0.005 grdn:0.170 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:53:00 ts/train.py:232 step:447K smpl:4M ep:18K epch:61.59 loss:0.006 grdn:0.178 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:54:25 ts/train.py:232 step:447K smpl:4M ep:18K epch:61.61 loss:0.005 grdn:0.172 lr:5.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:55:50 ts/train.py:232 step:448K smpl:4M ep:18K epch:61.64 loss:0.005 grdn:0.172 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 18:57:15 ts/train.py:232 step:448K smpl:4M ep:19K epch:61.67 loss:0.006 grdn:0.189 lr:5.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 18:58:42 ts/train.py:232 step:448K smpl:4M ep:19K epch:61.70 loss:0.005 grdn:0.181 lr:5.0e-05 updt_s:0.423 data_s:0.010 +INFO 2025-05-07 19:00:07 ts/train.py:232 step:448K smpl:4M ep:19K epch:61.72 loss:0.005 grdn:0.180 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:01:32 ts/train.py:232 step:448K smpl:4M ep:19K epch:61.75 loss:0.005 grdn:0.179 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:02:57 ts/train.py:232 step:449K smpl:4M ep:19K epch:61.78 loss:0.005 grdn:0.181 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:04:22 ts/train.py:232 step:449K smpl:4M ep:19K epch:61.81 loss:0.005 grdn:0.179 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:05:47 ts/train.py:232 step:449K smpl:4M ep:19K epch:61.84 loss:0.005 grdn:0.169 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:07:12 ts/train.py:232 step:449K smpl:4M ep:19K epch:61.86 loss:0.005 grdn:0.179 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:08:37 ts/train.py:232 step:449K smpl:4M ep:19K epch:61.89 loss:0.005 grdn:0.171 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:10:02 ts/train.py:232 step:450K smpl:4M ep:19K epch:61.92 loss:0.005 grdn:0.162 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:11:27 ts/train.py:232 step:450K smpl:4M ep:19K epch:61.95 loss:0.005 grdn:0.175 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:12:52 ts/train.py:232 step:450K smpl:4M ep:19K epch:61.97 loss:0.006 grdn:0.196 lr:5.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:12:52 ts/train.py:241 Checkpoint policy after step 450000 +INFO 2025-05-07 19:14:20 ts/train.py:232 step:450K smpl:4M ep:19K epch:62.00 loss:0.005 grdn:0.160 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:15:45 ts/train.py:232 step:450K smpl:4M ep:19K epch:62.03 loss:0.005 grdn:0.186 lr:5.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:17:10 ts/train.py:232 step:451K smpl:4M ep:19K epch:62.06 loss:0.005 grdn:0.175 lr:5.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:18:35 ts/train.py:232 step:451K smpl:4M ep:19K epch:62.08 loss:0.005 grdn:0.182 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:20:00 ts/train.py:232 step:451K smpl:4M ep:19K epch:62.11 loss:0.005 grdn:0.177 lr:5.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:21:26 ts/train.py:232 step:451K smpl:4M ep:19K epch:62.14 loss:0.005 grdn:0.172 lr:5.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:22:51 ts/train.py:232 step:451K smpl:4M ep:19K epch:62.17 loss:0.006 grdn:0.182 lr:5.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:24:16 ts/train.py:232 step:452K smpl:4M ep:19K epch:62.19 loss:0.005 grdn:0.174 lr:5.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:25:41 ts/train.py:232 step:452K smpl:4M ep:19K epch:62.22 loss:0.005 grdn:0.177 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:27:06 ts/train.py:232 step:452K smpl:4M ep:19K epch:62.25 loss:0.005 grdn:0.177 lr:5.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:28:31 ts/train.py:232 step:452K smpl:4M ep:19K epch:62.28 loss:0.006 grdn:0.194 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:29:56 ts/train.py:232 step:452K smpl:4M ep:19K epch:62.30 loss:0.005 grdn:0.177 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:31:21 ts/train.py:232 step:453K smpl:4M ep:19K epch:62.33 loss:0.005 grdn:0.175 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:32:46 ts/train.py:232 step:453K smpl:4M ep:19K epch:62.36 loss:0.005 grdn:0.172 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:34:10 ts/train.py:232 step:453K smpl:4M ep:19K epch:62.39 loss:0.005 grdn:0.173 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:35:35 ts/train.py:232 step:453K smpl:4M ep:19K epch:62.41 loss:0.005 grdn:0.173 lr:5.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:37:00 ts/train.py:232 step:453K smpl:4M ep:19K epch:62.44 loss:0.006 grdn:0.189 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:38:25 ts/train.py:232 step:454K smpl:4M ep:19K epch:62.47 loss:0.005 grdn:0.183 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:39:50 ts/train.py:232 step:454K smpl:4M ep:19K epch:62.50 loss:0.006 grdn:0.184 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:41:15 ts/train.py:232 step:454K smpl:4M ep:19K epch:62.52 loss:0.006 grdn:0.185 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:42:40 ts/train.py:232 step:454K smpl:4M ep:19K epch:62.55 loss:0.005 grdn:0.183 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:44:05 ts/train.py:232 step:454K smpl:4M ep:19K epch:62.58 loss:0.004 grdn:0.164 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:45:30 ts/train.py:232 step:455K smpl:4M ep:19K epch:62.61 loss:0.006 grdn:0.184 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:46:55 ts/train.py:232 step:455K smpl:4M ep:19K epch:62.63 loss:0.006 grdn:0.198 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:48:22 ts/train.py:232 step:455K smpl:4M ep:19K epch:62.66 loss:0.005 grdn:0.163 lr:4.9e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-07 19:49:47 ts/train.py:232 step:455K smpl:4M ep:19K epch:62.69 loss:0.004 grdn:0.166 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:51:12 ts/train.py:232 step:455K smpl:4M ep:19K epch:62.72 loss:0.006 grdn:0.194 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:52:37 ts/train.py:232 step:456K smpl:4M ep:19K epch:62.74 loss:0.005 grdn:0.186 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:54:02 ts/train.py:232 step:456K smpl:4M ep:19K epch:62.77 loss:0.005 grdn:0.176 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:55:27 ts/train.py:232 step:456K smpl:4M ep:19K epch:62.80 loss:0.005 grdn:0.181 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:56:52 ts/train.py:232 step:456K smpl:4M ep:19K epch:62.83 loss:0.006 grdn:0.178 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 19:58:17 ts/train.py:232 step:456K smpl:4M ep:19K epch:62.85 loss:0.006 grdn:0.187 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 19:59:42 ts/train.py:232 step:457K smpl:4M ep:19K epch:62.88 loss:0.006 grdn:0.189 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:01:07 ts/train.py:232 step:457K smpl:4M ep:19K epch:62.91 loss:0.006 grdn:0.183 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:02:32 ts/train.py:232 step:457K smpl:4M ep:19K epch:62.94 loss:0.005 grdn:0.170 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:03:57 ts/train.py:232 step:457K smpl:4M ep:19K epch:62.96 loss:0.005 grdn:0.171 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:05:22 ts/train.py:232 step:457K smpl:4M ep:19K epch:62.99 loss:0.005 grdn:0.170 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:06:47 ts/train.py:232 step:458K smpl:4M ep:19K epch:63.02 loss:0.006 grdn:0.189 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:08:12 ts/train.py:232 step:458K smpl:4M ep:19K epch:63.05 loss:0.005 grdn:0.182 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:09:37 ts/train.py:232 step:458K smpl:4M ep:19K epch:63.07 loss:0.005 grdn:0.191 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:11:02 ts/train.py:232 step:458K smpl:4M ep:19K epch:63.10 loss:0.005 grdn:0.179 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:12:27 ts/train.py:232 step:458K smpl:4M ep:19K epch:63.13 loss:0.005 grdn:0.175 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:13:52 ts/train.py:232 step:459K smpl:4M ep:19K epch:63.16 loss:0.005 grdn:0.184 lr:4.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:15:17 ts/train.py:232 step:459K smpl:4M ep:19K epch:63.18 loss:0.006 grdn:0.187 lr:4.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:16:42 ts/train.py:232 step:459K smpl:4M ep:19K epch:63.21 loss:0.005 grdn:0.167 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:18:07 ts/train.py:232 step:459K smpl:4M ep:19K epch:63.24 loss:0.005 grdn:0.173 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:19:32 ts/train.py:232 step:459K smpl:4M ep:19K epch:63.27 loss:0.005 grdn:0.171 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:20:57 ts/train.py:232 step:460K smpl:4M ep:19K epch:63.29 loss:0.005 grdn:0.177 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:22:22 ts/train.py:232 step:460K smpl:4M ep:19K epch:63.32 loss:0.005 grdn:0.168 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:23:47 ts/train.py:232 step:460K smpl:4M ep:19K epch:63.35 loss:0.005 grdn:0.173 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:25:12 ts/train.py:232 step:460K smpl:4M ep:19K epch:63.38 loss:0.005 grdn:0.174 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:26:37 ts/train.py:232 step:460K smpl:4M ep:19K epch:63.41 loss:0.006 grdn:0.185 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:28:02 ts/train.py:232 step:461K smpl:4M ep:19K epch:63.43 loss:0.005 grdn:0.182 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:29:27 ts/train.py:232 step:461K smpl:4M ep:19K epch:63.46 loss:0.005 grdn:0.182 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:30:52 ts/train.py:232 step:461K smpl:4M ep:19K epch:63.49 loss:0.005 grdn:0.175 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:32:17 ts/train.py:232 step:461K smpl:4M ep:19K epch:63.52 loss:0.005 grdn:0.185 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:33:42 ts/train.py:232 step:461K smpl:4M ep:19K epch:63.54 loss:0.005 grdn:0.169 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:35:07 ts/train.py:232 step:462K smpl:4M ep:19K epch:63.57 loss:0.004 grdn:0.167 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:36:32 ts/train.py:232 step:462K smpl:4M ep:19K epch:63.60 loss:0.006 grdn:0.187 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:37:58 ts/train.py:232 step:462K smpl:4M ep:19K epch:63.63 loss:0.005 grdn:0.176 lr:4.8e-05 updt_s:0.423 data_s:0.007 +INFO 2025-05-07 20:39:23 ts/train.py:232 step:462K smpl:4M ep:19K epch:63.65 loss:0.005 grdn:0.175 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:40:49 ts/train.py:232 step:462K smpl:4M ep:19K epch:63.68 loss:0.005 grdn:0.166 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:42:14 ts/train.py:232 step:463K smpl:4M ep:19K epch:63.71 loss:0.005 grdn:0.178 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:43:39 ts/train.py:232 step:463K smpl:4M ep:19K epch:63.74 loss:0.005 grdn:0.181 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:45:04 ts/train.py:232 step:463K smpl:4M ep:19K epch:63.76 loss:0.006 grdn:0.197 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:46:29 ts/train.py:232 step:463K smpl:4M ep:19K epch:63.79 loss:0.005 grdn:0.182 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:47:54 ts/train.py:232 step:463K smpl:4M ep:19K epch:63.82 loss:0.005 grdn:0.165 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:49:18 ts/train.py:232 step:464K smpl:4M ep:19K epch:63.85 loss:0.005 grdn:0.179 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:50:44 ts/train.py:232 step:464K smpl:4M ep:19K epch:63.87 loss:0.005 grdn:0.172 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:52:09 ts/train.py:232 step:464K smpl:4M ep:19K epch:63.90 loss:0.005 grdn:0.175 lr:4.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 20:53:34 ts/train.py:232 step:464K smpl:4M ep:19K epch:63.93 loss:0.005 grdn:0.177 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:54:59 ts/train.py:232 step:464K smpl:4M ep:19K epch:63.96 loss:0.005 grdn:0.175 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:56:23 ts/train.py:232 step:465K smpl:4M ep:19K epch:63.98 loss:0.005 grdn:0.174 lr:4.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:57:48 ts/train.py:232 step:465K smpl:4M ep:19K epch:64.01 loss:0.005 grdn:0.184 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 20:59:13 ts/train.py:232 step:465K smpl:4M ep:19K epch:64.04 loss:0.005 grdn:0.180 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:00:38 ts/train.py:232 step:465K smpl:4M ep:19K epch:64.07 loss:0.004 grdn:0.178 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:02:03 ts/train.py:232 step:465K smpl:4M ep:19K epch:64.09 loss:0.005 grdn:0.176 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:03:28 ts/train.py:232 step:466K smpl:4M ep:19K epch:64.12 loss:0.005 grdn:0.172 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:04:53 ts/train.py:232 step:466K smpl:4M ep:19K epch:64.15 loss:0.005 grdn:0.189 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:06:18 ts/train.py:232 step:466K smpl:4M ep:19K epch:64.18 loss:0.005 grdn:0.187 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:07:42 ts/train.py:232 step:466K smpl:4M ep:19K epch:64.20 loss:0.005 grdn:0.178 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:09:07 ts/train.py:232 step:466K smpl:4M ep:19K epch:64.23 loss:0.004 grdn:0.170 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:10:32 ts/train.py:232 step:467K smpl:4M ep:19K epch:64.26 loss:0.004 grdn:0.168 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:11:57 ts/train.py:232 step:467K smpl:4M ep:19K epch:64.29 loss:0.005 grdn:0.182 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:13:22 ts/train.py:232 step:467K smpl:4M ep:19K epch:64.31 loss:0.004 grdn:0.165 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:14:47 ts/train.py:232 step:467K smpl:4M ep:19K epch:64.34 loss:0.005 grdn:0.181 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:16:12 ts/train.py:232 step:467K smpl:4M ep:19K epch:64.37 loss:0.005 grdn:0.181 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:17:37 ts/train.py:232 step:468K smpl:4M ep:19K epch:64.40 loss:0.005 grdn:0.176 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:19:02 ts/train.py:232 step:468K smpl:4M ep:19K epch:64.42 loss:0.005 grdn:0.171 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:20:27 ts/train.py:232 step:468K smpl:4M ep:19K epch:64.45 loss:0.005 grdn:0.170 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:21:52 ts/train.py:232 step:468K smpl:4M ep:19K epch:64.48 loss:0.006 grdn:0.191 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:23:17 ts/train.py:232 step:468K smpl:4M ep:19K epch:64.51 loss:0.005 grdn:0.175 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:24:42 ts/train.py:232 step:469K smpl:4M ep:19K epch:64.53 loss:0.004 grdn:0.162 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:26:06 ts/train.py:232 step:469K smpl:4M ep:19K epch:64.56 loss:0.005 grdn:0.169 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:27:33 ts/train.py:232 step:469K smpl:4M ep:19K epch:64.59 loss:0.006 grdn:0.188 lr:4.7e-05 updt_s:0.422 data_s:0.007 +INFO 2025-05-07 21:28:57 ts/train.py:232 step:469K smpl:4M ep:19K epch:64.62 loss:0.004 grdn:0.164 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:30:22 ts/train.py:232 step:469K smpl:4M ep:19K epch:64.64 loss:0.004 grdn:0.163 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:31:47 ts/train.py:232 step:470K smpl:4M ep:19K epch:64.67 loss:0.004 grdn:0.171 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:33:12 ts/train.py:232 step:470K smpl:4M ep:19K epch:64.70 loss:0.005 grdn:0.181 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:34:37 ts/train.py:232 step:470K smpl:4M ep:19K epch:64.73 loss:0.005 grdn:0.193 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:36:01 ts/train.py:232 step:470K smpl:4M ep:19K epch:64.75 loss:0.004 grdn:0.159 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:37:26 ts/train.py:232 step:470K smpl:4M ep:19K epch:64.78 loss:0.005 grdn:0.181 lr:4.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:38:51 ts/train.py:232 step:471K smpl:4M ep:19K epch:64.81 loss:0.005 grdn:0.190 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:40:16 ts/train.py:232 step:471K smpl:4M ep:19K epch:64.84 loss:0.005 grdn:0.192 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:41:41 ts/train.py:232 step:471K smpl:4M ep:19K epch:64.86 loss:0.005 grdn:0.184 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:43:06 ts/train.py:232 step:471K smpl:4M ep:19K epch:64.89 loss:0.005 grdn:0.176 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:44:31 ts/train.py:232 step:471K smpl:4M ep:19K epch:64.92 loss:0.005 grdn:0.179 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:45:56 ts/train.py:232 step:472K smpl:4M ep:19K epch:64.95 loss:0.005 grdn:0.182 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:47:21 ts/train.py:232 step:472K smpl:4M ep:19K epch:64.98 loss:0.005 grdn:0.181 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:48:45 ts/train.py:232 step:472K smpl:4M ep:20K epch:65.00 loss:0.006 grdn:0.192 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:50:10 ts/train.py:232 step:472K smpl:4M ep:20K epch:65.03 loss:0.005 grdn:0.182 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:51:35 ts/train.py:232 step:472K smpl:4M ep:20K epch:65.06 loss:0.005 grdn:0.194 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:53:00 ts/train.py:232 step:473K smpl:4M ep:20K epch:65.09 loss:0.006 grdn:0.188 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:54:25 ts/train.py:232 step:473K smpl:4M ep:20K epch:65.11 loss:0.005 grdn:0.176 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:55:50 ts/train.py:232 step:473K smpl:4M ep:20K epch:65.14 loss:0.004 grdn:0.169 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:57:15 ts/train.py:232 step:473K smpl:4M ep:20K epch:65.17 loss:0.005 grdn:0.180 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 21:58:40 ts/train.py:232 step:473K smpl:4M ep:20K epch:65.20 loss:0.005 grdn:0.180 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:00:05 ts/train.py:232 step:474K smpl:4M ep:20K epch:65.22 loss:0.005 grdn:0.171 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:01:30 ts/train.py:232 step:474K smpl:4M ep:20K epch:65.25 loss:0.005 grdn:0.181 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:02:54 ts/train.py:232 step:474K smpl:4M ep:20K epch:65.28 loss:0.005 grdn:0.168 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:04:19 ts/train.py:232 step:474K smpl:4M ep:20K epch:65.31 loss:0.005 grdn:0.172 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:05:44 ts/train.py:232 step:474K smpl:4M ep:20K epch:65.33 loss:0.005 grdn:0.185 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:07:09 ts/train.py:232 step:475K smpl:4M ep:20K epch:65.36 loss:0.005 grdn:0.180 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:08:34 ts/train.py:232 step:475K smpl:4M ep:20K epch:65.39 loss:0.005 grdn:0.183 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:09:59 ts/train.py:232 step:475K smpl:4M ep:20K epch:65.42 loss:0.005 grdn:0.168 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:11:24 ts/train.py:232 step:475K smpl:4M ep:20K epch:65.44 loss:0.005 grdn:0.182 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:12:49 ts/train.py:232 step:475K smpl:4M ep:20K epch:65.47 loss:0.005 grdn:0.182 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:14:14 ts/train.py:232 step:476K smpl:4M ep:20K epch:65.50 loss:0.005 grdn:0.182 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:15:39 ts/train.py:232 step:476K smpl:4M ep:20K epch:65.53 loss:0.006 grdn:0.191 lr:4.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:17:05 ts/train.py:232 step:476K smpl:4M ep:20K epch:65.55 loss:0.005 grdn:0.175 lr:4.6e-05 updt_s:0.423 data_s:0.009 +INFO 2025-05-07 22:18:30 ts/train.py:232 step:476K smpl:4M ep:20K epch:65.58 loss:0.005 grdn:0.178 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:19:55 ts/train.py:232 step:476K smpl:4M ep:20K epch:65.61 loss:0.005 grdn:0.182 lr:4.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 22:21:20 ts/train.py:232 step:477K smpl:4M ep:20K epch:65.64 loss:0.005 grdn:0.196 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:22:45 ts/train.py:232 step:477K smpl:4M ep:20K epch:65.66 loss:0.005 grdn:0.177 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:24:10 ts/train.py:232 step:477K smpl:4M ep:20K epch:65.69 loss:0.005 grdn:0.176 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:25:35 ts/train.py:232 step:477K smpl:4M ep:20K epch:65.72 loss:0.005 grdn:0.182 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:27:00 ts/train.py:232 step:477K smpl:4M ep:20K epch:65.75 loss:0.005 grdn:0.194 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:28:25 ts/train.py:232 step:478K smpl:4M ep:20K epch:65.77 loss:0.005 grdn:0.185 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:29:50 ts/train.py:232 step:478K smpl:4M ep:20K epch:65.80 loss:0.005 grdn:0.171 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:31:15 ts/train.py:232 step:478K smpl:4M ep:20K epch:65.83 loss:0.005 grdn:0.170 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:32:40 ts/train.py:232 step:478K smpl:4M ep:20K epch:65.86 loss:0.005 grdn:0.178 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:34:05 ts/train.py:232 step:478K smpl:4M ep:20K epch:65.88 loss:0.005 grdn:0.177 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:35:30 ts/train.py:232 step:479K smpl:4M ep:20K epch:65.91 loss:0.004 grdn:0.170 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:36:55 ts/train.py:232 step:479K smpl:4M ep:20K epch:65.94 loss:0.005 grdn:0.177 lr:4.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 22:38:20 ts/train.py:232 step:479K smpl:4M ep:20K epch:65.97 loss:0.005 grdn:0.186 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:39:45 ts/train.py:232 step:479K smpl:4M ep:20K epch:65.99 loss:0.005 grdn:0.196 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:41:10 ts/train.py:232 step:479K smpl:4M ep:20K epch:66.02 loss:0.005 grdn:0.178 lr:4.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 22:42:35 ts/train.py:232 step:480K smpl:4M ep:20K epch:66.05 loss:0.005 grdn:0.178 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:44:00 ts/train.py:232 step:480K smpl:4M ep:20K epch:66.08 loss:0.005 grdn:0.171 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:45:25 ts/train.py:232 step:480K smpl:4M ep:20K epch:66.10 loss:0.005 grdn:0.177 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:45:25 ts/train.py:241 Checkpoint policy after step 480000 +INFO 2025-05-07 22:46:53 ts/train.py:232 step:480K smpl:4M ep:20K epch:66.13 loss:0.005 grdn:0.187 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:48:18 ts/train.py:232 step:480K smpl:4M ep:20K epch:66.16 loss:0.005 grdn:0.178 lr:4.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 22:49:43 ts/train.py:232 step:481K smpl:4M ep:20K epch:66.19 loss:0.005 grdn:0.186 lr:4.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 22:51:08 ts/train.py:232 step:481K smpl:4M ep:20K epch:66.21 loss:0.005 grdn:0.176 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:52:33 ts/train.py:232 step:481K smpl:4M ep:20K epch:66.24 loss:0.005 grdn:0.185 lr:4.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 22:53:58 ts/train.py:232 step:481K smpl:4M ep:20K epch:66.27 loss:0.005 grdn:0.184 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:55:23 ts/train.py:232 step:481K smpl:4M ep:20K epch:66.30 loss:0.005 grdn:0.175 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:56:48 ts/train.py:232 step:482K smpl:4M ep:20K epch:66.32 loss:0.005 grdn:0.174 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:58:13 ts/train.py:232 step:482K smpl:4M ep:20K epch:66.35 loss:0.004 grdn:0.159 lr:4.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 22:59:38 ts/train.py:232 step:482K smpl:4M ep:20K epch:66.38 loss:0.005 grdn:0.182 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:01:03 ts/train.py:232 step:482K smpl:4M ep:20K epch:66.41 loss:0.005 grdn:0.166 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:02:28 ts/train.py:232 step:482K smpl:4M ep:20K epch:66.43 loss:0.005 grdn:0.181 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:03:53 ts/train.py:232 step:483K smpl:4M ep:20K epch:66.46 loss:0.005 grdn:0.183 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:05:18 ts/train.py:232 step:483K smpl:4M ep:20K epch:66.49 loss:0.004 grdn:0.175 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:06:44 ts/train.py:232 step:483K smpl:4M ep:20K epch:66.52 loss:0.005 grdn:0.162 lr:4.4e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-07 23:08:09 ts/train.py:232 step:483K smpl:4M ep:20K epch:66.55 loss:0.005 grdn:0.182 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:09:35 ts/train.py:232 step:483K smpl:4M ep:20K epch:66.57 loss:0.005 grdn:0.172 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:11:00 ts/train.py:232 step:484K smpl:4M ep:20K epch:66.60 loss:0.005 grdn:0.175 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:12:25 ts/train.py:232 step:484K smpl:4M ep:20K epch:66.63 loss:0.005 grdn:0.178 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:13:50 ts/train.py:232 step:484K smpl:4M ep:20K epch:66.66 loss:0.005 grdn:0.176 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:15:15 ts/train.py:232 step:484K smpl:4M ep:20K epch:66.68 loss:0.005 grdn:0.183 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:16:40 ts/train.py:232 step:484K smpl:4M ep:20K epch:66.71 loss:0.005 grdn:0.169 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:18:05 ts/train.py:232 step:485K smpl:4M ep:20K epch:66.74 loss:0.005 grdn:0.179 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:19:30 ts/train.py:232 step:485K smpl:4M ep:20K epch:66.77 loss:0.006 grdn:0.191 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:20:55 ts/train.py:232 step:485K smpl:4M ep:20K epch:66.79 loss:0.005 grdn:0.183 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:22:20 ts/train.py:232 step:485K smpl:4M ep:20K epch:66.82 loss:0.005 grdn:0.176 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:23:45 ts/train.py:232 step:485K smpl:4M ep:20K epch:66.85 loss:0.006 grdn:0.195 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:25:10 ts/train.py:232 step:486K smpl:4M ep:20K epch:66.88 loss:0.005 grdn:0.178 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:26:34 ts/train.py:232 step:486K smpl:4M ep:20K epch:66.90 loss:0.005 grdn:0.178 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:27:59 ts/train.py:232 step:486K smpl:4M ep:20K epch:66.93 loss:0.004 grdn:0.172 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:29:24 ts/train.py:232 step:486K smpl:4M ep:20K epch:66.96 loss:0.006 grdn:0.201 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:30:49 ts/train.py:232 step:486K smpl:4M ep:20K epch:66.99 loss:0.005 grdn:0.175 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:32:14 ts/train.py:232 step:487K smpl:4M ep:20K epch:67.01 loss:0.005 grdn:0.176 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:33:39 ts/train.py:232 step:487K smpl:4M ep:20K epch:67.04 loss:0.005 grdn:0.178 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:35:04 ts/train.py:232 step:487K smpl:4M ep:20K epch:67.07 loss:0.004 grdn:0.179 lr:4.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:36:29 ts/train.py:232 step:487K smpl:4M ep:20K epch:67.10 loss:0.005 grdn:0.180 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:37:54 ts/train.py:232 step:487K smpl:4M ep:20K epch:67.12 loss:0.005 grdn:0.177 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:39:19 ts/train.py:232 step:488K smpl:4M ep:20K epch:67.15 loss:0.004 grdn:0.160 lr:4.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:40:44 ts/train.py:232 step:488K smpl:4M ep:20K epch:67.18 loss:0.005 grdn:0.175 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:42:09 ts/train.py:232 step:488K smpl:4M ep:20K epch:67.21 loss:0.004 grdn:0.169 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:43:34 ts/train.py:232 step:488K smpl:4M ep:20K epch:67.23 loss:0.005 grdn:0.174 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:44:59 ts/train.py:232 step:488K smpl:4M ep:20K epch:67.26 loss:0.005 grdn:0.177 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:46:24 ts/train.py:232 step:489K smpl:4M ep:20K epch:67.29 loss:0.005 grdn:0.172 lr:4.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:47:49 ts/train.py:232 step:489K smpl:4M ep:20K epch:67.32 loss:0.004 grdn:0.170 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:49:14 ts/train.py:232 step:489K smpl:4M ep:20K epch:67.34 loss:0.005 grdn:0.171 lr:4.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:50:39 ts/train.py:232 step:489K smpl:4M ep:20K epch:67.37 loss:0.005 grdn:0.188 lr:4.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 23:52:04 ts/train.py:232 step:489K smpl:4M ep:20K epch:67.40 loss:0.005 grdn:0.177 lr:4.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:53:29 ts/train.py:232 step:490K smpl:4M ep:20K epch:67.43 loss:0.005 grdn:0.185 lr:4.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:54:54 ts/train.py:232 step:490K smpl:4M ep:20K epch:67.45 loss:0.006 grdn:0.192 lr:4.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 23:56:20 ts/train.py:232 step:490K smpl:4M ep:20K epch:67.48 loss:0.005 grdn:0.170 lr:4.3e-05 updt_s:0.422 data_s:0.007 +INFO 2025-05-07 23:57:45 ts/train.py:232 step:490K smpl:4M ep:20K epch:67.51 loss:0.005 grdn:0.188 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 23:59:10 ts/train.py:232 step:490K smpl:4M ep:20K epch:67.54 loss:0.005 grdn:0.184 lr:4.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:00:35 ts/train.py:232 step:491K smpl:4M ep:20K epch:67.56 loss:0.005 grdn:0.188 lr:4.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:02:00 ts/train.py:232 step:491K smpl:4M ep:20K epch:67.59 loss:0.005 grdn:0.188 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:03:25 ts/train.py:232 step:491K smpl:4M ep:20K epch:67.62 loss:0.005 grdn:0.176 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:04:50 ts/train.py:232 step:491K smpl:4M ep:20K epch:67.65 loss:0.005 grdn:0.192 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:06:15 ts/train.py:232 step:491K smpl:4M ep:20K epch:67.67 loss:0.004 grdn:0.151 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:07:40 ts/train.py:232 step:492K smpl:4M ep:20K epch:67.70 loss:0.004 grdn:0.171 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:09:05 ts/train.py:232 step:492K smpl:4M ep:20K epch:67.73 loss:0.004 grdn:0.166 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:10:30 ts/train.py:232 step:492K smpl:4M ep:20K epch:67.76 loss:0.005 grdn:0.188 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:11:55 ts/train.py:232 step:492K smpl:4M ep:20K epch:67.78 loss:0.006 grdn:0.196 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:13:20 ts/train.py:232 step:492K smpl:4M ep:20K epch:67.81 loss:0.005 grdn:0.178 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:14:46 ts/train.py:232 step:493K smpl:4M ep:20K epch:67.84 loss:0.005 grdn:0.166 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:16:11 ts/train.py:232 step:493K smpl:4M ep:20K epch:67.87 loss:0.005 grdn:0.191 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:17:36 ts/train.py:232 step:493K smpl:4M ep:20K epch:67.89 loss:0.005 grdn:0.186 lr:4.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 00:19:01 ts/train.py:232 step:493K smpl:4M ep:20K epch:67.92 loss:0.005 grdn:0.168 lr:4.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:20:25 ts/train.py:232 step:493K smpl:4M ep:20K epch:67.95 loss:0.005 grdn:0.183 lr:4.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:21:50 ts/train.py:232 step:494K smpl:4M ep:20K epch:67.98 loss:0.005 grdn:0.183 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:23:15 ts/train.py:232 step:494K smpl:4M ep:20K epch:68.00 loss:0.005 grdn:0.182 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:24:40 ts/train.py:232 step:494K smpl:4M ep:20K epch:68.03 loss:0.005 grdn:0.178 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:26:05 ts/train.py:232 step:494K smpl:4M ep:20K epch:68.06 loss:0.005 grdn:0.176 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:27:30 ts/train.py:232 step:494K smpl:4M ep:20K epch:68.09 loss:0.005 grdn:0.181 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:28:55 ts/train.py:232 step:495K smpl:4M ep:20K epch:68.11 loss:0.005 grdn:0.185 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:30:20 ts/train.py:232 step:495K smpl:4M ep:20K epch:68.14 loss:0.004 grdn:0.163 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:31:44 ts/train.py:232 step:495K smpl:4M ep:20K epch:68.17 loss:0.005 grdn:0.183 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:33:09 ts/train.py:232 step:495K smpl:4M ep:20K epch:68.20 loss:0.004 grdn:0.166 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:34:34 ts/train.py:232 step:495K smpl:4M ep:20K epch:68.23 loss:0.005 grdn:0.180 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:35:59 ts/train.py:232 step:496K smpl:4M ep:20K epch:68.25 loss:0.004 grdn:0.166 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:37:24 ts/train.py:232 step:496K smpl:4M ep:20K epch:68.28 loss:0.005 grdn:0.188 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:38:49 ts/train.py:232 step:496K smpl:4M ep:20K epch:68.31 loss:0.004 grdn:0.172 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:40:14 ts/train.py:232 step:496K smpl:4M ep:21K epch:68.34 loss:0.005 grdn:0.195 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:41:39 ts/train.py:232 step:496K smpl:4M ep:21K epch:68.36 loss:0.005 grdn:0.185 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:43:04 ts/train.py:232 step:497K smpl:4M ep:21K epch:68.39 loss:0.005 grdn:0.184 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:44:28 ts/train.py:232 step:497K smpl:4M ep:21K epch:68.42 loss:0.005 grdn:0.186 lr:4.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-08 00:45:55 ts/train.py:232 step:497K smpl:4M ep:21K epch:68.45 loss:0.005 grdn:0.183 lr:4.2e-05 updt_s:0.422 data_s:0.008 +INFO 2025-05-08 00:47:20 ts/train.py:232 step:497K smpl:4M ep:21K epch:68.47 loss:0.005 grdn:0.182 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:48:45 ts/train.py:232 step:497K smpl:4M ep:21K epch:68.50 loss:0.006 grdn:0.194 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:50:09 ts/train.py:232 step:498K smpl:4M ep:21K epch:68.53 loss:0.004 grdn:0.175 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:51:34 ts/train.py:232 step:498K smpl:4M ep:21K epch:68.56 loss:0.005 grdn:0.184 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:52:59 ts/train.py:232 step:498K smpl:4M ep:21K epch:68.58 loss:0.005 grdn:0.170 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:54:24 ts/train.py:232 step:498K smpl:4M ep:21K epch:68.61 loss:0.005 grdn:0.187 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:55:49 ts/train.py:232 step:498K smpl:4M ep:21K epch:68.64 loss:0.005 grdn:0.190 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:57:14 ts/train.py:232 step:499K smpl:4M ep:21K epch:68.67 loss:0.005 grdn:0.182 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 00:58:39 ts/train.py:232 step:499K smpl:4M ep:21K epch:68.69 loss:0.005 grdn:0.183 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:00:04 ts/train.py:232 step:499K smpl:4M ep:21K epch:68.72 loss:0.005 grdn:0.173 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:01:29 ts/train.py:232 step:499K smpl:4M ep:21K epch:68.75 loss:0.005 grdn:0.173 lr:4.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:02:54 ts/train.py:232 step:499K smpl:4M ep:21K epch:68.78 loss:0.005 grdn:0.192 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:04:19 ts/train.py:232 step:500K smpl:4M ep:21K epch:68.80 loss:0.005 grdn:0.179 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:05:44 ts/train.py:232 step:500K smpl:4M ep:21K epch:68.83 loss:0.005 grdn:0.175 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:07:09 ts/train.py:232 step:500K smpl:4M ep:21K epch:68.86 loss:0.005 grdn:0.190 lr:4.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 01:08:34 ts/train.py:232 step:500K smpl:4M ep:21K epch:68.89 loss:0.005 grdn:0.193 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:09:59 ts/train.py:232 step:500K smpl:4M ep:21K epch:68.91 loss:0.005 grdn:0.164 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:11:23 ts/train.py:232 step:501K smpl:4M ep:21K epch:68.94 loss:0.004 grdn:0.177 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:12:49 ts/train.py:232 step:501K smpl:4M ep:21K epch:68.97 loss:0.005 grdn:0.179 lr:4.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 01:14:13 ts/train.py:232 step:501K smpl:4M ep:21K epch:69.00 loss:0.005 grdn:0.179 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:15:38 ts/train.py:232 step:501K smpl:4M ep:21K epch:69.02 loss:0.005 grdn:0.172 lr:4.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 01:17:03 ts/train.py:232 step:501K smpl:4M ep:21K epch:69.05 loss:0.005 grdn:0.184 lr:4.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 01:18:28 ts/train.py:232 step:502K smpl:4M ep:21K epch:69.08 loss:0.005 grdn:0.188 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:19:53 ts/train.py:232 step:502K smpl:4M ep:21K epch:69.11 loss:0.005 grdn:0.180 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:21:18 ts/train.py:232 step:502K smpl:4M ep:21K epch:69.13 loss:0.004 grdn:0.174 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:22:43 ts/train.py:232 step:502K smpl:4M ep:21K epch:69.16 loss:0.004 grdn:0.178 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:24:08 ts/train.py:232 step:502K smpl:4M ep:21K epch:69.19 loss:0.004 grdn:0.174 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:25:33 ts/train.py:232 step:503K smpl:4M ep:21K epch:69.22 loss:0.005 grdn:0.189 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:26:58 ts/train.py:232 step:503K smpl:4M ep:21K epch:69.24 loss:0.005 grdn:0.198 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:28:23 ts/train.py:232 step:503K smpl:4M ep:21K epch:69.27 loss:0.005 grdn:0.202 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:29:48 ts/train.py:232 step:503K smpl:4M ep:21K epch:69.30 loss:0.005 grdn:0.186 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:31:13 ts/train.py:232 step:503K smpl:4M ep:21K epch:69.33 loss:0.005 grdn:0.176 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:32:38 ts/train.py:232 step:504K smpl:4M ep:21K epch:69.35 loss:0.005 grdn:0.188 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:34:03 ts/train.py:232 step:504K smpl:4M ep:21K epch:69.38 loss:0.005 grdn:0.184 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:35:29 ts/train.py:232 step:504K smpl:4M ep:21K epch:69.41 loss:0.004 grdn:0.173 lr:4.1e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-08 01:36:54 ts/train.py:232 step:504K smpl:4M ep:21K epch:69.44 loss:0.005 grdn:0.184 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:38:19 ts/train.py:232 step:504K smpl:4M ep:21K epch:69.46 loss:0.005 grdn:0.176 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:39:44 ts/train.py:232 step:505K smpl:4M ep:21K epch:69.49 loss:0.004 grdn:0.182 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:41:09 ts/train.py:232 step:505K smpl:4M ep:21K epch:69.52 loss:0.005 grdn:0.192 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:42:34 ts/train.py:232 step:505K smpl:4M ep:21K epch:69.55 loss:0.005 grdn:0.182 lr:4.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:43:59 ts/train.py:232 step:505K smpl:4M ep:21K epch:69.57 loss:0.005 grdn:0.183 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:45:23 ts/train.py:232 step:505K smpl:4M ep:21K epch:69.60 loss:0.005 grdn:0.178 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:46:48 ts/train.py:232 step:506K smpl:4M ep:21K epch:69.63 loss:0.004 grdn:0.178 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:48:14 ts/train.py:232 step:506K smpl:4M ep:21K epch:69.66 loss:0.005 grdn:0.190 lr:4.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 01:49:39 ts/train.py:232 step:506K smpl:4M ep:21K epch:69.68 loss:0.005 grdn:0.187 lr:4.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 01:51:04 ts/train.py:232 step:506K smpl:4M ep:21K epch:69.71 loss:0.005 grdn:0.181 lr:4.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 01:52:29 ts/train.py:232 step:506K smpl:4M ep:21K epch:69.74 loss:0.005 grdn:0.185 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:53:54 ts/train.py:232 step:507K smpl:4M ep:21K epch:69.77 loss:0.005 grdn:0.191 lr:4.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 01:55:19 ts/train.py:232 step:507K smpl:4M ep:21K epch:69.80 loss:0.004 grdn:0.172 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:56:44 ts/train.py:232 step:507K smpl:4M ep:21K epch:69.82 loss:0.004 grdn:0.179 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 01:58:09 ts/train.py:232 step:507K smpl:4M ep:21K epch:69.85 loss:0.004 grdn:0.180 lr:4.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 01:59:34 ts/train.py:232 step:507K smpl:4M ep:21K epch:69.88 loss:0.005 grdn:0.182 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:00:58 ts/train.py:232 step:508K smpl:4M ep:21K epch:69.91 loss:0.005 grdn:0.187 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:02:23 ts/train.py:232 step:508K smpl:4M ep:21K epch:69.93 loss:0.005 grdn:0.183 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:03:48 ts/train.py:232 step:508K smpl:4M ep:21K epch:69.96 loss:0.004 grdn:0.168 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:05:13 ts/train.py:232 step:508K smpl:4M ep:21K epch:69.99 loss:0.004 grdn:0.171 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:06:38 ts/train.py:232 step:508K smpl:4M ep:21K epch:70.02 loss:0.004 grdn:0.170 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:08:03 ts/train.py:232 step:509K smpl:4M ep:21K epch:70.04 loss:0.005 grdn:0.193 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:09:28 ts/train.py:232 step:509K smpl:4M ep:21K epch:70.07 loss:0.004 grdn:0.172 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:10:53 ts/train.py:232 step:509K smpl:4M ep:21K epch:70.10 loss:0.004 grdn:0.175 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:12:18 ts/train.py:232 step:509K smpl:4M ep:21K epch:70.13 loss:0.005 grdn:0.186 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:13:43 ts/train.py:232 step:509K smpl:4M ep:21K epch:70.15 loss:0.005 grdn:0.170 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:15:08 ts/train.py:232 step:510K smpl:4M ep:21K epch:70.18 loss:0.005 grdn:0.177 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:16:33 ts/train.py:232 step:510K smpl:4M ep:21K epch:70.21 loss:0.005 grdn:0.188 lr:4.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 02:17:58 ts/train.py:232 step:510K smpl:4M ep:21K epch:70.24 loss:0.006 grdn:0.201 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:17:58 ts/train.py:241 Checkpoint policy after step 510000 +INFO 2025-05-08 02:19:26 ts/train.py:232 step:510K smpl:4M ep:21K epch:70.26 loss:0.005 grdn:0.186 lr:4.0e-05 updt_s:0.422 data_s:0.000 +INFO 2025-05-08 02:20:51 ts/train.py:232 step:510K smpl:4M ep:21K epch:70.29 loss:0.005 grdn:0.186 lr:4.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:22:15 ts/train.py:232 step:511K smpl:4M ep:21K epch:70.32 loss:0.005 grdn:0.179 lr:4.0e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:23:40 ts/train.py:232 step:511K smpl:4M ep:21K epch:70.35 loss:0.005 grdn:0.184 lr:4.0e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:25:06 ts/train.py:232 step:511K smpl:4M ep:21K epch:70.37 loss:0.004 grdn:0.174 lr:3.9e-05 updt_s:0.422 data_s:0.007 +INFO 2025-05-08 02:26:31 ts/train.py:232 step:511K smpl:4M ep:21K epch:70.40 loss:0.004 grdn:0.178 lr:3.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:27:56 ts/train.py:232 step:511K smpl:4M ep:21K epch:70.43 loss:0.005 grdn:0.180 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:29:21 ts/train.py:232 step:512K smpl:4M ep:21K epch:70.46 loss:0.005 grdn:0.197 lr:3.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-08 02:30:45 ts/train.py:232 step:512K smpl:4M ep:21K epch:70.48 loss:0.005 grdn:0.185 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:32:10 ts/train.py:232 step:512K smpl:4M ep:21K epch:70.51 loss:0.004 grdn:0.177 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:33:35 ts/train.py:232 step:512K smpl:4M ep:21K epch:70.54 loss:0.005 grdn:0.186 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:35:00 ts/train.py:232 step:512K smpl:4M ep:21K epch:70.57 loss:0.005 grdn:0.184 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:36:25 ts/train.py:232 step:513K smpl:4M ep:21K epch:70.59 loss:0.004 grdn:0.174 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:37:49 ts/train.py:232 step:513K smpl:4M ep:21K epch:70.62 loss:0.004 grdn:0.180 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:39:14 ts/train.py:232 step:513K smpl:4M ep:21K epch:70.65 loss:0.004 grdn:0.177 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:40:39 ts/train.py:232 step:513K smpl:4M ep:21K epch:70.68 loss:0.005 grdn:0.183 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:42:04 ts/train.py:232 step:513K smpl:4M ep:21K epch:70.70 loss:0.005 grdn:0.184 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:43:29 ts/train.py:232 step:514K smpl:4M ep:21K epch:70.73 loss:0.004 grdn:0.176 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:44:54 ts/train.py:232 step:514K smpl:4M ep:21K epch:70.76 loss:0.004 grdn:0.172 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:46:18 ts/train.py:232 step:514K smpl:4M ep:21K epch:70.79 loss:0.004 grdn:0.173 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:47:43 ts/train.py:232 step:514K smpl:4M ep:21K epch:70.81 loss:0.004 grdn:0.176 lr:3.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 02:49:08 ts/train.py:232 step:514K smpl:4M ep:21K epch:70.84 loss:0.004 grdn:0.177 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:50:33 ts/train.py:232 step:515K smpl:4M ep:21K epch:70.87 loss:0.004 grdn:0.172 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:51:58 ts/train.py:232 step:515K smpl:4M ep:21K epch:70.90 loss:0.004 grdn:0.174 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:53:23 ts/train.py:232 step:515K smpl:4M ep:21K epch:70.92 loss:0.005 grdn:0.179 lr:3.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 02:54:47 ts/train.py:232 step:515K smpl:4M ep:21K epch:70.95 loss:0.004 grdn:0.176 lr:3.9e-05 updt_s:0.422 data_s:0.000 +INFO 2025-05-08 02:56:12 ts/train.py:232 step:515K smpl:4M ep:21K epch:70.98 loss:0.005 grdn:0.181 lr:3.9e-05 updt_s:0.422 data_s:0.000 +INFO 2025-05-08 02:57:37 ts/train.py:232 step:516K smpl:4M ep:21K epch:71.01 loss:0.005 grdn:0.182 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 02:59:01 ts/train.py:232 step:516K smpl:4M ep:21K epch:71.03 loss:0.005 grdn:0.192 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 03:00:26 ts/train.py:232 step:516K smpl:4M ep:21K epch:71.06 loss:0.005 grdn:0.182 lr:3.9e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 03:01:51 ts/train.py:232 step:516K smpl:4M ep:21K epch:71.09 loss:0.005 grdn:0.198 lr:3.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 03:03:17 ts/train.py:232 step:516K smpl:4M ep:21K epch:71.12 loss:0.005 grdn:0.181 lr:3.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-08 03:04:42 ts/train.py:232 step:517K smpl:4M ep:21K epch:71.14 loss:0.005 grdn:0.203 lr:3.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-08 03:06:07 ts/train.py:232 step:517K smpl:4M ep:21K epch:71.17 loss:0.004 grdn:0.187 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:07:33 ts/train.py:232 step:517K smpl:4M ep:21K epch:71.20 loss:0.005 grdn:0.185 lr:3.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 03:08:58 ts/train.py:232 step:517K smpl:4M ep:21K epch:71.23 loss:0.004 grdn:0.172 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:10:23 ts/train.py:232 step:517K smpl:4M ep:21K epch:71.25 loss:0.005 grdn:0.183 lr:3.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 03:11:48 ts/train.py:232 step:518K smpl:4M ep:21K epch:71.28 loss:0.005 grdn:0.184 lr:3.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 03:13:13 ts/train.py:232 step:518K smpl:4M ep:21K epch:71.31 loss:0.005 grdn:0.186 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:14:40 ts/train.py:232 step:518K smpl:4M ep:21K epch:71.34 loss:0.004 grdn:0.169 lr:3.8e-05 updt_s:0.425 data_s:0.008 +INFO 2025-05-08 03:16:05 ts/train.py:232 step:518K smpl:4M ep:21K epch:71.37 loss:0.004 grdn:0.166 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:17:30 ts/train.py:232 step:518K smpl:4M ep:21K epch:71.39 loss:0.004 grdn:0.175 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:18:56 ts/train.py:232 step:519K smpl:4M ep:21K epch:71.42 loss:0.005 grdn:0.191 lr:3.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 03:20:21 ts/train.py:232 step:519K smpl:4M ep:21K epch:71.45 loss:0.005 grdn:0.186 lr:3.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 03:21:46 ts/train.py:232 step:519K smpl:4M ep:21K epch:71.48 loss:0.005 grdn:0.193 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:23:11 ts/train.py:232 step:519K smpl:4M ep:21K epch:71.50 loss:0.004 grdn:0.174 lr:3.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 03:24:36 ts/train.py:232 step:519K smpl:4M ep:21K epch:71.53 loss:0.004 grdn:0.180 lr:3.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 03:26:01 ts/train.py:232 step:520K smpl:4M ep:21K epch:71.56 loss:0.004 grdn:0.163 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:27:26 ts/train.py:232 step:520K smpl:4M ep:21K epch:71.59 loss:0.004 grdn:0.178 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:28:52 ts/train.py:232 step:520K smpl:4M ep:21K epch:71.61 loss:0.004 grdn:0.182 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:30:17 ts/train.py:232 step:520K smpl:4M ep:21K epch:71.64 loss:0.004 grdn:0.186 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:31:42 ts/train.py:232 step:520K smpl:4M ep:22K epch:71.67 loss:0.004 grdn:0.169 lr:3.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 03:33:07 ts/train.py:232 step:521K smpl:4M ep:22K epch:71.70 loss:0.005 grdn:0.184 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:34:33 ts/train.py:232 step:521K smpl:4M ep:22K epch:71.72 loss:0.005 grdn:0.192 lr:3.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 03:35:58 ts/train.py:232 step:521K smpl:4M ep:22K epch:71.75 loss:0.005 grdn:0.182 lr:3.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 03:37:23 ts/train.py:232 step:521K smpl:4M ep:22K epch:71.78 loss:0.004 grdn:0.180 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:38:48 ts/train.py:232 step:521K smpl:4M ep:22K epch:71.81 loss:0.004 grdn:0.199 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:40:14 ts/train.py:232 step:522K smpl:4M ep:22K epch:71.83 loss:0.004 grdn:0.183 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:41:39 ts/train.py:232 step:522K smpl:4M ep:22K epch:71.86 loss:0.004 grdn:0.187 lr:3.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 03:43:04 ts/train.py:232 step:522K smpl:4M ep:22K epch:71.89 loss:0.005 grdn:0.194 lr:3.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 03:44:29 ts/train.py:232 step:522K smpl:4M ep:22K epch:71.92 loss:0.004 grdn:0.171 lr:3.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 03:45:54 ts/train.py:232 step:522K smpl:4M ep:22K epch:71.94 loss:0.004 grdn:0.176 lr:3.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:47:20 ts/train.py:232 step:523K smpl:4M ep:22K epch:71.97 loss:0.005 grdn:0.193 lr:3.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 03:48:45 ts/train.py:232 step:523K smpl:4M ep:22K epch:72.00 loss:0.005 grdn:0.193 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:50:10 ts/train.py:232 step:523K smpl:4M ep:22K epch:72.03 loss:0.004 grdn:0.182 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 03:51:35 ts/train.py:232 step:523K smpl:4M ep:22K epch:72.05 loss:0.004 grdn:0.176 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:53:01 ts/train.py:232 step:523K smpl:4M ep:22K epch:72.08 loss:0.004 grdn:0.183 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:54:26 ts/train.py:232 step:524K smpl:4M ep:22K epch:72.11 loss:0.004 grdn:0.177 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:55:51 ts/train.py:232 step:524K smpl:4M ep:22K epch:72.14 loss:0.004 grdn:0.181 lr:3.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 03:57:16 ts/train.py:232 step:524K smpl:4M ep:22K epch:72.16 loss:0.004 grdn:0.170 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 03:58:41 ts/train.py:232 step:524K smpl:4M ep:22K epch:72.19 loss:0.004 grdn:0.169 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:00:07 ts/train.py:232 step:524K smpl:4M ep:22K epch:72.22 loss:0.004 grdn:0.175 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:01:32 ts/train.py:232 step:525K smpl:4M ep:22K epch:72.25 loss:0.004 grdn:0.188 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:02:57 ts/train.py:232 step:525K smpl:4M ep:22K epch:72.27 loss:0.004 grdn:0.175 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:04:24 ts/train.py:232 step:525K smpl:4M ep:22K epch:72.30 loss:0.005 grdn:0.192 lr:3.7e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-08 04:05:49 ts/train.py:232 step:525K smpl:4M ep:22K epch:72.33 loss:0.004 grdn:0.171 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:07:15 ts/train.py:232 step:525K smpl:4M ep:22K epch:72.36 loss:0.005 grdn:0.187 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:08:40 ts/train.py:232 step:526K smpl:4M ep:22K epch:72.38 loss:0.004 grdn:0.180 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:10:05 ts/train.py:232 step:526K smpl:4M ep:22K epch:72.41 loss:0.004 grdn:0.183 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:11:30 ts/train.py:232 step:526K smpl:4M ep:22K epch:72.44 loss:0.005 grdn:0.195 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:12:55 ts/train.py:232 step:526K smpl:4M ep:22K epch:72.47 loss:0.004 grdn:0.181 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:14:20 ts/train.py:232 step:526K smpl:4M ep:22K epch:72.49 loss:0.005 grdn:0.190 lr:3.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 04:15:45 ts/train.py:232 step:527K smpl:4M ep:22K epch:72.52 loss:0.004 grdn:0.178 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:17:11 ts/train.py:232 step:527K smpl:4M ep:22K epch:72.55 loss:0.004 grdn:0.174 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:18:36 ts/train.py:232 step:527K smpl:4M ep:22K epch:72.58 loss:0.005 grdn:0.188 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:20:01 ts/train.py:232 step:527K smpl:4M ep:22K epch:72.60 loss:0.005 grdn:0.207 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:21:26 ts/train.py:232 step:527K smpl:4M ep:22K epch:72.63 loss:0.005 grdn:0.201 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:22:52 ts/train.py:232 step:528K smpl:4M ep:22K epch:72.66 loss:0.004 grdn:0.176 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:24:17 ts/train.py:232 step:528K smpl:4M ep:22K epch:72.69 loss:0.005 grdn:0.185 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:25:42 ts/train.py:232 step:528K smpl:4M ep:22K epch:72.71 loss:0.004 grdn:0.176 lr:3.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:27:08 ts/train.py:232 step:528K smpl:4M ep:22K epch:72.74 loss:0.005 grdn:0.183 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:28:33 ts/train.py:232 step:528K smpl:4M ep:22K epch:72.77 loss:0.004 grdn:0.178 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:29:58 ts/train.py:232 step:529K smpl:4M ep:22K epch:72.80 loss:0.004 grdn:0.182 lr:3.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:31:23 ts/train.py:232 step:529K smpl:4M ep:22K epch:72.82 loss:0.004 grdn:0.181 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:32:48 ts/train.py:232 step:529K smpl:4M ep:22K epch:72.85 loss:0.004 grdn:0.181 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:34:13 ts/train.py:232 step:529K smpl:4M ep:22K epch:72.88 loss:0.004 grdn:0.183 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:35:38 ts/train.py:232 step:529K smpl:4M ep:22K epch:72.91 loss:0.004 grdn:0.177 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:37:04 ts/train.py:232 step:530K smpl:4M ep:22K epch:72.94 loss:0.004 grdn:0.174 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:38:29 ts/train.py:232 step:530K smpl:4M ep:22K epch:72.96 loss:0.004 grdn:0.159 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:39:54 ts/train.py:232 step:530K smpl:4M ep:22K epch:72.99 loss:0.004 grdn:0.171 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:41:19 ts/train.py:232 step:530K smpl:4M ep:22K epch:73.02 loss:0.004 grdn:0.188 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:42:44 ts/train.py:232 step:530K smpl:4M ep:22K epch:73.05 loss:0.004 grdn:0.172 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:44:09 ts/train.py:232 step:531K smpl:4M ep:22K epch:73.07 loss:0.005 grdn:0.189 lr:3.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 04:45:34 ts/train.py:232 step:531K smpl:4M ep:22K epch:73.10 loss:0.004 grdn:0.182 lr:3.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 04:47:00 ts/train.py:232 step:531K smpl:4M ep:22K epch:73.13 loss:0.004 grdn:0.176 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:48:25 ts/train.py:232 step:531K smpl:4M ep:22K epch:73.16 loss:0.004 grdn:0.179 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:49:50 ts/train.py:232 step:531K smpl:4M ep:22K epch:73.18 loss:0.004 grdn:0.177 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:51:15 ts/train.py:232 step:532K smpl:4M ep:22K epch:73.21 loss:0.004 grdn:0.169 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:52:40 ts/train.py:232 step:532K smpl:4M ep:22K epch:73.24 loss:0.004 grdn:0.175 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:54:07 ts/train.py:232 step:532K smpl:4M ep:22K epch:73.27 loss:0.004 grdn:0.186 lr:3.6e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-08 04:55:33 ts/train.py:232 step:532K smpl:4M ep:22K epch:73.29 loss:0.005 grdn:0.190 lr:3.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:56:58 ts/train.py:232 step:532K smpl:4M ep:22K epch:73.32 loss:0.005 grdn:0.192 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 04:58:23 ts/train.py:232 step:533K smpl:4M ep:22K epch:73.35 loss:0.005 grdn:0.199 lr:3.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 04:59:48 ts/train.py:232 step:533K smpl:4M ep:22K epch:73.38 loss:0.004 grdn:0.179 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:01:14 ts/train.py:232 step:533K smpl:4M ep:22K epch:73.40 loss:0.005 grdn:0.193 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:02:39 ts/train.py:232 step:533K smpl:4M ep:22K epch:73.43 loss:0.005 grdn:0.183 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:04:04 ts/train.py:232 step:533K smpl:4M ep:22K epch:73.46 loss:0.004 grdn:0.167 lr:3.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:05:29 ts/train.py:232 step:534K smpl:4M ep:22K epch:73.49 loss:0.004 grdn:0.164 lr:3.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:06:55 ts/train.py:232 step:534K smpl:4M ep:22K epch:73.51 loss:0.005 grdn:0.186 lr:3.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:08:20 ts/train.py:232 step:534K smpl:4M ep:22K epch:73.54 loss:0.004 grdn:0.170 lr:3.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:09:45 ts/train.py:232 step:534K smpl:4M ep:22K epch:73.57 loss:0.005 grdn:0.193 lr:3.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:11:11 ts/train.py:232 step:534K smpl:4M ep:22K epch:73.60 loss:0.004 grdn:0.175 lr:3.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:12:36 ts/train.py:232 step:535K smpl:4M ep:22K epch:73.62 loss:0.004 grdn:0.186 lr:3.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:14:01 ts/train.py:232 step:535K smpl:4M ep:22K epch:73.65 loss:0.004 grdn:0.180 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:15:26 ts/train.py:232 step:535K smpl:4M ep:22K epch:73.68 loss:0.005 grdn:0.183 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:16:52 ts/train.py:232 step:535K smpl:4M ep:22K epch:73.71 loss:0.004 grdn:0.182 lr:3.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:18:17 ts/train.py:232 step:535K smpl:4M ep:22K epch:73.73 loss:0.005 grdn:0.192 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:19:42 ts/train.py:232 step:536K smpl:4M ep:22K epch:73.76 loss:0.004 grdn:0.189 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:21:07 ts/train.py:232 step:536K smpl:4M ep:22K epch:73.79 loss:0.005 grdn:0.192 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:22:33 ts/train.py:232 step:536K smpl:4M ep:22K epch:73.82 loss:0.005 grdn:0.182 lr:3.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:23:58 ts/train.py:232 step:536K smpl:4M ep:22K epch:73.84 loss:0.004 grdn:0.172 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:25:23 ts/train.py:232 step:536K smpl:4M ep:22K epch:73.87 loss:0.005 grdn:0.195 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:26:48 ts/train.py:232 step:537K smpl:4M ep:22K epch:73.90 loss:0.004 grdn:0.169 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:28:13 ts/train.py:232 step:537K smpl:4M ep:22K epch:73.93 loss:0.004 grdn:0.172 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:29:39 ts/train.py:232 step:537K smpl:4M ep:22K epch:73.95 loss:0.005 grdn:0.196 lr:3.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:31:04 ts/train.py:232 step:537K smpl:4M ep:22K epch:73.98 loss:0.005 grdn:0.195 lr:3.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:32:29 ts/train.py:232 step:537K smpl:4M ep:22K epch:74.01 loss:0.004 grdn:0.173 lr:3.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:33:55 ts/train.py:232 step:538K smpl:4M ep:22K epch:74.04 loss:0.004 grdn:0.175 lr:3.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:35:20 ts/train.py:232 step:538K smpl:4M ep:22K epch:74.06 loss:0.004 grdn:0.188 lr:3.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 05:36:46 ts/train.py:232 step:538K smpl:4M ep:22K epch:74.09 loss:0.004 grdn:0.179 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:38:11 ts/train.py:232 step:538K smpl:4M ep:22K epch:74.12 loss:0.004 grdn:0.172 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:39:36 ts/train.py:232 step:538K smpl:4M ep:22K epch:74.15 loss:0.005 grdn:0.196 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:41:01 ts/train.py:232 step:539K smpl:4M ep:22K epch:74.17 loss:0.005 grdn:0.195 lr:3.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 05:42:26 ts/train.py:232 step:539K smpl:4M ep:22K epch:74.20 loss:0.004 grdn:0.172 lr:3.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 05:43:53 ts/train.py:232 step:539K smpl:4M ep:22K epch:74.23 loss:0.004 grdn:0.174 lr:3.5e-05 updt_s:0.424 data_s:0.007 +INFO 2025-05-08 05:45:18 ts/train.py:232 step:539K smpl:4M ep:22K epch:74.26 loss:0.004 grdn:0.182 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:46:43 ts/train.py:232 step:539K smpl:4M ep:22K epch:74.28 loss:0.004 grdn:0.183 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:48:08 ts/train.py:232 step:540K smpl:4M ep:22K epch:74.31 loss:0.005 grdn:0.194 lr:3.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 05:49:33 ts/train.py:232 step:540K smpl:4M ep:22K epch:74.34 loss:0.005 grdn:0.195 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:50:58 ts/train.py:232 step:540K smpl:4M ep:22K epch:74.37 loss:0.004 grdn:0.171 lr:3.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:50:58 ts/train.py:241 Checkpoint policy after step 540000 +INFO 2025-05-08 05:52:27 ts/train.py:232 step:540K smpl:4M ep:22K epch:74.39 loss:0.004 grdn:0.187 lr:3.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 05:53:52 ts/train.py:232 step:540K smpl:4M ep:22K epch:74.42 loss:0.004 grdn:0.173 lr:3.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 05:55:17 ts/train.py:232 step:541K smpl:4M ep:22K epch:74.45 loss:0.004 grdn:0.198 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 05:56:42 ts/train.py:232 step:541K smpl:4M ep:22K epch:74.48 loss:0.004 grdn:0.163 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 05:58:07 ts/train.py:232 step:541K smpl:4M ep:22K epch:74.51 loss:0.004 grdn:0.181 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 05:59:32 ts/train.py:232 step:541K smpl:4M ep:22K epch:74.53 loss:0.004 grdn:0.166 lr:3.4e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 06:00:58 ts/train.py:232 step:541K smpl:4M ep:22K epch:74.56 loss:0.004 grdn:0.172 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:02:23 ts/train.py:232 step:542K smpl:4M ep:22K epch:74.59 loss:0.004 grdn:0.163 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:03:48 ts/train.py:232 step:542K smpl:4M ep:22K epch:74.62 loss:0.004 grdn:0.185 lr:3.4e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 06:05:13 ts/train.py:232 step:542K smpl:4M ep:22K epch:74.64 loss:0.004 grdn:0.175 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:06:38 ts/train.py:232 step:542K smpl:4M ep:22K epch:74.67 loss:0.004 grdn:0.183 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:08:04 ts/train.py:232 step:542K smpl:4M ep:22K epch:74.70 loss:0.004 grdn:0.165 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:09:29 ts/train.py:232 step:543K smpl:4M ep:22K epch:74.73 loss:0.004 grdn:0.178 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:10:54 ts/train.py:232 step:543K smpl:4M ep:22K epch:74.75 loss:0.004 grdn:0.187 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:12:19 ts/train.py:232 step:543K smpl:4M ep:22K epch:74.78 loss:0.005 grdn:0.188 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:13:44 ts/train.py:232 step:543K smpl:4M ep:22K epch:74.81 loss:0.004 grdn:0.166 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:15:09 ts/train.py:232 step:543K smpl:4M ep:22K epch:74.84 loss:0.004 grdn:0.179 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:16:34 ts/train.py:232 step:544K smpl:4M ep:22K epch:74.86 loss:0.004 grdn:0.177 lr:3.4e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 06:17:59 ts/train.py:232 step:544K smpl:4M ep:22K epch:74.89 loss:0.004 grdn:0.173 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:19:24 ts/train.py:232 step:544K smpl:4M ep:22K epch:74.92 loss:0.005 grdn:0.184 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:20:49 ts/train.py:232 step:544K smpl:4M ep:22K epch:74.95 loss:0.004 grdn:0.175 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:22:14 ts/train.py:232 step:544K smpl:4M ep:22K epch:74.97 loss:0.004 grdn:0.185 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:23:40 ts/train.py:232 step:545K smpl:4M ep:23K epch:75.00 loss:0.004 grdn:0.167 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:25:05 ts/train.py:232 step:545K smpl:4M ep:23K epch:75.03 loss:0.004 grdn:0.168 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:26:30 ts/train.py:232 step:545K smpl:4M ep:23K epch:75.06 loss:0.004 grdn:0.170 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:27:55 ts/train.py:232 step:545K smpl:4M ep:23K epch:75.08 loss:0.005 grdn:0.197 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:29:20 ts/train.py:232 step:545K smpl:4M ep:23K epch:75.11 loss:0.005 grdn:0.191 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:30:45 ts/train.py:232 step:546K smpl:4M ep:23K epch:75.14 loss:0.005 grdn:0.189 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:32:10 ts/train.py:232 step:546K smpl:4M ep:23K epch:75.17 loss:0.004 grdn:0.176 lr:3.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:33:36 ts/train.py:232 step:546K smpl:4M ep:23K epch:75.19 loss:0.004 grdn:0.176 lr:3.4e-05 updt_s:0.424 data_s:0.007 +INFO 2025-05-08 06:35:01 ts/train.py:232 step:546K smpl:4M ep:23K epch:75.22 loss:0.004 grdn:0.181 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:36:27 ts/train.py:232 step:546K smpl:4M ep:23K epch:75.25 loss:0.004 grdn:0.186 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:37:52 ts/train.py:232 step:547K smpl:4M ep:23K epch:75.28 loss:0.004 grdn:0.191 lr:3.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:39:17 ts/train.py:232 step:547K smpl:4M ep:23K epch:75.30 loss:0.004 grdn:0.174 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:40:42 ts/train.py:232 step:547K smpl:4M ep:23K epch:75.33 loss:0.004 grdn:0.173 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:42:07 ts/train.py:232 step:547K smpl:4M ep:23K epch:75.36 loss:0.004 grdn:0.180 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:43:33 ts/train.py:232 step:547K smpl:4M ep:23K epch:75.39 loss:0.005 grdn:0.193 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:44:58 ts/train.py:232 step:548K smpl:4M ep:23K epch:75.41 loss:0.005 grdn:0.195 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:46:23 ts/train.py:232 step:548K smpl:4M ep:23K epch:75.44 loss:0.004 grdn:0.179 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:47:48 ts/train.py:232 step:548K smpl:4M ep:23K epch:75.47 loss:0.005 grdn:0.199 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:49:13 ts/train.py:232 step:548K smpl:4M ep:23K epch:75.50 loss:0.004 grdn:0.187 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:50:39 ts/train.py:232 step:548K smpl:4M ep:23K epch:75.52 loss:0.004 grdn:0.185 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:52:04 ts/train.py:232 step:549K smpl:4M ep:23K epch:75.55 loss:0.003 grdn:0.162 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:53:29 ts/train.py:232 step:549K smpl:4M ep:23K epch:75.58 loss:0.004 grdn:0.197 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:54:54 ts/train.py:232 step:549K smpl:4M ep:23K epch:75.61 loss:0.004 grdn:0.188 lr:3.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:56:19 ts/train.py:232 step:549K smpl:4M ep:23K epch:75.63 loss:0.004 grdn:0.174 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 06:57:44 ts/train.py:232 step:549K smpl:4M ep:23K epch:75.66 loss:0.004 grdn:0.180 lr:3.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 06:59:09 ts/train.py:232 step:550K smpl:4M ep:23K epch:75.69 loss:0.004 grdn:0.184 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:00:34 ts/train.py:232 step:550K smpl:4M ep:23K epch:75.72 loss:0.004 grdn:0.195 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:01:59 ts/train.py:232 step:550K smpl:4M ep:23K epch:75.74 loss:0.004 grdn:0.178 lr:3.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:03:24 ts/train.py:232 step:550K smpl:4M ep:23K epch:75.77 loss:0.003 grdn:0.168 lr:3.3e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 07:04:49 ts/train.py:232 step:550K smpl:4M ep:23K epch:75.80 loss:0.004 grdn:0.183 lr:3.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:06:14 ts/train.py:232 step:551K smpl:4M ep:23K epch:75.83 loss:0.003 grdn:0.166 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:07:39 ts/train.py:232 step:551K smpl:4M ep:23K epch:75.85 loss:0.004 grdn:0.182 lr:3.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:09:05 ts/train.py:232 step:551K smpl:4M ep:23K epch:75.88 loss:0.005 grdn:0.199 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:10:30 ts/train.py:232 step:551K smpl:4M ep:23K epch:75.91 loss:0.004 grdn:0.192 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:11:55 ts/train.py:232 step:551K smpl:4M ep:23K epch:75.94 loss:0.004 grdn:0.181 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:13:21 ts/train.py:232 step:552K smpl:4M ep:23K epch:75.96 loss:0.005 grdn:0.193 lr:3.3e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 07:14:46 ts/train.py:232 step:552K smpl:4M ep:23K epch:75.99 loss:0.004 grdn:0.177 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:16:11 ts/train.py:232 step:552K smpl:4M ep:23K epch:76.02 loss:0.004 grdn:0.182 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:17:36 ts/train.py:232 step:552K smpl:4M ep:23K epch:76.05 loss:0.004 grdn:0.181 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:19:01 ts/train.py:232 step:552K smpl:4M ep:23K epch:76.08 loss:0.004 grdn:0.179 lr:3.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:20:26 ts/train.py:232 step:553K smpl:4M ep:23K epch:76.10 loss:0.004 grdn:0.199 lr:3.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:21:52 ts/train.py:232 step:553K smpl:4M ep:23K epch:76.13 loss:0.005 grdn:0.193 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:23:18 ts/train.py:232 step:553K smpl:4M ep:23K epch:76.16 loss:0.004 grdn:0.186 lr:3.2e-05 updt_s:0.424 data_s:0.008 +INFO 2025-05-08 07:24:43 ts/train.py:232 step:553K smpl:4M ep:23K epch:76.19 loss:0.004 grdn:0.171 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:26:09 ts/train.py:232 step:553K smpl:4M ep:23K epch:76.21 loss:0.004 grdn:0.183 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:27:34 ts/train.py:232 step:554K smpl:4M ep:23K epch:76.24 loss:0.004 grdn:0.179 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:28:59 ts/train.py:232 step:554K smpl:4M ep:23K epch:76.27 loss:0.005 grdn:0.196 lr:3.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:30:24 ts/train.py:232 step:554K smpl:4M ep:23K epch:76.30 loss:0.005 grdn:0.196 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:31:49 ts/train.py:232 step:554K smpl:4M ep:23K epch:76.32 loss:0.004 grdn:0.175 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:33:15 ts/train.py:232 step:554K smpl:4M ep:23K epch:76.35 loss:0.004 grdn:0.177 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:34:39 ts/train.py:232 step:555K smpl:4M ep:23K epch:76.38 loss:0.004 grdn:0.184 lr:3.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:36:05 ts/train.py:232 step:555K smpl:4M ep:23K epch:76.41 loss:0.004 grdn:0.186 lr:3.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:37:30 ts/train.py:232 step:555K smpl:4M ep:23K epch:76.43 loss:0.004 grdn:0.189 lr:3.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:38:55 ts/train.py:232 step:555K smpl:4M ep:23K epch:76.46 loss:0.004 grdn:0.186 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:40:20 ts/train.py:232 step:555K smpl:4M ep:23K epch:76.49 loss:0.004 grdn:0.185 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:41:45 ts/train.py:232 step:556K smpl:4M ep:23K epch:76.52 loss:0.004 grdn:0.169 lr:3.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:43:10 ts/train.py:232 step:556K smpl:4M ep:23K epch:76.54 loss:0.004 grdn:0.181 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:44:35 ts/train.py:232 step:556K smpl:4M ep:23K epch:76.57 loss:0.004 grdn:0.173 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:46:00 ts/train.py:232 step:556K smpl:4M ep:23K epch:76.60 loss:0.004 grdn:0.182 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:47:25 ts/train.py:232 step:556K smpl:4M ep:23K epch:76.63 loss:0.004 grdn:0.172 lr:3.2e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 07:48:50 ts/train.py:232 step:557K smpl:4M ep:23K epch:76.65 loss:0.004 grdn:0.178 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:50:16 ts/train.py:232 step:557K smpl:4M ep:23K epch:76.68 loss:0.004 grdn:0.181 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:51:41 ts/train.py:232 step:557K smpl:4M ep:23K epch:76.71 loss:0.003 grdn:0.164 lr:3.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:53:06 ts/train.py:232 step:557K smpl:4M ep:23K epch:76.74 loss:0.004 grdn:0.169 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:54:31 ts/train.py:232 step:557K smpl:4M ep:23K epch:76.76 loss:0.004 grdn:0.183 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 07:55:56 ts/train.py:232 step:558K smpl:4M ep:23K epch:76.79 loss:0.004 grdn:0.181 lr:3.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:57:21 ts/train.py:232 step:558K smpl:4M ep:23K epch:76.82 loss:0.005 grdn:0.192 lr:3.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 07:58:46 ts/train.py:232 step:558K smpl:4M ep:23K epch:76.85 loss:0.004 grdn:0.190 lr:3.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 08:00:11 ts/train.py:232 step:558K smpl:4M ep:23K epch:76.87 loss:0.005 grdn:0.192 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:01:36 ts/train.py:232 step:558K smpl:4M ep:23K epch:76.90 loss:0.004 grdn:0.180 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:03:02 ts/train.py:232 step:559K smpl:4M ep:23K epch:76.93 loss:0.004 grdn:0.179 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:04:27 ts/train.py:232 step:559K smpl:4M ep:23K epch:76.96 loss:0.004 grdn:0.184 lr:3.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:05:52 ts/train.py:232 step:559K smpl:4M ep:23K epch:76.98 loss:0.005 grdn:0.197 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:07:17 ts/train.py:232 step:559K smpl:4M ep:23K epch:77.01 loss:0.004 grdn:0.191 lr:3.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 08:08:43 ts/train.py:232 step:559K smpl:4M ep:23K epch:77.04 loss:0.004 grdn:0.182 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:10:08 ts/train.py:232 step:560K smpl:4M ep:23K epch:77.07 loss:0.004 grdn:0.181 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:11:33 ts/train.py:232 step:560K smpl:4M ep:23K epch:77.09 loss:0.003 grdn:0.164 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:12:59 ts/train.py:232 step:560K smpl:4M ep:23K epch:77.12 loss:0.004 grdn:0.188 lr:3.1e-05 updt_s:0.424 data_s:0.006 +INFO 2025-05-08 08:14:24 ts/train.py:232 step:560K smpl:4M ep:23K epch:77.15 loss:0.004 grdn:0.172 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:15:50 ts/train.py:232 step:560K smpl:4M ep:23K epch:77.18 loss:0.004 grdn:0.170 lr:3.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 08:17:15 ts/train.py:232 step:561K smpl:4M ep:23K epch:77.20 loss:0.004 grdn:0.181 lr:3.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 08:18:40 ts/train.py:232 step:561K smpl:4M ep:23K epch:77.23 loss:0.004 grdn:0.198 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:20:06 ts/train.py:232 step:561K smpl:4M ep:23K epch:77.26 loss:0.005 grdn:0.200 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:21:31 ts/train.py:232 step:561K smpl:4M ep:23K epch:77.29 loss:0.004 grdn:0.168 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:22:56 ts/train.py:232 step:561K smpl:4M ep:23K epch:77.31 loss:0.004 grdn:0.178 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:24:21 ts/train.py:232 step:562K smpl:4M ep:23K epch:77.34 loss:0.004 grdn:0.201 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:25:46 ts/train.py:232 step:562K smpl:4M ep:23K epch:77.37 loss:0.005 grdn:0.207 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:27:12 ts/train.py:232 step:562K smpl:4M ep:23K epch:77.40 loss:0.004 grdn:0.179 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:28:37 ts/train.py:232 step:562K smpl:4M ep:23K epch:77.42 loss:0.004 grdn:0.167 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:30:02 ts/train.py:232 step:562K smpl:4M ep:23K epch:77.45 loss:0.004 grdn:0.172 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:31:28 ts/train.py:232 step:563K smpl:5M ep:23K epch:77.48 loss:0.003 grdn:0.164 lr:3.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 08:32:53 ts/train.py:232 step:563K smpl:5M ep:23K epch:77.51 loss:0.004 grdn:0.174 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:34:18 ts/train.py:232 step:563K smpl:5M ep:23K epch:77.53 loss:0.004 grdn:0.176 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:35:43 ts/train.py:232 step:563K smpl:5M ep:23K epch:77.56 loss:0.004 grdn:0.173 lr:3.1e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 08:37:08 ts/train.py:232 step:563K smpl:5M ep:23K epch:77.59 loss:0.004 grdn:0.187 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:38:33 ts/train.py:232 step:564K smpl:5M ep:23K epch:77.62 loss:0.004 grdn:0.178 lr:3.1e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 08:39:59 ts/train.py:232 step:564K smpl:5M ep:23K epch:77.65 loss:0.004 grdn:0.173 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:41:24 ts/train.py:232 step:564K smpl:5M ep:23K epch:77.67 loss:0.004 grdn:0.180 lr:3.1e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 08:42:49 ts/train.py:232 step:564K smpl:5M ep:23K epch:77.70 loss:0.004 grdn:0.179 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:44:14 ts/train.py:232 step:564K smpl:5M ep:23K epch:77.73 loss:0.004 grdn:0.183 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:45:40 ts/train.py:232 step:565K smpl:5M ep:23K epch:77.76 loss:0.004 grdn:0.186 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:47:05 ts/train.py:232 step:565K smpl:5M ep:23K epch:77.78 loss:0.004 grdn:0.183 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:48:30 ts/train.py:232 step:565K smpl:5M ep:23K epch:77.81 loss:0.003 grdn:0.164 lr:3.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:49:55 ts/train.py:232 step:565K smpl:5M ep:23K epch:77.84 loss:0.004 grdn:0.179 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:51:20 ts/train.py:232 step:565K smpl:5M ep:23K epch:77.87 loss:0.004 grdn:0.181 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:52:46 ts/train.py:232 step:566K smpl:5M ep:23K epch:77.89 loss:0.004 grdn:0.193 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:54:11 ts/train.py:232 step:566K smpl:5M ep:23K epch:77.92 loss:0.004 grdn:0.185 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:55:36 ts/train.py:232 step:566K smpl:5M ep:23K epch:77.95 loss:0.004 grdn:0.165 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:57:01 ts/train.py:232 step:566K smpl:5M ep:23K epch:77.98 loss:0.004 grdn:0.172 lr:3.0e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 08:58:26 ts/train.py:232 step:566K smpl:5M ep:23K epch:78.00 loss:0.004 grdn:0.188 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 08:59:51 ts/train.py:232 step:567K smpl:5M ep:23K epch:78.03 loss:0.004 grdn:0.175 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:01:16 ts/train.py:232 step:567K smpl:5M ep:23K epch:78.06 loss:0.004 grdn:0.187 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:02:43 ts/train.py:232 step:567K smpl:5M ep:23K epch:78.09 loss:0.004 grdn:0.179 lr:3.0e-05 updt_s:0.425 data_s:0.006 +INFO 2025-05-08 09:04:08 ts/train.py:232 step:567K smpl:5M ep:23K epch:78.11 loss:0.004 grdn:0.169 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:05:33 ts/train.py:232 step:567K smpl:5M ep:23K epch:78.14 loss:0.005 grdn:0.207 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:06:58 ts/train.py:232 step:568K smpl:5M ep:23K epch:78.17 loss:0.004 grdn:0.191 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:08:23 ts/train.py:232 step:568K smpl:5M ep:23K epch:78.20 loss:0.004 grdn:0.177 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:09:49 ts/train.py:232 step:568K smpl:5M ep:23K epch:78.22 loss:0.004 grdn:0.187 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:11:14 ts/train.py:232 step:568K smpl:5M ep:23K epch:78.25 loss:0.004 grdn:0.188 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:12:39 ts/train.py:232 step:568K smpl:5M ep:23K epch:78.28 loss:0.004 grdn:0.188 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:14:04 ts/train.py:232 step:569K smpl:5M ep:23K epch:78.31 loss:0.004 grdn:0.177 lr:3.0e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 09:15:29 ts/train.py:232 step:569K smpl:5M ep:24K epch:78.33 loss:0.004 grdn:0.170 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:16:54 ts/train.py:232 step:569K smpl:5M ep:24K epch:78.36 loss:0.004 grdn:0.183 lr:3.0e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 09:18:19 ts/train.py:232 step:569K smpl:5M ep:24K epch:78.39 loss:0.004 grdn:0.198 lr:3.0e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 09:19:44 ts/train.py:232 step:569K smpl:5M ep:24K epch:78.42 loss:0.005 grdn:0.206 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:21:09 ts/train.py:232 step:570K smpl:5M ep:24K epch:78.44 loss:0.004 grdn:0.183 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:22:35 ts/train.py:232 step:570K smpl:5M ep:24K epch:78.47 loss:0.004 grdn:0.189 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:24:00 ts/train.py:232 step:570K smpl:5M ep:24K epch:78.50 loss:0.003 grdn:0.164 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:24:00 ts/train.py:241 Checkpoint policy after step 570000 +INFO 2025-05-08 09:25:28 ts/train.py:232 step:570K smpl:5M ep:24K epch:78.53 loss:0.004 grdn:0.183 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:26:53 ts/train.py:232 step:570K smpl:5M ep:24K epch:78.55 loss:0.004 grdn:0.172 lr:3.0e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 09:28:18 ts/train.py:232 step:571K smpl:5M ep:24K epch:78.58 loss:0.004 grdn:0.182 lr:3.0e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 09:29:43 ts/train.py:232 step:571K smpl:5M ep:24K epch:78.61 loss:0.004 grdn:0.177 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:31:08 ts/train.py:232 step:571K smpl:5M ep:24K epch:78.64 loss:0.003 grdn:0.163 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:32:34 ts/train.py:232 step:571K smpl:5M ep:24K epch:78.66 loss:0.004 grdn:0.186 lr:3.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:33:59 ts/train.py:232 step:571K smpl:5M ep:24K epch:78.69 loss:0.004 grdn:0.175 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:35:24 ts/train.py:232 step:572K smpl:5M ep:24K epch:78.72 loss:0.004 grdn:0.198 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:36:49 ts/train.py:232 step:572K smpl:5M ep:24K epch:78.75 loss:0.004 grdn:0.191 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:38:14 ts/train.py:232 step:572K smpl:5M ep:24K epch:78.77 loss:0.004 grdn:0.185 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:39:40 ts/train.py:232 step:572K smpl:5M ep:24K epch:78.80 loss:0.004 grdn:0.177 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:41:05 ts/train.py:232 step:572K smpl:5M ep:24K epch:78.83 loss:0.004 grdn:0.173 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:42:30 ts/train.py:232 step:573K smpl:5M ep:24K epch:78.86 loss:0.004 grdn:0.183 lr:2.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 09:43:55 ts/train.py:232 step:573K smpl:5M ep:24K epch:78.88 loss:0.004 grdn:0.173 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:45:20 ts/train.py:232 step:573K smpl:5M ep:24K epch:78.91 loss:0.004 grdn:0.174 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:46:45 ts/train.py:232 step:573K smpl:5M ep:24K epch:78.94 loss:0.004 grdn:0.180 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:48:10 ts/train.py:232 step:573K smpl:5M ep:24K epch:78.97 loss:0.004 grdn:0.183 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:49:36 ts/train.py:232 step:574K smpl:5M ep:24K epch:78.99 loss:0.004 grdn:0.192 lr:2.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 09:51:01 ts/train.py:232 step:574K smpl:5M ep:24K epch:79.02 loss:0.004 grdn:0.184 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:52:27 ts/train.py:232 step:574K smpl:5M ep:24K epch:79.05 loss:0.005 grdn:0.206 lr:2.9e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-08 09:53:52 ts/train.py:232 step:574K smpl:5M ep:24K epch:79.08 loss:0.004 grdn:0.164 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:55:17 ts/train.py:232 step:574K smpl:5M ep:24K epch:79.10 loss:0.005 grdn:0.213 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:56:43 ts/train.py:232 step:575K smpl:5M ep:24K epch:79.13 loss:0.004 grdn:0.174 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 09:58:08 ts/train.py:232 step:575K smpl:5M ep:24K epch:79.16 loss:0.004 grdn:0.176 lr:2.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 09:59:33 ts/train.py:232 step:575K smpl:5M ep:24K epch:79.19 loss:0.004 grdn:0.188 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:00:58 ts/train.py:232 step:575K smpl:5M ep:24K epch:79.22 loss:0.004 grdn:0.183 lr:2.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 10:02:23 ts/train.py:232 step:575K smpl:5M ep:24K epch:79.24 loss:0.003 grdn:0.179 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:03:48 ts/train.py:232 step:576K smpl:5M ep:24K epch:79.27 loss:0.003 grdn:0.166 lr:2.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 10:05:14 ts/train.py:232 step:576K smpl:5M ep:24K epch:79.30 loss:0.003 grdn:0.171 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:06:39 ts/train.py:232 step:576K smpl:5M ep:24K epch:79.33 loss:0.004 grdn:0.187 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:08:04 ts/train.py:232 step:576K smpl:5M ep:24K epch:79.35 loss:0.004 grdn:0.185 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:09:29 ts/train.py:232 step:576K smpl:5M ep:24K epch:79.38 loss:0.004 grdn:0.198 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:10:55 ts/train.py:232 step:577K smpl:5M ep:24K epch:79.41 loss:0.004 grdn:0.183 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:12:20 ts/train.py:232 step:577K smpl:5M ep:24K epch:79.44 loss:0.004 grdn:0.180 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:13:45 ts/train.py:232 step:577K smpl:5M ep:24K epch:79.46 loss:0.004 grdn:0.177 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:15:10 ts/train.py:232 step:577K smpl:5M ep:24K epch:79.49 loss:0.003 grdn:0.181 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:16:35 ts/train.py:232 step:577K smpl:5M ep:24K epch:79.52 loss:0.004 grdn:0.174 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:18:01 ts/train.py:232 step:578K smpl:5M ep:24K epch:79.55 loss:0.004 grdn:0.181 lr:2.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:19:26 ts/train.py:232 step:578K smpl:5M ep:24K epch:79.57 loss:0.004 grdn:0.178 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:20:51 ts/train.py:232 step:578K smpl:5M ep:24K epch:79.60 loss:0.004 grdn:0.176 lr:2.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 10:22:16 ts/train.py:232 step:578K smpl:5M ep:24K epch:79.63 loss:0.004 grdn:0.187 lr:2.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 10:23:42 ts/train.py:232 step:578K smpl:5M ep:24K epch:79.66 loss:0.004 grdn:0.188 lr:2.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 10:25:07 ts/train.py:232 step:579K smpl:5M ep:24K epch:79.68 loss:0.004 grdn:0.194 lr:2.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 10:26:32 ts/train.py:232 step:579K smpl:5M ep:24K epch:79.71 loss:0.003 grdn:0.172 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:27:57 ts/train.py:232 step:579K smpl:5M ep:24K epch:79.74 loss:0.004 grdn:0.177 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:29:22 ts/train.py:232 step:579K smpl:5M ep:24K epch:79.77 loss:0.004 grdn:0.198 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:30:47 ts/train.py:232 step:579K smpl:5M ep:24K epch:79.79 loss:0.004 grdn:0.188 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:32:13 ts/train.py:232 step:580K smpl:5M ep:24K epch:79.82 loss:0.004 grdn:0.182 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:33:38 ts/train.py:232 step:580K smpl:5M ep:24K epch:79.85 loss:0.004 grdn:0.203 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:35:03 ts/train.py:232 step:580K smpl:5M ep:24K epch:79.88 loss:0.004 grdn:0.184 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:36:28 ts/train.py:232 step:580K smpl:5M ep:24K epch:79.90 loss:0.004 grdn:0.174 lr:2.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 10:37:53 ts/train.py:232 step:580K smpl:5M ep:24K epch:79.93 loss:0.004 grdn:0.188 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:39:18 ts/train.py:232 step:581K smpl:5M ep:24K epch:79.96 loss:0.004 grdn:0.174 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:40:44 ts/train.py:232 step:581K smpl:5M ep:24K epch:79.99 loss:0.004 grdn:0.197 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:42:10 ts/train.py:232 step:581K smpl:5M ep:24K epch:80.01 loss:0.004 grdn:0.182 lr:2.8e-05 updt_s:0.424 data_s:0.008 +INFO 2025-05-08 10:43:35 ts/train.py:232 step:581K smpl:5M ep:24K epch:80.04 loss:0.003 grdn:0.174 lr:2.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 10:45:00 ts/train.py:232 step:581K smpl:5M ep:24K epch:80.07 loss:0.003 grdn:0.174 lr:2.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 10:46:26 ts/train.py:232 step:582K smpl:5M ep:24K epch:80.10 loss:0.003 grdn:0.174 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:47:51 ts/train.py:232 step:582K smpl:5M ep:24K epch:80.12 loss:0.003 grdn:0.173 lr:2.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 10:49:16 ts/train.py:232 step:582K smpl:5M ep:24K epch:80.15 loss:0.004 grdn:0.171 lr:2.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 10:50:41 ts/train.py:232 step:582K smpl:5M ep:24K epch:80.18 loss:0.004 grdn:0.177 lr:2.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 10:52:07 ts/train.py:232 step:582K smpl:5M ep:24K epch:80.21 loss:0.004 grdn:0.192 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:53:32 ts/train.py:232 step:583K smpl:5M ep:24K epch:80.23 loss:0.004 grdn:0.198 lr:2.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 10:54:57 ts/train.py:232 step:583K smpl:5M ep:24K epch:80.26 loss:0.004 grdn:0.194 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:56:22 ts/train.py:232 step:583K smpl:5M ep:24K epch:80.29 loss:0.004 grdn:0.184 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 10:57:48 ts/train.py:232 step:583K smpl:5M ep:24K epch:80.32 loss:0.004 grdn:0.195 lr:2.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 10:59:13 ts/train.py:232 step:583K smpl:5M ep:24K epch:80.34 loss:0.003 grdn:0.172 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:00:38 ts/train.py:232 step:584K smpl:5M ep:24K epch:80.37 loss:0.004 grdn:0.184 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:02:04 ts/train.py:232 step:584K smpl:5M ep:24K epch:80.40 loss:0.003 grdn:0.180 lr:2.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:03:29 ts/train.py:232 step:584K smpl:5M ep:24K epch:80.43 loss:0.003 grdn:0.173 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:04:54 ts/train.py:232 step:584K smpl:5M ep:24K epch:80.45 loss:0.004 grdn:0.186 lr:2.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 11:06:19 ts/train.py:232 step:584K smpl:5M ep:24K epch:80.48 loss:0.004 grdn:0.207 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:07:45 ts/train.py:232 step:585K smpl:5M ep:24K epch:80.51 loss:0.004 grdn:0.193 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:09:10 ts/train.py:232 step:585K smpl:5M ep:24K epch:80.54 loss:0.004 grdn:0.182 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:10:35 ts/train.py:232 step:585K smpl:5M ep:24K epch:80.56 loss:0.004 grdn:0.179 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:12:00 ts/train.py:232 step:585K smpl:5M ep:24K epch:80.59 loss:0.004 grdn:0.193 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:13:25 ts/train.py:232 step:585K smpl:5M ep:24K epch:80.62 loss:0.004 grdn:0.176 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:14:50 ts/train.py:232 step:586K smpl:5M ep:24K epch:80.65 loss:0.004 grdn:0.190 lr:2.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 11:16:15 ts/train.py:232 step:586K smpl:5M ep:24K epch:80.67 loss:0.003 grdn:0.149 lr:2.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 11:17:40 ts/train.py:232 step:586K smpl:5M ep:24K epch:80.70 loss:0.004 grdn:0.188 lr:2.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 11:19:05 ts/train.py:232 step:586K smpl:5M ep:24K epch:80.73 loss:0.004 grdn:0.189 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:20:31 ts/train.py:232 step:586K smpl:5M ep:24K epch:80.76 loss:0.004 grdn:0.193 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:21:56 ts/train.py:232 step:587K smpl:5M ep:24K epch:80.78 loss:0.004 grdn:0.193 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:23:21 ts/train.py:232 step:587K smpl:5M ep:24K epch:80.81 loss:0.004 grdn:0.182 lr:2.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 11:24:47 ts/train.py:232 step:587K smpl:5M ep:24K epch:80.84 loss:0.004 grdn:0.199 lr:2.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 11:26:12 ts/train.py:232 step:587K smpl:5M ep:24K epch:80.87 loss:0.004 grdn:0.176 lr:2.7e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 11:27:37 ts/train.py:232 step:587K smpl:5M ep:24K epch:80.90 loss:0.003 grdn:0.172 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:29:02 ts/train.py:232 step:588K smpl:5M ep:24K epch:80.92 loss:0.004 grdn:0.185 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:30:27 ts/train.py:232 step:588K smpl:5M ep:24K epch:80.95 loss:0.004 grdn:0.181 lr:2.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 11:31:54 ts/train.py:232 step:588K smpl:5M ep:24K epch:80.98 loss:0.004 grdn:0.198 lr:2.7e-05 updt_s:0.424 data_s:0.007 +INFO 2025-05-08 11:33:19 ts/train.py:232 step:588K smpl:5M ep:24K epch:81.01 loss:0.004 grdn:0.184 lr:2.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 11:34:44 ts/train.py:232 step:588K smpl:5M ep:24K epch:81.03 loss:0.003 grdn:0.181 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:36:09 ts/train.py:232 step:589K smpl:5M ep:24K epch:81.06 loss:0.003 grdn:0.184 lr:2.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 11:37:34 ts/train.py:232 step:589K smpl:5M ep:24K epch:81.09 loss:0.004 grdn:0.187 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:38:59 ts/train.py:232 step:589K smpl:5M ep:24K epch:81.12 loss:0.004 grdn:0.179 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:40:24 ts/train.py:232 step:589K smpl:5M ep:24K epch:81.14 loss:0.004 grdn:0.191 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:41:50 ts/train.py:232 step:589K smpl:5M ep:24K epch:81.17 loss:0.004 grdn:0.193 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:43:15 ts/train.py:232 step:590K smpl:5M ep:24K epch:81.20 loss:0.003 grdn:0.164 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:44:40 ts/train.py:232 step:590K smpl:5M ep:24K epch:81.23 loss:0.004 grdn:0.187 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:46:05 ts/train.py:232 step:590K smpl:5M ep:24K epch:81.25 loss:0.004 grdn:0.186 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:47:30 ts/train.py:232 step:590K smpl:5M ep:24K epch:81.28 loss:0.004 grdn:0.180 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:48:56 ts/train.py:232 step:590K smpl:5M ep:24K epch:81.31 loss:0.003 grdn:0.174 lr:2.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:50:21 ts/train.py:232 step:591K smpl:5M ep:24K epch:81.34 loss:0.003 grdn:0.160 lr:2.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 11:51:46 ts/train.py:232 step:591K smpl:5M ep:24K epch:81.36 loss:0.003 grdn:0.183 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:53:12 ts/train.py:232 step:591K smpl:5M ep:24K epch:81.39 loss:0.004 grdn:0.187 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:54:37 ts/train.py:232 step:591K smpl:5M ep:24K epch:81.42 loss:0.004 grdn:0.195 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:56:02 ts/train.py:232 step:591K smpl:5M ep:24K epch:81.45 loss:0.003 grdn:0.180 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:57:27 ts/train.py:232 step:592K smpl:5M ep:24K epch:81.47 loss:0.004 grdn:0.195 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 11:58:52 ts/train.py:232 step:592K smpl:5M ep:24K epch:81.50 loss:0.004 grdn:0.191 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:00:18 ts/train.py:232 step:592K smpl:5M ep:24K epch:81.53 loss:0.004 grdn:0.185 lr:2.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 12:01:43 ts/train.py:232 step:592K smpl:5M ep:24K epch:81.56 loss:0.003 grdn:0.173 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:03:08 ts/train.py:232 step:592K smpl:5M ep:24K epch:81.58 loss:0.004 grdn:0.194 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:04:33 ts/train.py:232 step:593K smpl:5M ep:24K epch:81.61 loss:0.004 grdn:0.179 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:05:58 ts/train.py:232 step:593K smpl:5M ep:24K epch:81.64 loss:0.003 grdn:0.179 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:07:23 ts/train.py:232 step:593K smpl:5M ep:24K epch:81.67 loss:0.004 grdn:0.182 lr:2.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 12:08:49 ts/train.py:232 step:593K smpl:5M ep:25K epch:81.69 loss:0.004 grdn:0.177 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:10:14 ts/train.py:232 step:593K smpl:5M ep:25K epch:81.72 loss:0.004 grdn:0.191 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:11:39 ts/train.py:232 step:594K smpl:5M ep:25K epch:81.75 loss:0.004 grdn:0.186 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:13:04 ts/train.py:232 step:594K smpl:5M ep:25K epch:81.78 loss:0.003 grdn:0.159 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:14:29 ts/train.py:232 step:594K smpl:5M ep:25K epch:81.80 loss:0.003 grdn:0.168 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:15:55 ts/train.py:232 step:594K smpl:5M ep:25K epch:81.83 loss:0.004 grdn:0.184 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:17:20 ts/train.py:232 step:594K smpl:5M ep:25K epch:81.86 loss:0.004 grdn:0.191 lr:2.6e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 12:18:45 ts/train.py:232 step:595K smpl:5M ep:25K epch:81.89 loss:0.003 grdn:0.175 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:20:10 ts/train.py:232 step:595K smpl:5M ep:25K epch:81.91 loss:0.003 grdn:0.173 lr:2.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 12:21:37 ts/train.py:232 step:595K smpl:5M ep:25K epch:81.94 loss:0.004 grdn:0.177 lr:2.6e-05 updt_s:0.424 data_s:0.008 +INFO 2025-05-08 12:23:02 ts/train.py:232 step:595K smpl:5M ep:25K epch:81.97 loss:0.003 grdn:0.187 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:24:27 ts/train.py:232 step:595K smpl:5M ep:25K epch:82.00 loss:0.003 grdn:0.171 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:25:52 ts/train.py:232 step:596K smpl:5M ep:25K epch:82.02 loss:0.004 grdn:0.185 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:27:18 ts/train.py:232 step:596K smpl:5M ep:25K epch:82.05 loss:0.004 grdn:0.200 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:28:43 ts/train.py:232 step:596K smpl:5M ep:25K epch:82.08 loss:0.003 grdn:0.167 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:30:08 ts/train.py:232 step:596K smpl:5M ep:25K epch:82.11 loss:0.003 grdn:0.178 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:31:33 ts/train.py:232 step:596K smpl:5M ep:25K epch:82.13 loss:0.003 grdn:0.171 lr:2.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 12:32:58 ts/train.py:232 step:597K smpl:5M ep:25K epch:82.16 loss:0.004 grdn:0.184 lr:2.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 12:34:23 ts/train.py:232 step:597K smpl:5M ep:25K epch:82.19 loss:0.004 grdn:0.186 lr:2.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:35:49 ts/train.py:232 step:597K smpl:5M ep:25K epch:82.22 loss:0.004 grdn:0.185 lr:2.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 12:37:14 ts/train.py:232 step:597K smpl:5M ep:25K epch:82.24 loss:0.004 grdn:0.191 lr:2.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 12:38:39 ts/train.py:232 step:597K smpl:5M ep:25K epch:82.27 loss:0.004 grdn:0.190 lr:2.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 12:40:04 ts/train.py:232 step:598K smpl:5M ep:25K epch:82.30 loss:0.004 grdn:0.182 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:41:29 ts/train.py:232 step:598K smpl:5M ep:25K epch:82.33 loss:0.004 grdn:0.186 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:42:55 ts/train.py:232 step:598K smpl:5M ep:25K epch:82.35 loss:0.003 grdn:0.190 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:44:20 ts/train.py:232 step:598K smpl:5M ep:25K epch:82.38 loss:0.003 grdn:0.181 lr:2.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 12:45:45 ts/train.py:232 step:598K smpl:5M ep:25K epch:82.41 loss:0.004 grdn:0.195 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:47:10 ts/train.py:232 step:599K smpl:5M ep:25K epch:82.44 loss:0.004 grdn:0.186 lr:2.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 12:48:35 ts/train.py:232 step:599K smpl:5M ep:25K epch:82.47 loss:0.003 grdn:0.180 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:50:00 ts/train.py:232 step:599K smpl:5M ep:25K epch:82.49 loss:0.003 grdn:0.180 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:51:26 ts/train.py:232 step:599K smpl:5M ep:25K epch:82.52 loss:0.004 grdn:0.188 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:52:51 ts/train.py:232 step:599K smpl:5M ep:25K epch:82.55 loss:0.003 grdn:0.179 lr:2.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 12:54:15 ts/train.py:232 step:600K smpl:5M ep:25K epch:82.58 loss:0.004 grdn:0.189 lr:2.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 12:55:41 ts/train.py:232 step:600K smpl:5M ep:25K epch:82.60 loss:0.004 grdn:0.187 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:57:06 ts/train.py:232 step:600K smpl:5M ep:25K epch:82.63 loss:0.004 grdn:0.185 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:57:06 ts/train.py:241 Checkpoint policy after step 600000 +INFO 2025-05-08 12:58:34 ts/train.py:232 step:600K smpl:5M ep:25K epch:82.66 loss:0.004 grdn:0.185 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 12:59:59 ts/train.py:232 step:600K smpl:5M ep:25K epch:82.69 loss:0.003 grdn:0.169 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:01:25 ts/train.py:232 step:601K smpl:5M ep:25K epch:82.71 loss:0.004 grdn:0.202 lr:2.5e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 13:02:50 ts/train.py:232 step:601K smpl:5M ep:25K epch:82.74 loss:0.003 grdn:0.180 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:04:15 ts/train.py:232 step:601K smpl:5M ep:25K epch:82.77 loss:0.003 grdn:0.179 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:05:40 ts/train.py:232 step:601K smpl:5M ep:25K epch:82.80 loss:0.004 grdn:0.188 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:07:06 ts/train.py:232 step:601K smpl:5M ep:25K epch:82.82 loss:0.003 grdn:0.169 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:08:31 ts/train.py:232 step:602K smpl:5M ep:25K epch:82.85 loss:0.004 grdn:0.182 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:09:56 ts/train.py:232 step:602K smpl:5M ep:25K epch:82.88 loss:0.004 grdn:0.180 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:11:23 ts/train.py:232 step:602K smpl:5M ep:25K epch:82.91 loss:0.004 grdn:0.180 lr:2.5e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-08 13:12:48 ts/train.py:232 step:602K smpl:5M ep:25K epch:82.93 loss:0.003 grdn:0.171 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:14:13 ts/train.py:232 step:602K smpl:5M ep:25K epch:82.96 loss:0.004 grdn:0.191 lr:2.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 13:15:38 ts/train.py:232 step:603K smpl:5M ep:25K epch:82.99 loss:0.003 grdn:0.167 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:17:03 ts/train.py:232 step:603K smpl:5M ep:25K epch:83.02 loss:0.003 grdn:0.184 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:18:29 ts/train.py:232 step:603K smpl:5M ep:25K epch:83.04 loss:0.003 grdn:0.179 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:19:54 ts/train.py:232 step:603K smpl:5M ep:25K epch:83.07 loss:0.003 grdn:0.163 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:21:19 ts/train.py:232 step:603K smpl:5M ep:25K epch:83.10 loss:0.004 grdn:0.189 lr:2.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:22:44 ts/train.py:232 step:604K smpl:5M ep:25K epch:83.13 loss:0.003 grdn:0.165 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:24:10 ts/train.py:232 step:604K smpl:5M ep:25K epch:83.15 loss:0.004 grdn:0.195 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:25:35 ts/train.py:232 step:604K smpl:5M ep:25K epch:83.18 loss:0.004 grdn:0.192 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:27:00 ts/train.py:232 step:604K smpl:5M ep:25K epch:83.21 loss:0.004 grdn:0.184 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:28:25 ts/train.py:232 step:604K smpl:5M ep:25K epch:83.24 loss:0.003 grdn:0.180 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:29:50 ts/train.py:232 step:605K smpl:5M ep:25K epch:83.26 loss:0.004 grdn:0.190 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:31:16 ts/train.py:232 step:605K smpl:5M ep:25K epch:83.29 loss:0.003 grdn:0.168 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:32:41 ts/train.py:232 step:605K smpl:5M ep:25K epch:83.32 loss:0.004 grdn:0.188 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:34:06 ts/train.py:232 step:605K smpl:5M ep:25K epch:83.35 loss:0.003 grdn:0.179 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:35:31 ts/train.py:232 step:605K smpl:5M ep:25K epch:83.37 loss:0.003 grdn:0.178 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:36:56 ts/train.py:232 step:606K smpl:5M ep:25K epch:83.40 loss:0.003 grdn:0.188 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:38:22 ts/train.py:232 step:606K smpl:5M ep:25K epch:83.43 loss:0.004 grdn:0.184 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:39:47 ts/train.py:232 step:606K smpl:5M ep:25K epch:83.46 loss:0.003 grdn:0.173 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:41:12 ts/train.py:232 step:606K smpl:5M ep:25K epch:83.48 loss:0.003 grdn:0.181 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:42:37 ts/train.py:232 step:606K smpl:5M ep:25K epch:83.51 loss:0.003 grdn:0.177 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:44:02 ts/train.py:232 step:607K smpl:5M ep:25K epch:83.54 loss:0.004 grdn:0.184 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:45:27 ts/train.py:232 step:607K smpl:5M ep:25K epch:83.57 loss:0.004 grdn:0.189 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:46:52 ts/train.py:232 step:607K smpl:5M ep:25K epch:83.59 loss:0.003 grdn:0.168 lr:2.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 13:48:17 ts/train.py:232 step:607K smpl:5M ep:25K epch:83.62 loss:0.004 grdn:0.181 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:49:43 ts/train.py:232 step:607K smpl:5M ep:25K epch:83.65 loss:0.003 grdn:0.182 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:51:08 ts/train.py:232 step:608K smpl:5M ep:25K epch:83.68 loss:0.003 grdn:0.190 lr:2.4e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 13:52:33 ts/train.py:232 step:608K smpl:5M ep:25K epch:83.70 loss:0.003 grdn:0.188 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:53:58 ts/train.py:232 step:608K smpl:5M ep:25K epch:83.73 loss:0.003 grdn:0.167 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:55:23 ts/train.py:232 step:608K smpl:5M ep:25K epch:83.76 loss:0.003 grdn:0.181 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:56:49 ts/train.py:232 step:608K smpl:5M ep:25K epch:83.79 loss:0.003 grdn:0.163 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:58:14 ts/train.py:232 step:609K smpl:5M ep:25K epch:83.81 loss:0.004 grdn:0.197 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 13:59:39 ts/train.py:232 step:609K smpl:5M ep:25K epch:83.84 loss:0.004 grdn:0.197 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:01:05 ts/train.py:232 step:609K smpl:5M ep:25K epch:83.87 loss:0.004 grdn:0.189 lr:2.4e-05 updt_s:0.425 data_s:0.006 +INFO 2025-05-08 14:02:30 ts/train.py:232 step:609K smpl:5M ep:25K epch:83.90 loss:0.003 grdn:0.174 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:03:56 ts/train.py:232 step:609K smpl:5M ep:25K epch:83.92 loss:0.004 grdn:0.188 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:05:21 ts/train.py:232 step:610K smpl:5M ep:25K epch:83.95 loss:0.003 grdn:0.179 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:06:46 ts/train.py:232 step:610K smpl:5M ep:25K epch:83.98 loss:0.003 grdn:0.183 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:08:11 ts/train.py:232 step:610K smpl:5M ep:25K epch:84.01 loss:0.003 grdn:0.187 lr:2.4e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:09:37 ts/train.py:232 step:610K smpl:5M ep:25K epch:84.04 loss:0.003 grdn:0.170 lr:2.4e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 14:11:02 ts/train.py:232 step:610K smpl:5M ep:25K epch:84.06 loss:0.003 grdn:0.185 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:12:27 ts/train.py:232 step:611K smpl:5M ep:25K epch:84.09 loss:0.003 grdn:0.173 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:13:52 ts/train.py:232 step:611K smpl:5M ep:25K epch:84.12 loss:0.003 grdn:0.174 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:15:17 ts/train.py:232 step:611K smpl:5M ep:25K epch:84.15 loss:0.004 grdn:0.208 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:16:43 ts/train.py:232 step:611K smpl:5M ep:25K epch:84.17 loss:0.004 grdn:0.189 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:18:08 ts/train.py:232 step:611K smpl:5M ep:25K epch:84.20 loss:0.003 grdn:0.181 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:19:33 ts/train.py:232 step:612K smpl:5M ep:25K epch:84.23 loss:0.004 grdn:0.205 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:20:58 ts/train.py:232 step:612K smpl:5M ep:25K epch:84.26 loss:0.003 grdn:0.177 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:22:23 ts/train.py:232 step:612K smpl:5M ep:25K epch:84.28 loss:0.003 grdn:0.176 lr:2.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 14:23:48 ts/train.py:232 step:612K smpl:5M ep:25K epch:84.31 loss:0.004 grdn:0.196 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:25:13 ts/train.py:232 step:612K smpl:5M ep:25K epch:84.34 loss:0.003 grdn:0.172 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:26:39 ts/train.py:232 step:613K smpl:5M ep:25K epch:84.37 loss:0.003 grdn:0.184 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:28:04 ts/train.py:232 step:613K smpl:5M ep:25K epch:84.39 loss:0.003 grdn:0.183 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:29:29 ts/train.py:232 step:613K smpl:5M ep:25K epch:84.42 loss:0.003 grdn:0.186 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:30:54 ts/train.py:232 step:613K smpl:5M ep:25K epch:84.45 loss:0.003 grdn:0.172 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:32:20 ts/train.py:232 step:613K smpl:5M ep:25K epch:84.48 loss:0.004 grdn:0.192 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:33:45 ts/train.py:232 step:614K smpl:5M ep:25K epch:84.50 loss:0.003 grdn:0.185 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:35:10 ts/train.py:232 step:614K smpl:5M ep:25K epch:84.53 loss:0.003 grdn:0.178 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:36:35 ts/train.py:232 step:614K smpl:5M ep:25K epch:84.56 loss:0.004 grdn:0.199 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:38:00 ts/train.py:232 step:614K smpl:5M ep:25K epch:84.59 loss:0.003 grdn:0.172 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:39:25 ts/train.py:232 step:614K smpl:5M ep:25K epch:84.61 loss:0.004 grdn:0.190 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:40:51 ts/train.py:232 step:615K smpl:5M ep:25K epch:84.64 loss:0.003 grdn:0.186 lr:2.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 14:42:16 ts/train.py:232 step:615K smpl:5M ep:25K epch:84.67 loss:0.003 grdn:0.177 lr:2.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 14:43:41 ts/train.py:232 step:615K smpl:5M ep:25K epch:84.70 loss:0.003 grdn:0.185 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:45:06 ts/train.py:232 step:615K smpl:5M ep:25K epch:84.72 loss:0.003 grdn:0.174 lr:2.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 14:46:31 ts/train.py:232 step:615K smpl:5M ep:25K epch:84.75 loss:0.003 grdn:0.165 lr:2.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 14:47:56 ts/train.py:232 step:616K smpl:5M ep:25K epch:84.78 loss:0.004 grdn:0.197 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:49:21 ts/train.py:232 step:616K smpl:5M ep:25K epch:84.81 loss:0.003 grdn:0.180 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:50:47 ts/train.py:232 step:616K smpl:5M ep:25K epch:84.83 loss:0.004 grdn:0.196 lr:2.3e-05 updt_s:0.425 data_s:0.007 +INFO 2025-05-08 14:52:13 ts/train.py:232 step:616K smpl:5M ep:25K epch:84.86 loss:0.004 grdn:0.190 lr:2.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 14:53:38 ts/train.py:232 step:616K smpl:5M ep:25K epch:84.89 loss:0.004 grdn:0.186 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:55:03 ts/train.py:232 step:617K smpl:5M ep:25K epch:84.92 loss:0.003 grdn:0.191 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:56:28 ts/train.py:232 step:617K smpl:5M ep:25K epch:84.94 loss:0.004 grdn:0.180 lr:2.3e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 14:57:53 ts/train.py:232 step:617K smpl:5M ep:25K epch:84.97 loss:0.003 grdn:0.186 lr:2.3e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 14:59:18 ts/train.py:232 step:617K smpl:5M ep:25K epch:85.00 loss:0.003 grdn:0.182 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:00:44 ts/train.py:232 step:617K smpl:5M ep:26K epch:85.03 loss:0.003 grdn:0.185 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:02:09 ts/train.py:232 step:618K smpl:5M ep:26K epch:85.05 loss:0.004 grdn:0.201 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:03:34 ts/train.py:232 step:618K smpl:5M ep:26K epch:85.08 loss:0.003 grdn:0.168 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:05:00 ts/train.py:232 step:618K smpl:5M ep:26K epch:85.11 loss:0.004 grdn:0.211 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:06:25 ts/train.py:232 step:618K smpl:5M ep:26K epch:85.14 loss:0.004 grdn:0.200 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:07:50 ts/train.py:232 step:618K smpl:5M ep:26K epch:85.16 loss:0.004 grdn:0.204 lr:2.2e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 15:09:15 ts/train.py:232 step:619K smpl:5M ep:26K epch:85.19 loss:0.004 grdn:0.191 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:10:40 ts/train.py:232 step:619K smpl:5M ep:26K epch:85.22 loss:0.003 grdn:0.177 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:12:06 ts/train.py:232 step:619K smpl:5M ep:26K epch:85.25 loss:0.003 grdn:0.179 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:13:31 ts/train.py:232 step:619K smpl:5M ep:26K epch:85.27 loss:0.003 grdn:0.177 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:14:56 ts/train.py:232 step:619K smpl:5M ep:26K epch:85.30 loss:0.004 grdn:0.197 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:16:22 ts/train.py:232 step:620K smpl:5M ep:26K epch:85.33 loss:0.003 grdn:0.186 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:17:47 ts/train.py:232 step:620K smpl:5M ep:26K epch:85.36 loss:0.003 grdn:0.192 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:19:12 ts/train.py:232 step:620K smpl:5M ep:26K epch:85.38 loss:0.003 grdn:0.188 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:20:37 ts/train.py:232 step:620K smpl:5M ep:26K epch:85.41 loss:0.003 grdn:0.181 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:22:03 ts/train.py:232 step:620K smpl:5M ep:26K epch:85.44 loss:0.004 grdn:0.184 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:23:28 ts/train.py:232 step:621K smpl:5M ep:26K epch:85.47 loss:0.003 grdn:0.177 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:24:53 ts/train.py:232 step:621K smpl:5M ep:26K epch:85.49 loss:0.003 grdn:0.174 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:26:19 ts/train.py:232 step:621K smpl:5M ep:26K epch:85.52 loss:0.004 grdn:0.198 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:27:44 ts/train.py:232 step:621K smpl:5M ep:26K epch:85.55 loss:0.003 grdn:0.178 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:29:09 ts/train.py:232 step:621K smpl:5M ep:26K epch:85.58 loss:0.003 grdn:0.186 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:30:35 ts/train.py:232 step:622K smpl:5M ep:26K epch:85.61 loss:0.003 grdn:0.181 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:32:00 ts/train.py:232 step:622K smpl:5M ep:26K epch:85.63 loss:0.003 grdn:0.171 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:33:25 ts/train.py:232 step:622K smpl:5M ep:26K epch:85.66 loss:0.003 grdn:0.181 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:34:51 ts/train.py:232 step:622K smpl:5M ep:26K epch:85.69 loss:0.003 grdn:0.184 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:36:16 ts/train.py:232 step:622K smpl:5M ep:26K epch:85.72 loss:0.003 grdn:0.188 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:37:41 ts/train.py:232 step:623K smpl:5M ep:26K epch:85.74 loss:0.004 grdn:0.189 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:39:07 ts/train.py:232 step:623K smpl:5M ep:26K epch:85.77 loss:0.004 grdn:0.197 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:40:34 ts/train.py:232 step:623K smpl:5M ep:26K epch:85.80 loss:0.003 grdn:0.183 lr:2.2e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-08 15:41:59 ts/train.py:232 step:623K smpl:5M ep:26K epch:85.83 loss:0.003 grdn:0.176 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:43:24 ts/train.py:232 step:623K smpl:5M ep:26K epch:85.85 loss:0.003 grdn:0.180 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:44:50 ts/train.py:232 step:624K smpl:5M ep:26K epch:85.88 loss:0.003 grdn:0.161 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:46:15 ts/train.py:232 step:624K smpl:5M ep:26K epch:85.91 loss:0.004 grdn:0.191 lr:2.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:47:40 ts/train.py:232 step:624K smpl:5M ep:26K epch:85.94 loss:0.004 grdn:0.203 lr:2.2e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:49:05 ts/train.py:232 step:624K smpl:5M ep:26K epch:85.96 loss:0.003 grdn:0.181 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:50:31 ts/train.py:232 step:624K smpl:5M ep:26K epch:85.99 loss:0.003 grdn:0.187 lr:2.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 15:51:56 ts/train.py:232 step:625K smpl:5M ep:26K epch:86.02 loss:0.003 grdn:0.176 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:53:21 ts/train.py:232 step:625K smpl:5M ep:26K epch:86.05 loss:0.003 grdn:0.188 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:54:46 ts/train.py:232 step:625K smpl:5M ep:26K epch:86.07 loss:0.004 grdn:0.197 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:56:12 ts/train.py:232 step:625K smpl:5M ep:26K epch:86.10 loss:0.004 grdn:0.195 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:57:37 ts/train.py:232 step:625K smpl:5M ep:26K epch:86.13 loss:0.003 grdn:0.167 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 15:59:02 ts/train.py:232 step:626K smpl:5M ep:26K epch:86.16 loss:0.003 grdn:0.175 lr:2.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:00:27 ts/train.py:232 step:626K smpl:5M ep:26K epch:86.18 loss:0.003 grdn:0.184 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:01:53 ts/train.py:232 step:626K smpl:5M ep:26K epch:86.21 loss:0.003 grdn:0.171 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:03:18 ts/train.py:232 step:626K smpl:5M ep:26K epch:86.24 loss:0.003 grdn:0.192 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:04:43 ts/train.py:232 step:626K smpl:5M ep:26K epch:86.27 loss:0.003 grdn:0.169 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:06:08 ts/train.py:232 step:627K smpl:5M ep:26K epch:86.29 loss:0.003 grdn:0.183 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:07:34 ts/train.py:232 step:627K smpl:5M ep:26K epch:86.32 loss:0.004 grdn:0.195 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:08:59 ts/train.py:232 step:627K smpl:5M ep:26K epch:86.35 loss:0.003 grdn:0.183 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:10:24 ts/train.py:232 step:627K smpl:5M ep:26K epch:86.38 loss:0.003 grdn:0.171 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:11:49 ts/train.py:232 step:627K smpl:5M ep:26K epch:86.40 loss:0.003 grdn:0.173 lr:2.1e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 16:13:14 ts/train.py:232 step:628K smpl:5M ep:26K epch:86.43 loss:0.003 grdn:0.171 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:14:39 ts/train.py:232 step:628K smpl:5M ep:26K epch:86.46 loss:0.004 grdn:0.189 lr:2.1e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 16:16:04 ts/train.py:232 step:628K smpl:5M ep:26K epch:86.49 loss:0.003 grdn:0.181 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:17:30 ts/train.py:232 step:628K smpl:5M ep:26K epch:86.51 loss:0.003 grdn:0.188 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:18:55 ts/train.py:232 step:628K smpl:5M ep:26K epch:86.54 loss:0.003 grdn:0.185 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:20:20 ts/train.py:232 step:629K smpl:5M ep:26K epch:86.57 loss:0.004 grdn:0.203 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:21:45 ts/train.py:232 step:629K smpl:5M ep:26K epch:86.60 loss:0.004 grdn:0.194 lr:2.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:23:10 ts/train.py:232 step:629K smpl:5M ep:26K epch:86.62 loss:0.003 grdn:0.177 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:24:35 ts/train.py:232 step:629K smpl:5M ep:26K epch:86.65 loss:0.003 grdn:0.171 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:26:01 ts/train.py:232 step:629K smpl:5M ep:26K epch:86.68 loss:0.003 grdn:0.194 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:27:26 ts/train.py:232 step:630K smpl:5M ep:26K epch:86.71 loss:0.004 grdn:0.215 lr:2.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:28:51 ts/train.py:232 step:630K smpl:5M ep:26K epch:86.73 loss:0.004 grdn:0.187 lr:2.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:30:18 ts/train.py:232 step:630K smpl:5M ep:26K epch:86.76 loss:0.003 grdn:0.169 lr:2.1e-05 updt_s:0.426 data_s:0.008 +INFO 2025-05-08 16:30:18 ts/train.py:241 Checkpoint policy after step 630000 +INFO 2025-05-08 16:31:47 ts/train.py:232 step:630K smpl:5M ep:26K epch:86.79 loss:0.003 grdn:0.182 lr:2.1e-05 updt_s:0.427 data_s:0.000 +INFO 2025-05-08 16:33:12 ts/train.py:232 step:630K smpl:5M ep:26K epch:86.82 loss:0.003 grdn:0.190 lr:2.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:34:38 ts/train.py:232 step:631K smpl:5M ep:26K epch:86.84 loss:0.004 grdn:0.198 lr:2.1e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:36:03 ts/train.py:232 step:631K smpl:5M ep:26K epch:86.87 loss:0.003 grdn:0.176 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:37:28 ts/train.py:232 step:631K smpl:5M ep:26K epch:86.90 loss:0.003 grdn:0.183 lr:2.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:38:54 ts/train.py:232 step:631K smpl:5M ep:26K epch:86.93 loss:0.003 grdn:0.183 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:40:19 ts/train.py:232 step:631K smpl:5M ep:26K epch:86.95 loss:0.003 grdn:0.179 lr:2.0e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 16:41:44 ts/train.py:232 step:632K smpl:5M ep:26K epch:86.98 loss:0.003 grdn:0.173 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:43:09 ts/train.py:232 step:632K smpl:5M ep:26K epch:87.01 loss:0.003 grdn:0.184 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:44:34 ts/train.py:232 step:632K smpl:5M ep:26K epch:87.04 loss:0.003 grdn:0.186 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:45:59 ts/train.py:232 step:632K smpl:5M ep:26K epch:87.06 loss:0.003 grdn:0.193 lr:2.0e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:47:25 ts/train.py:232 step:632K smpl:5M ep:26K epch:87.09 loss:0.003 grdn:0.184 lr:2.0e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:48:50 ts/train.py:232 step:633K smpl:5M ep:26K epch:87.12 loss:0.003 grdn:0.181 lr:2.0e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:50:15 ts/train.py:232 step:633K smpl:5M ep:26K epch:87.15 loss:0.003 grdn:0.169 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:51:41 ts/train.py:232 step:633K smpl:5M ep:26K epch:87.18 loss:0.003 grdn:0.187 lr:2.0e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 16:53:06 ts/train.py:232 step:633K smpl:5M ep:26K epch:87.20 loss:0.003 grdn:0.183 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:54:31 ts/train.py:232 step:633K smpl:5M ep:26K epch:87.23 loss:0.003 grdn:0.167 lr:2.0e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 16:55:56 ts/train.py:232 step:634K smpl:5M ep:26K epch:87.26 loss:0.003 grdn:0.175 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:57:21 ts/train.py:232 step:634K smpl:5M ep:26K epch:87.29 loss:0.003 grdn:0.179 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 16:58:46 ts/train.py:232 step:634K smpl:5M ep:26K epch:87.31 loss:0.003 grdn:0.178 lr:2.0e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 17:00:12 ts/train.py:232 step:634K smpl:5M ep:26K epch:87.34 loss:0.003 grdn:0.186 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:01:37 ts/train.py:232 step:634K smpl:5M ep:26K epch:87.37 loss:0.004 grdn:0.198 lr:2.0e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:03:02 ts/train.py:232 step:635K smpl:5M ep:26K epch:87.40 loss:0.004 grdn:0.193 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:04:28 ts/train.py:232 step:635K smpl:5M ep:26K epch:87.42 loss:0.003 grdn:0.175 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:05:53 ts/train.py:232 step:635K smpl:5M ep:26K epch:87.45 loss:0.003 grdn:0.194 lr:2.0e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:07:18 ts/train.py:232 step:635K smpl:5M ep:26K epch:87.48 loss:0.004 grdn:0.187 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:08:43 ts/train.py:232 step:635K smpl:5M ep:26K epch:87.51 loss:0.002 grdn:0.147 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:10:09 ts/train.py:232 step:636K smpl:5M ep:26K epch:87.53 loss:0.004 grdn:0.197 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:11:34 ts/train.py:232 step:636K smpl:5M ep:26K epch:87.56 loss:0.003 grdn:0.198 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:12:59 ts/train.py:232 step:636K smpl:5M ep:26K epch:87.59 loss:0.003 grdn:0.188 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:14:24 ts/train.py:232 step:636K smpl:5M ep:26K epch:87.62 loss:0.003 grdn:0.172 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:15:50 ts/train.py:232 step:636K smpl:5M ep:26K epch:87.64 loss:0.003 grdn:0.190 lr:2.0e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:17:15 ts/train.py:232 step:637K smpl:5M ep:26K epch:87.67 loss:0.003 grdn:0.184 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:18:40 ts/train.py:232 step:637K smpl:5M ep:26K epch:87.70 loss:0.003 grdn:0.170 lr:2.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-08 17:20:07 ts/train.py:232 step:637K smpl:5M ep:26K epch:87.73 loss:0.003 grdn:0.191 lr:2.0e-05 updt_s:0.425 data_s:0.007 +INFO 2025-05-08 17:21:32 ts/train.py:232 step:637K smpl:5M ep:26K epch:87.75 loss:0.003 grdn:0.178 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:22:57 ts/train.py:232 step:637K smpl:5M ep:26K epch:87.78 loss:0.004 grdn:0.206 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:24:22 ts/train.py:232 step:638K smpl:5M ep:26K epch:87.81 loss:0.003 grdn:0.192 lr:2.0e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:25:48 ts/train.py:232 step:638K smpl:5M ep:26K epch:87.84 loss:0.004 grdn:0.218 lr:2.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-08 17:27:13 ts/train.py:232 step:638K smpl:5M ep:26K epch:87.86 loss:0.003 grdn:0.191 lr:2.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-08 17:28:38 ts/train.py:232 step:638K smpl:5M ep:26K epch:87.89 loss:0.004 grdn:0.211 lr:1.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 17:30:03 ts/train.py:232 step:638K smpl:5M ep:26K epch:87.92 loss:0.003 grdn:0.171 lr:1.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-08 17:31:28 ts/train.py:232 step:639K smpl:5M ep:26K epch:87.95 loss:0.002 grdn:0.156 lr:1.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 17:32:53 ts/train.py:232 step:639K smpl:5M ep:26K epch:87.97 loss:0.003 grdn:0.177 lr:1.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 17:34:18 ts/train.py:232 step:639K smpl:5M ep:26K epch:88.00 loss:0.003 grdn:0.203 lr:1.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 17:35:43 ts/train.py:232 step:639K smpl:5M ep:26K epch:88.03 loss:0.003 grdn:0.187 lr:1.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 17:37:08 ts/train.py:232 step:639K smpl:5M ep:26K epch:88.06 loss:0.003 grdn:0.191 lr:1.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 17:38:34 ts/train.py:232 step:640K smpl:5M ep:26K epch:88.08 loss:0.003 grdn:0.170 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:39:59 ts/train.py:232 step:640K smpl:5M ep:26K epch:88.11 loss:0.003 grdn:0.177 lr:1.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 17:41:24 ts/train.py:232 step:640K smpl:5M ep:26K epch:88.14 loss:0.003 grdn:0.193 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:42:49 ts/train.py:232 step:640K smpl:5M ep:26K epch:88.17 loss:0.004 grdn:0.195 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:44:15 ts/train.py:232 step:640K smpl:5M ep:26K epch:88.19 loss:0.004 grdn:0.188 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:45:40 ts/train.py:232 step:641K smpl:5M ep:26K epch:88.22 loss:0.003 grdn:0.184 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:47:06 ts/train.py:232 step:641K smpl:5M ep:26K epch:88.25 loss:0.003 grdn:0.174 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:48:31 ts/train.py:232 step:641K smpl:5M ep:26K epch:88.28 loss:0.003 grdn:0.169 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:49:56 ts/train.py:232 step:641K smpl:5M ep:26K epch:88.30 loss:0.003 grdn:0.188 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:51:22 ts/train.py:232 step:641K smpl:5M ep:26K epch:88.33 loss:0.003 grdn:0.197 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:52:47 ts/train.py:232 step:642K smpl:5M ep:27K epch:88.36 loss:0.003 grdn:0.194 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:54:12 ts/train.py:232 step:642K smpl:5M ep:27K epch:88.39 loss:0.003 grdn:0.176 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:55:38 ts/train.py:232 step:642K smpl:5M ep:27K epch:88.41 loss:0.003 grdn:0.179 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:57:03 ts/train.py:232 step:642K smpl:5M ep:27K epch:88.44 loss:0.003 grdn:0.189 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 17:58:28 ts/train.py:232 step:642K smpl:5M ep:27K epch:88.47 loss:0.003 grdn:0.176 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 17:59:54 ts/train.py:232 step:643K smpl:5M ep:27K epch:88.50 loss:0.003 grdn:0.185 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:01:19 ts/train.py:232 step:643K smpl:5M ep:27K epch:88.52 loss:0.003 grdn:0.191 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:02:44 ts/train.py:232 step:643K smpl:5M ep:27K epch:88.55 loss:0.004 grdn:0.198 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:04:10 ts/train.py:232 step:643K smpl:5M ep:27K epch:88.58 loss:0.003 grdn:0.178 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:05:35 ts/train.py:232 step:643K smpl:5M ep:27K epch:88.61 loss:0.003 grdn:0.176 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:07:00 ts/train.py:232 step:644K smpl:5M ep:27K epch:88.63 loss:0.003 grdn:0.181 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:08:26 ts/train.py:232 step:644K smpl:5M ep:27K epch:88.66 loss:0.003 grdn:0.179 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:09:52 ts/train.py:232 step:644K smpl:5M ep:27K epch:88.69 loss:0.003 grdn:0.184 lr:1.9e-05 updt_s:0.426 data_s:0.007 +INFO 2025-05-08 18:11:18 ts/train.py:232 step:644K smpl:5M ep:27K epch:88.72 loss:0.003 grdn:0.190 lr:1.9e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:12:43 ts/train.py:232 step:644K smpl:5M ep:27K epch:88.75 loss:0.004 grdn:0.213 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:14:08 ts/train.py:232 step:645K smpl:5M ep:27K epch:88.77 loss:0.003 grdn:0.178 lr:1.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 18:15:33 ts/train.py:232 step:645K smpl:5M ep:27K epch:88.80 loss:0.003 grdn:0.173 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:16:58 ts/train.py:232 step:645K smpl:5M ep:27K epch:88.83 loss:0.003 grdn:0.187 lr:1.9e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 18:18:23 ts/train.py:232 step:645K smpl:5M ep:27K epch:88.86 loss:0.003 grdn:0.178 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:19:48 ts/train.py:232 step:645K smpl:5M ep:27K epch:88.88 loss:0.003 grdn:0.186 lr:1.9e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:21:14 ts/train.py:232 step:646K smpl:5M ep:27K epch:88.91 loss:0.003 grdn:0.161 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:22:39 ts/train.py:232 step:646K smpl:5M ep:27K epch:88.94 loss:0.003 grdn:0.195 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:24:05 ts/train.py:232 step:646K smpl:5M ep:27K epch:88.97 loss:0.003 grdn:0.174 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:25:30 ts/train.py:232 step:646K smpl:5M ep:27K epch:88.99 loss:0.004 grdn:0.206 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:26:55 ts/train.py:232 step:646K smpl:5M ep:27K epch:89.02 loss:0.004 grdn:0.200 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:28:20 ts/train.py:232 step:647K smpl:5M ep:27K epch:89.05 loss:0.003 grdn:0.190 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:29:45 ts/train.py:232 step:647K smpl:5M ep:27K epch:89.08 loss:0.003 grdn:0.176 lr:1.8e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-08 18:31:10 ts/train.py:232 step:647K smpl:5M ep:27K epch:89.10 loss:0.003 grdn:0.188 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:32:36 ts/train.py:232 step:647K smpl:5M ep:27K epch:89.13 loss:0.003 grdn:0.195 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:34:01 ts/train.py:232 step:647K smpl:5M ep:27K epch:89.16 loss:0.003 grdn:0.178 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:35:26 ts/train.py:232 step:648K smpl:5M ep:27K epch:89.19 loss:0.003 grdn:0.187 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:36:51 ts/train.py:232 step:648K smpl:5M ep:27K epch:89.21 loss:0.004 grdn:0.211 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:38:17 ts/train.py:232 step:648K smpl:5M ep:27K epch:89.24 loss:0.003 grdn:0.166 lr:1.8e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 18:39:42 ts/train.py:232 step:648K smpl:5M ep:27K epch:89.27 loss:0.003 grdn:0.185 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:41:07 ts/train.py:232 step:648K smpl:5M ep:27K epch:89.30 loss:0.004 grdn:0.198 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:42:32 ts/train.py:232 step:649K smpl:5M ep:27K epch:89.32 loss:0.003 grdn:0.183 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:43:57 ts/train.py:232 step:649K smpl:5M ep:27K epch:89.35 loss:0.003 grdn:0.180 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:45:23 ts/train.py:232 step:649K smpl:5M ep:27K epch:89.38 loss:0.003 grdn:0.183 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:46:48 ts/train.py:232 step:649K smpl:5M ep:27K epch:89.41 loss:0.003 grdn:0.192 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:48:13 ts/train.py:232 step:649K smpl:5M ep:27K epch:89.43 loss:0.003 grdn:0.164 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:49:39 ts/train.py:232 step:650K smpl:5M ep:27K epch:89.46 loss:0.003 grdn:0.174 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:51:04 ts/train.py:232 step:650K smpl:5M ep:27K epch:89.49 loss:0.003 grdn:0.199 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:52:29 ts/train.py:232 step:650K smpl:5M ep:27K epch:89.52 loss:0.003 grdn:0.185 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:53:54 ts/train.py:232 step:650K smpl:5M ep:27K epch:89.54 loss:0.003 grdn:0.162 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:55:20 ts/train.py:232 step:650K smpl:5M ep:27K epch:89.57 loss:0.003 grdn:0.179 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:56:45 ts/train.py:232 step:651K smpl:5M ep:27K epch:89.60 loss:0.003 grdn:0.184 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 18:58:10 ts/train.py:232 step:651K smpl:5M ep:27K epch:89.63 loss:0.003 grdn:0.177 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 18:59:37 ts/train.py:232 step:651K smpl:5M ep:27K epch:89.65 loss:0.003 grdn:0.207 lr:1.8e-05 updt_s:0.425 data_s:0.008 +INFO 2025-05-08 19:01:02 ts/train.py:232 step:651K smpl:5M ep:27K epch:89.68 loss:0.003 grdn:0.202 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:02:27 ts/train.py:232 step:651K smpl:5M ep:27K epch:89.71 loss:0.003 grdn:0.195 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:03:53 ts/train.py:232 step:652K smpl:5M ep:27K epch:89.74 loss:0.003 grdn:0.187 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:05:18 ts/train.py:232 step:652K smpl:5M ep:27K epch:89.76 loss:0.003 grdn:0.196 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:06:43 ts/train.py:232 step:652K smpl:5M ep:27K epch:89.79 loss:0.003 grdn:0.178 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:08:09 ts/train.py:232 step:652K smpl:5M ep:27K epch:89.82 loss:0.003 grdn:0.179 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 19:09:34 ts/train.py:232 step:652K smpl:5M ep:27K epch:89.85 loss:0.004 grdn:0.210 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:10:59 ts/train.py:232 step:653K smpl:5M ep:27K epch:89.87 loss:0.003 grdn:0.204 lr:1.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-08 19:12:24 ts/train.py:232 step:653K smpl:5M ep:27K epch:89.90 loss:0.003 grdn:0.184 lr:1.8e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:13:50 ts/train.py:232 step:653K smpl:5M ep:27K epch:89.93 loss:0.003 grdn:0.193 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:15:15 ts/train.py:232 step:653K smpl:5M ep:27K epch:89.96 loss:0.003 grdn:0.181 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:16:40 ts/train.py:232 step:653K smpl:5M ep:27K epch:89.98 loss:0.003 grdn:0.177 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:18:05 ts/train.py:232 step:654K smpl:5M ep:27K epch:90.01 loss:0.003 grdn:0.174 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:19:30 ts/train.py:232 step:654K smpl:5M ep:27K epch:90.04 loss:0.003 grdn:0.173 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 19:20:55 ts/train.py:232 step:654K smpl:5M ep:27K epch:90.07 loss:0.003 grdn:0.205 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:22:20 ts/train.py:232 step:654K smpl:5M ep:27K epch:90.09 loss:0.003 grdn:0.184 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:23:46 ts/train.py:232 step:654K smpl:5M ep:27K epch:90.12 loss:0.003 grdn:0.166 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 19:25:10 ts/train.py:232 step:655K smpl:5M ep:27K epch:90.15 loss:0.003 grdn:0.192 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 19:26:36 ts/train.py:232 step:655K smpl:5M ep:27K epch:90.18 loss:0.003 grdn:0.172 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:28:01 ts/train.py:232 step:655K smpl:5M ep:27K epch:90.20 loss:0.003 grdn:0.175 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 19:29:26 ts/train.py:232 step:655K smpl:5M ep:27K epch:90.23 loss:0.003 grdn:0.182 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:30:51 ts/train.py:232 step:655K smpl:5M ep:27K epch:90.26 loss:0.003 grdn:0.175 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:32:16 ts/train.py:232 step:656K smpl:5M ep:27K epch:90.29 loss:0.003 grdn:0.192 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:33:42 ts/train.py:232 step:656K smpl:5M ep:27K epch:90.32 loss:0.003 grdn:0.182 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:35:07 ts/train.py:232 step:656K smpl:5M ep:27K epch:90.34 loss:0.003 grdn:0.188 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:36:32 ts/train.py:232 step:656K smpl:5M ep:27K epch:90.37 loss:0.003 grdn:0.182 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:37:57 ts/train.py:232 step:656K smpl:5M ep:27K epch:90.40 loss:0.003 grdn:0.181 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:39:22 ts/train.py:232 step:657K smpl:5M ep:27K epch:90.43 loss:0.003 grdn:0.195 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:40:47 ts/train.py:232 step:657K smpl:5M ep:27K epch:90.45 loss:0.003 grdn:0.166 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 19:42:12 ts/train.py:232 step:657K smpl:5M ep:27K epch:90.48 loss:0.003 grdn:0.184 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:43:38 ts/train.py:232 step:657K smpl:5M ep:27K epch:90.51 loss:0.003 grdn:0.177 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:45:03 ts/train.py:232 step:657K smpl:5M ep:27K epch:90.54 loss:0.003 grdn:0.202 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 19:46:28 ts/train.py:232 step:658K smpl:5M ep:27K epch:90.56 loss:0.003 grdn:0.190 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:47:53 ts/train.py:232 step:658K smpl:5M ep:27K epch:90.59 loss:0.003 grdn:0.177 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:49:19 ts/train.py:232 step:658K smpl:5M ep:27K epch:90.62 loss:0.003 grdn:0.169 lr:1.7e-05 updt_s:0.423 data_s:0.006 +INFO 2025-05-08 19:50:44 ts/train.py:232 step:658K smpl:5M ep:27K epch:90.65 loss:0.003 grdn:0.179 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 19:52:09 ts/train.py:232 step:658K smpl:5M ep:27K epch:90.67 loss:0.003 grdn:0.185 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 19:53:34 ts/train.py:232 step:659K smpl:5M ep:27K epch:90.70 loss:0.003 grdn:0.202 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:55:00 ts/train.py:232 step:659K smpl:5M ep:27K epch:90.73 loss:0.003 grdn:0.188 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:56:25 ts/train.py:232 step:659K smpl:5M ep:27K epch:90.76 loss:0.003 grdn:0.201 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 19:57:50 ts/train.py:232 step:659K smpl:5M ep:27K epch:90.78 loss:0.003 grdn:0.187 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 19:59:15 ts/train.py:232 step:659K smpl:5M ep:27K epch:90.81 loss:0.003 grdn:0.183 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:00:40 ts/train.py:232 step:660K smpl:5M ep:27K epch:90.84 loss:0.003 grdn:0.194 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:02:05 ts/train.py:232 step:660K smpl:5M ep:27K epch:90.87 loss:0.003 grdn:0.175 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:03:30 ts/train.py:232 step:660K smpl:5M ep:27K epch:90.89 loss:0.003 grdn:0.165 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:03:30 ts/train.py:241 Checkpoint policy after step 660000 +INFO 2025-05-08 20:04:58 ts/train.py:232 step:660K smpl:5M ep:27K epch:90.92 loss:0.003 grdn:0.200 lr:1.7e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:06:24 ts/train.py:232 step:660K smpl:5M ep:27K epch:90.95 loss:0.003 grdn:0.181 lr:1.7e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:07:49 ts/train.py:232 step:661K smpl:5M ep:27K epch:90.98 loss:0.003 grdn:0.183 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:09:14 ts/train.py:232 step:661K smpl:5M ep:27K epch:91.00 loss:0.003 grdn:0.190 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:10:39 ts/train.py:232 step:661K smpl:5M ep:27K epch:91.03 loss:0.003 grdn:0.179 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:12:04 ts/train.py:232 step:661K smpl:5M ep:27K epch:91.06 loss:0.002 grdn:0.160 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:13:29 ts/train.py:232 step:661K smpl:5M ep:27K epch:91.09 loss:0.003 grdn:0.170 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:14:54 ts/train.py:232 step:662K smpl:5M ep:27K epch:91.11 loss:0.003 grdn:0.173 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:16:20 ts/train.py:232 step:662K smpl:5M ep:27K epch:91.14 loss:0.003 grdn:0.177 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:17:45 ts/train.py:232 step:662K smpl:5M ep:27K epch:91.17 loss:0.003 grdn:0.206 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:19:10 ts/train.py:232 step:662K smpl:5M ep:27K epch:91.20 loss:0.003 grdn:0.184 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:20:35 ts/train.py:232 step:662K smpl:5M ep:27K epch:91.22 loss:0.003 grdn:0.208 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:22:00 ts/train.py:232 step:663K smpl:5M ep:27K epch:91.25 loss:0.003 grdn:0.201 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:23:25 ts/train.py:232 step:663K smpl:5M ep:27K epch:91.28 loss:0.003 grdn:0.199 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:24:50 ts/train.py:232 step:663K smpl:5M ep:27K epch:91.31 loss:0.003 grdn:0.182 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:26:15 ts/train.py:232 step:663K smpl:5M ep:27K epch:91.33 loss:0.003 grdn:0.199 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:27:40 ts/train.py:232 step:663K smpl:5M ep:27K epch:91.36 loss:0.003 grdn:0.204 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:29:05 ts/train.py:232 step:664K smpl:5M ep:27K epch:91.39 loss:0.003 grdn:0.187 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:30:31 ts/train.py:232 step:664K smpl:5M ep:27K epch:91.42 loss:0.003 grdn:0.174 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:31:56 ts/train.py:232 step:664K smpl:5M ep:27K epch:91.44 loss:0.003 grdn:0.174 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:33:21 ts/train.py:232 step:664K smpl:5M ep:27K epch:91.47 loss:0.003 grdn:0.181 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:34:46 ts/train.py:232 step:664K smpl:5M ep:27K epch:91.50 loss:0.003 grdn:0.191 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:36:11 ts/train.py:232 step:665K smpl:5M ep:27K epch:91.53 loss:0.003 grdn:0.184 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:37:36 ts/train.py:232 step:665K smpl:5M ep:27K epch:91.55 loss:0.003 grdn:0.184 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:39:02 ts/train.py:232 step:665K smpl:5M ep:27K epch:91.58 loss:0.003 grdn:0.200 lr:1.6e-05 updt_s:0.424 data_s:0.007 +INFO 2025-05-08 20:40:27 ts/train.py:232 step:665K smpl:5M ep:27K epch:91.61 loss:0.003 grdn:0.194 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:41:52 ts/train.py:232 step:665K smpl:5M ep:27K epch:91.64 loss:0.003 grdn:0.204 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:43:18 ts/train.py:232 step:666K smpl:5M ep:27K epch:91.66 loss:0.002 grdn:0.167 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:44:43 ts/train.py:232 step:666K smpl:5M ep:28K epch:91.69 loss:0.003 grdn:0.183 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:46:08 ts/train.py:232 step:666K smpl:5M ep:28K epch:91.72 loss:0.003 grdn:0.166 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:47:33 ts/train.py:232 step:666K smpl:5M ep:28K epch:91.75 loss:0.003 grdn:0.188 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:48:58 ts/train.py:232 step:666K smpl:5M ep:28K epch:91.77 loss:0.003 grdn:0.165 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 20:50:23 ts/train.py:232 step:667K smpl:5M ep:28K epch:91.80 loss:0.003 grdn:0.189 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:51:48 ts/train.py:232 step:667K smpl:5M ep:28K epch:91.83 loss:0.003 grdn:0.188 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:53:14 ts/train.py:232 step:667K smpl:5M ep:28K epch:91.86 loss:0.003 grdn:0.184 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:54:39 ts/train.py:232 step:667K smpl:5M ep:28K epch:91.89 loss:0.002 grdn:0.160 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:56:04 ts/train.py:232 step:667K smpl:5M ep:28K epch:91.91 loss:0.003 grdn:0.186 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:57:29 ts/train.py:232 step:668K smpl:5M ep:28K epch:91.94 loss:0.003 grdn:0.196 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 20:58:54 ts/train.py:232 step:668K smpl:5M ep:28K epch:91.97 loss:0.003 grdn:0.185 lr:1.6e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 21:00:19 ts/train.py:232 step:668K smpl:5M ep:28K epch:92.00 loss:0.003 grdn:0.190 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:01:44 ts/train.py:232 step:668K smpl:5M ep:28K epch:92.02 loss:0.003 grdn:0.186 lr:1.6e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:03:09 ts/train.py:232 step:668K smpl:5M ep:28K epch:92.05 loss:0.003 grdn:0.185 lr:1.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:04:34 ts/train.py:232 step:669K smpl:5M ep:28K epch:92.08 loss:0.003 grdn:0.189 lr:1.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:05:59 ts/train.py:232 step:669K smpl:5M ep:28K epch:92.11 loss:0.003 grdn:0.174 lr:1.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:07:24 ts/train.py:232 step:669K smpl:5M ep:28K epch:92.13 loss:0.003 grdn:0.177 lr:1.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:08:49 ts/train.py:232 step:669K smpl:5M ep:28K epch:92.16 loss:0.002 grdn:0.160 lr:1.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:10:14 ts/train.py:232 step:669K smpl:5M ep:28K epch:92.19 loss:0.003 grdn:0.168 lr:1.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:11:39 ts/train.py:232 step:670K smpl:5M ep:28K epch:92.22 loss:0.003 grdn:0.180 lr:1.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:13:04 ts/train.py:232 step:670K smpl:5M ep:28K epch:92.24 loss:0.003 grdn:0.191 lr:1.5e-05 updt_s:0.424 data_s:0.000 +INFO 2025-05-08 21:14:29 ts/train.py:232 step:670K smpl:5M ep:28K epch:92.27 loss:0.003 grdn:0.182 lr:1.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-08 21:15:54 ts/train.py:232 step:670K smpl:5M ep:28K epch:92.30 loss:0.003 grdn:0.182 lr:1.5e-05 updt_s:0.424 data_s:0.000 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-core.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-core.log index b50695ef0cfebf5e276b7e882d110d6b57ed749b..c49e9b4c88bae707a01ec143d3c837ccb3143559 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-core.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-core.log @@ -4,3 +4,5 @@ {"time":"2025-05-05T13:50:12.193687277Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:60862"} {"time":"2025-05-05T13:50:12.201074417Z","level":"INFO","msg":"handleInformInit: received","streamId":"fjhp50hn","id":"127.0.0.1:60862"} {"time":"2025-05-05T13:50:12.532899082Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"fjhp50hn","id":"127.0.0.1:60862"} +{"time":"2025-05-08T21:16:33.512989091Z","level":"ERROR","msg":"connection: fatal error reading connection","error":"read tcp 127.0.0.1:37201->127.0.0.1:60862: read: connection reset by peer","id":"127.0.0.1:60862"} +{"time":"2025-05-08T21:16:34.031318779Z","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log index eb106f8bffbb1de9a45650186cca3cf6bd2790c4..0a006fd387e91b2e6238bbb58a5e80b464887fc8 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log @@ -13,3 +13,4 @@ {"time":"2025-05-06T12:44:48.040371676Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:43741->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T16:50:48.039369614Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:38891->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T18:02:28.241678384Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream","body":"\n\n\nPlease try again in 30 seconds.\n
\n\n"} +{"time":"2025-05-07T16:25:48.040273802Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:42598->141.223.7.51:53: i/o timeout"} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb index fdd48abb34d1c18998692657f13cfd3e37850457..68058422e3348c4965e59e68c73648aa60de5c5d 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f42dd539beda5413d25e35410a3c4bdb0d43c667fd6a26ed875149d6e1db6b13 -size 36175872 +oid sha256:af6740d381ca11aba92ace061c319b70ed103399ce7a4a63ce9b67a134c65eab +size 62816256