diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6b6a487dbb31a34be4ee7c288705ee5c07deab25 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cecacf3bded505dae51b4e7c8c6ba582e6be0e59267ef53f1b766d2dfeb15b8 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..c95ffe05c43624624f92810f8806852a0be08830 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.550066611157049e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..87afb85bcabc4fd16b9d98cffa6c323173c887bd --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ce39ed8d5f17c22d6003a065fcb7155c27e2778a94c56ffd8e56afe7d5f6f36 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2ceb52c38c20af874aae5e1c64a4a6681de1a4be --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51d4c655898ba6d66825b41e21d6d453e892f3b25ad9fcd53d9a13b32e25e67f +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a39d6e5bb2a4966dfcb991d5bc882f8c4dadb29b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 360000, + "verbose": false, + "_step_count": 360001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.550066611157049e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..bd79b4d9c6aaae91547528200cf9573568cb1ede --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 360000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2c74ee75b590c2f037ae566b4156d57a7095b70f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:088669696e52fc9f1567403751ac68fed12efb0eb03ea57cefb04d4ab67c617b +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1c33646195ba601cca271c8543025c3f774bc81a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.044397670300381e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..994ac6227f5941f080ef7d1cd57ccb56d925c49d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:010f52889b6ac4e073bdf41bf9cac66cabe5772c276ec01ae5083606eabd3a11 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b217d76c01a05d5649fcbcd5869c9def9c127674 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:033a888794e13e2d7317678f1b9db22470ea681047e7f8890f48a6c8ae8fd046 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a15f677a8636acb8dc9409520ae800be927a2679 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 390000, + "verbose": false, + "_step_count": 390001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.044397670300381e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..9b8e02f046c68b25df949961c99b3b5917065d80 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/390000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 390000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1f7876e43c5a0dc76c4df0af5a37f062bcedaa12 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:486c5214eb3c9c7e6f18737589cb0f5e18b73c2d20e87612b6820f55fa937785 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1d40674d60d7b8af26656472e6e042f9db106ad6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 5.527273377010439e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..79a9b6d45d3d819d327d7ae1bb1bd9dea5ad063c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c5b8259b261b2649795043332585cc823bdb918f53f4d382e8e516ff7b42e31 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7701bfccdf71fbee773937080558480802b96052 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:217feddfd6a17d36cd76607b1b9cd2091a5cdda2c3aac83f7837289234f63506 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..56b95070343527bb3d50e47b311c1b9e0f69978f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 420000, + "verbose": false, + "_step_count": 420001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.527273377010439e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..bb0ce907623865596f71441ca75ab257971c1b55 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/420000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 420000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..56c39181a0d7fb31dccc7272329bd4ca512161e1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c703a3917b6dd99760837e8c620a021d420c0fe0ee4652cc7f3484edf3e4b3b6 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..2f02db3040a7a6bdfad0336ac978af3b3bf8432f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 5.004365747991111e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..beed5315ceda993111e31dd80010a16e0713265a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:395720815e0f3cdc9b5156427ab340d19e24833daa0d6c06ba2b8783e8b6d2fe +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e2b78d9ad8c50ada6a52d26ecdb78c8f587e40fa --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4cc2b031a851bc510633f8fda6032f999d92028cfa692cbe1accf72b5889bd9 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ee8adcba04bf026c6be79d4168d85cdce285b910 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 450000, + "verbose": false, + "_step_count": 450001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.004365747991111e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..07e97e165f1e6245e7d2e431c0587c506b887c3e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/450000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 450000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b21741e06fa861e1992b1acc29c3966464c38690 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b37d1dee87d16104df04187e08e8c0c78f95e7bf7196d926d249e32181cedbbf +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..185ed4617482b40deb15b33a188fe7f97e23f4c9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 4.4814102337808293e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..70c8c31ce0353471c5b3640d94ee1861cb5cc1cc --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f299812db505142b632880e226e95583e3dce0256abd9d7f47d1e1bfa28a941 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e0969b67a4cf0b761667720f186a377215d015b0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e93f9e062cacd3d3598ed0a04e36a66a78b5407532c8648fe84317da5a32be1 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a0b5034bbc249bb90d3558fd2d0584df557319e4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 480000, + "verbose": false, + "_step_count": 480001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 4.4814102337808293e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..a54500e6a42a0151bd2a06726edce1d9ecf0a2e2 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/480000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 480000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..08b3fff3da8c8e5b27710645a56f836340f9646c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aab31c6270525e4b2451eea051fa99b7a32ce43c32896cc7909055856183121b +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..088e05678fc18c310620072b05f782dd216ba833 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 3.9641428101410884e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a7bc11981a32d6ec314fc70da00cab86b00f57cc --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f7b8acd44f3fd3dbc2965fc1ab4590c1aa25e13feb6de1eb70a3e44cbb501e4 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..97010a927e4152f935c358e73973d027d5ab3286 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79137566b8d413b6783bc119417622b5d246ba8e55bca94ce8f35aabff413e68 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..60573d75e75dda18fcda9031c35db214eaf9bbd6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 510000, + "verbose": false, + "_step_count": 510001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 3.9641428101410884e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8c7035b86e69cd307e77133d7211848fb02474a9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/510000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 510000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3f8a0417bf94a57c6a082ed11355e3cc603eeee8 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:363898f1772a93d10f22cf08a33c60ae0b9d9be818f60550326cf91d90cbffe0 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..88b4b9ccc1b52457955627c808b873afd081d43e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 3.4582370636841024e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1cebf1205339dcd95b6506bc3e8cb292a3592c05 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:731c52d2435a4624327d8a812e6033f354879e2a576c023c3716164853928e2b +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..51cac8cdccbbfe23a6b54ee0ccaa493442dbd6f4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bad50971b9a21fd4afdb851d3987b00e98ea7ed7867776d3811696b7ec547086 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6ba6b261244956225f1e9c33461cb499b6610fff --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 540000, + "verbose": false, + "_step_count": 540001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 3.4582370636841024e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..3c2f6e37b4db10fefdc1e301eb1523863fc3a230 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/540000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 540000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..90f88dcd8fe20163d81f6a304aae38188052b376 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2edb295b29fe00d1cd86e1aa942f8e6e154b956c7a0a85cafa8b51afc55bb85c +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..d7f1d9d7949b75022e1f956d21929a89f2c30d46 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.969241961808482e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8973505d5ac9ba7b0206146e8baa9b5f241bf375 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:461e91b0d99fb1a224962ece58b65e60c3086666abcefda71a6d8d5c881fb1a1 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..22c0013e441e9100711f4dfaa38f07d2c59adc12 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0140befeb610cdb9ae24ce9f03e4b7fdd0acdce34df2607ba9e92365e90dcd49 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..241852cc90a19937813ad01d9b852a9b5acc0c75 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 570000, + "verbose": false, + "_step_count": 570001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.969241961808482e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..25cf656c471a27bc70b42dcccf3b85bb11e144cf --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/570000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 570000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d4269afcc02918c06091419d0fafd384a3ddc1f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c98b9f4cf3fe766848991bc9955b92f7df67950c1c481a07a3a6ef7f00e6650e +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1e06b28115903bca73e927bd34166cbe105f9008 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.5025209895060332e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f965739e9d0506d85374b316fd0566387bf91d54 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca446a51d8c34b937dc0c78ec8193795e999c65ca815f90706cca0de6b3b555d +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..83a5cfb98d9948e2aa4e5923692722457a57b203 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c8b195703584bb0fd9fadf79c8f47966570fb39967e4a500430db84b73fe7d5 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9ce41c0f5550f876c36888584be6ca09bdc67fa6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 600000, + "verbose": false, + "_step_count": 600001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.5025209895060332e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..13ce6152ad24f61b4c05746d5cd0bf93c31d7c0f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/600000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 600000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cfab12b6cb2eb4710404a70112bef4b25f2c17fb --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:197048e737faf872f02f2a327453ac0de55a83cb92091b2cf3b0d35b16956c89 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..85d863715dc241fd41b24c5316e09855e12972da --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.0631933206103992e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2292c14d0ab4127dcc511c91bcda2d6db5963c28 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72ed227ac46458a9a97b4cad0dc374610e1b263b5483328985231dfc94d0113d +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..189f42c0b6ce44b88cc14393cd14f2b9eccc337e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ba23041b6cc06e4967906e6f4b29b0f0a6a8fc9e52859fcb6bc6420115735cb +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a7d20ad7dd005e2ca9bb3b3b833686c191654091 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 630000, + "verbose": false, + "_step_count": 630001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.0631933206103992e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..71be4f013a155caed9df1f4d0513d9218f407ac3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/630000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 630000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log index 5f62ebc4cb523ae8972361205830e5ed5decd03e..6dd46a936b68234dcfbedfd645937ba693897139 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log @@ -12,3 +12,4 @@ {"time":"2025-05-06T10:17:58.342619055Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:59259->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T10:36:28.34160936Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:59320->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T11:17:58.342345052Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49666->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T23:06:26.097134902Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/output.log b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/output.log index a2523ad6b0601f7f7039439893f9b30aca2f0779..ef4de12d4fb6130554cfd50f28798cafd9783314 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/output.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/output.log @@ -2933,3 +2933,1606 @@ INFO 2025-05-06 14:12:22 ts/train.py:232 step:340K smpl:5M ep:28K epch:93.54 los INFO 2025-05-06 14:13:12 ts/train.py:232 step:340K smpl:5M ep:28K epch:93.59 loss:0.006 grdn:0.167 lr:6.9e-05 updt_s:0.161 data_s:0.089 INFO 2025-05-06 14:14:03 ts/train.py:232 step:340K smpl:5M ep:28K epch:93.65 loss:0.006 grdn:0.167 lr:6.9e-05 updt_s:0.160 data_s:0.092 INFO 2025-05-06 14:14:52 ts/train.py:232 step:340K smpl:5M ep:28K epch:93.70 loss:0.006 grdn:0.173 lr:6.9e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-06 14:15:40 ts/train.py:232 step:340K smpl:5M ep:28K epch:93.76 loss:0.005 grdn:0.157 lr:6.9e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 14:16:29 ts/train.py:232 step:341K smpl:5M ep:28K epch:93.81 loss:0.005 grdn:0.162 lr:6.9e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 14:17:18 ts/train.py:232 step:341K smpl:5M ep:28K epch:93.87 loss:0.005 grdn:0.155 lr:6.9e-05 updt_s:0.161 data_s:0.082 +INFO 2025-05-06 14:18:07 ts/train.py:232 step:341K smpl:5M ep:28K epch:93.92 loss:0.006 grdn:0.174 lr:6.9e-05 updt_s:0.160 data_s:0.086 +INFO 2025-05-06 14:18:57 ts/train.py:232 step:341K smpl:5M ep:28K epch:93.98 loss:0.006 grdn:0.164 lr:6.9e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-06 14:19:46 ts/train.py:232 step:341K smpl:5M ep:28K epch:94.03 loss:0.005 grdn:0.138 lr:6.9e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 14:20:35 ts/train.py:232 step:342K smpl:5M ep:28K epch:94.09 loss:0.006 grdn:0.162 lr:6.9e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 14:21:23 ts/train.py:232 step:342K smpl:5M ep:28K epch:94.14 loss:0.005 grdn:0.146 lr:6.9e-05 updt_s:0.160 data_s:0.080 +INFO 2025-05-06 14:22:11 ts/train.py:232 step:342K smpl:5M ep:28K epch:94.20 loss:0.006 grdn:0.167 lr:6.8e-05 updt_s:0.160 data_s:0.080 +INFO 2025-05-06 14:23:01 ts/train.py:232 step:342K smpl:5M ep:28K epch:94.25 loss:0.006 grdn:0.163 lr:6.8e-05 updt_s:0.160 data_s:0.088 +INFO 2025-05-06 14:23:50 ts/train.py:232 step:342K smpl:5M ep:28K epch:94.31 loss:0.005 grdn:0.143 lr:6.8e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-06 14:24:38 ts/train.py:232 step:343K smpl:5M ep:28K epch:94.36 loss:0.006 grdn:0.161 lr:6.8e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 14:25:27 ts/train.py:232 step:343K smpl:5M ep:28K epch:94.42 loss:0.005 grdn:0.151 lr:6.8e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-06 14:26:15 ts/train.py:232 step:343K smpl:5M ep:28K epch:94.47 loss:0.005 grdn:0.158 lr:6.8e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 14:27:07 ts/train.py:232 step:343K smpl:5M ep:28K epch:94.53 loss:0.006 grdn:0.171 lr:6.8e-05 updt_s:0.160 data_s:0.097 +INFO 2025-05-06 14:27:55 ts/train.py:232 step:343K smpl:5M ep:28K epch:94.58 loss:0.005 grdn:0.154 lr:6.8e-05 updt_s:0.160 data_s:0.080 +INFO 2025-05-06 14:28:44 ts/train.py:232 step:344K smpl:5M ep:28K epch:94.64 loss:0.005 grdn:0.160 lr:6.8e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 14:29:32 ts/train.py:232 step:344K smpl:6M ep:28K epch:94.69 loss:0.005 grdn:0.145 lr:6.8e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 14:30:21 ts/train.py:232 step:344K smpl:6M ep:28K epch:94.75 loss:0.006 grdn:0.156 lr:6.8e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-06 14:31:09 ts/train.py:232 step:344K smpl:6M ep:28K epch:94.80 loss:0.005 grdn:0.155 lr:6.8e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 14:31:58 ts/train.py:232 step:344K smpl:6M ep:28K epch:94.86 loss:0.005 grdn:0.149 lr:6.8e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 14:32:48 ts/train.py:232 step:345K smpl:6M ep:28K epch:94.91 loss:0.005 grdn:0.154 lr:6.8e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 14:33:38 ts/train.py:232 step:345K smpl:6M ep:28K epch:94.97 loss:0.005 grdn:0.150 lr:6.8e-05 updt_s:0.160 data_s:0.089 +INFO 2025-05-06 14:34:27 ts/train.py:232 step:345K smpl:6M ep:29K epch:95.02 loss:0.005 grdn:0.161 lr:6.8e-05 updt_s:0.160 data_s:0.086 +INFO 2025-05-06 14:35:16 ts/train.py:232 step:345K smpl:6M ep:29K epch:95.08 loss:0.005 grdn:0.155 lr:6.8e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-06 14:36:05 ts/train.py:232 step:345K smpl:6M ep:29K epch:95.14 loss:0.006 grdn:0.174 lr:6.8e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 14:36:54 ts/train.py:232 step:346K smpl:6M ep:29K epch:95.19 loss:0.006 grdn:0.161 lr:6.8e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 14:37:44 ts/train.py:232 step:346K smpl:6M ep:29K epch:95.25 loss:0.006 grdn:0.160 lr:6.8e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 14:38:33 ts/train.py:232 step:346K smpl:6M ep:29K epch:95.30 loss:0.005 grdn:0.149 lr:6.8e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 14:39:22 ts/train.py:232 step:346K smpl:6M ep:29K epch:95.36 loss:0.006 grdn:0.164 lr:6.8e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 14:40:11 ts/train.py:232 step:346K smpl:6M ep:29K epch:95.41 loss:0.005 grdn:0.155 lr:6.8e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 14:41:02 ts/train.py:232 step:347K smpl:6M ep:29K epch:95.47 loss:0.005 grdn:0.162 lr:6.8e-05 updt_s:0.159 data_s:0.097 +INFO 2025-05-06 14:41:51 ts/train.py:232 step:347K smpl:6M ep:29K epch:95.52 loss:0.005 grdn:0.153 lr:6.8e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 14:42:39 ts/train.py:232 step:347K smpl:6M ep:29K epch:95.58 loss:0.005 grdn:0.152 lr:6.8e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-06 14:43:27 ts/train.py:232 step:347K smpl:6M ep:29K epch:95.63 loss:0.005 grdn:0.154 lr:6.8e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 14:44:15 ts/train.py:232 step:347K smpl:6M ep:29K epch:95.69 loss:0.005 grdn:0.151 lr:6.8e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 14:45:02 ts/train.py:232 step:348K smpl:6M ep:29K epch:95.74 loss:0.006 grdn:0.164 lr:6.8e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 14:45:50 ts/train.py:232 step:348K smpl:6M ep:29K epch:95.80 loss:0.006 grdn:0.164 lr:6.8e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 14:46:38 ts/train.py:232 step:348K smpl:6M ep:29K epch:95.85 loss:0.005 grdn:0.160 lr:6.7e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 14:47:25 ts/train.py:232 step:348K smpl:6M ep:29K epch:95.91 loss:0.005 grdn:0.163 lr:6.7e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 14:48:11 ts/train.py:232 step:348K smpl:6M ep:29K epch:95.96 loss:0.006 grdn:0.157 lr:6.7e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-06 14:48:58 ts/train.py:232 step:349K smpl:6M ep:29K epch:96.02 loss:0.005 grdn:0.159 lr:6.7e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 14:49:45 ts/train.py:232 step:349K smpl:6M ep:29K epch:96.07 loss:0.005 grdn:0.164 lr:6.7e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 14:50:32 ts/train.py:232 step:349K smpl:6M ep:29K epch:96.13 loss:0.006 grdn:0.173 lr:6.7e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 14:51:19 ts/train.py:232 step:349K smpl:6M ep:29K epch:96.18 loss:0.005 grdn:0.158 lr:6.7e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 14:52:06 ts/train.py:232 step:349K smpl:6M ep:29K epch:96.24 loss:0.006 grdn:0.166 lr:6.7e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 14:52:59 ts/train.py:232 step:350K smpl:6M ep:29K epch:96.29 loss:0.006 grdn:0.178 lr:6.7e-05 updt_s:0.159 data_s:0.103 +INFO 2025-05-06 14:53:47 ts/train.py:232 step:350K smpl:6M ep:29K epch:96.35 loss:0.005 grdn:0.155 lr:6.7e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 14:54:33 ts/train.py:232 step:350K smpl:6M ep:29K epch:96.40 loss:0.005 grdn:0.151 lr:6.7e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-06 14:55:23 ts/train.py:232 step:350K smpl:6M ep:29K epch:96.46 loss:0.005 grdn:0.157 lr:6.7e-05 updt_s:0.159 data_s:0.092 +INFO 2025-05-06 14:56:13 ts/train.py:232 step:350K smpl:6M ep:29K epch:96.51 loss:0.005 grdn:0.145 lr:6.7e-05 updt_s:0.159 data_s:0.089 +INFO 2025-05-06 14:57:02 ts/train.py:232 step:351K smpl:6M ep:29K epch:96.57 loss:0.005 grdn:0.149 lr:6.7e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 14:57:50 ts/train.py:232 step:351K smpl:6M ep:29K epch:96.62 loss:0.005 grdn:0.157 lr:6.7e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 14:58:35 ts/train.py:232 step:351K smpl:6M ep:29K epch:96.68 loss:0.006 grdn:0.160 lr:6.7e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 14:59:21 ts/train.py:232 step:351K smpl:6M ep:29K epch:96.73 loss:0.006 grdn:0.160 lr:6.7e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-06 15:00:07 ts/train.py:232 step:351K smpl:6M ep:29K epch:96.79 loss:0.006 grdn:0.163 lr:6.7e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 15:00:54 ts/train.py:232 step:352K smpl:6M ep:29K epch:96.84 loss:0.005 grdn:0.152 lr:6.7e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 15:01:40 ts/train.py:232 step:352K smpl:6M ep:29K epch:96.90 loss:0.006 grdn:0.162 lr:6.7e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 15:02:26 ts/train.py:232 step:352K smpl:6M ep:29K epch:96.95 loss:0.005 grdn:0.147 lr:6.7e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-06 15:03:12 ts/train.py:232 step:352K smpl:6M ep:29K epch:97.01 loss:0.005 grdn:0.164 lr:6.7e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 15:03:57 ts/train.py:232 step:352K smpl:6M ep:29K epch:97.06 loss:0.005 grdn:0.156 lr:6.7e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 15:04:43 ts/train.py:232 step:353K smpl:6M ep:29K epch:97.12 loss:0.006 grdn:0.166 lr:6.7e-05 updt_s:0.160 data_s:0.068 +INFO 2025-05-06 15:05:29 ts/train.py:232 step:353K smpl:6M ep:29K epch:97.17 loss:0.005 grdn:0.154 lr:6.7e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-06 15:06:15 ts/train.py:232 step:353K smpl:6M ep:29K epch:97.23 loss:0.005 grdn:0.154 lr:6.7e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-06 15:07:01 ts/train.py:232 step:353K smpl:6M ep:29K epch:97.28 loss:0.005 grdn:0.155 lr:6.7e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 15:07:46 ts/train.py:232 step:353K smpl:6M ep:29K epch:97.34 loss:0.005 grdn:0.158 lr:6.7e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-06 15:08:35 ts/train.py:232 step:354K smpl:6M ep:29K epch:97.39 loss:0.005 grdn:0.150 lr:6.7e-05 updt_s:0.158 data_s:0.086 +INFO 2025-05-06 15:09:24 ts/train.py:232 step:354K smpl:6M ep:29K epch:97.45 loss:0.005 grdn:0.148 lr:6.7e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 15:10:13 ts/train.py:232 step:354K smpl:6M ep:29K epch:97.50 loss:0.004 grdn:0.144 lr:6.7e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 15:11:02 ts/train.py:232 step:354K smpl:6M ep:29K epch:97.56 loss:0.005 grdn:0.155 lr:6.6e-05 updt_s:0.160 data_s:0.080 +INFO 2025-05-06 15:11:50 ts/train.py:232 step:354K smpl:6M ep:29K epch:97.61 loss:0.005 grdn:0.155 lr:6.6e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 15:12:39 ts/train.py:232 step:355K smpl:6M ep:29K epch:97.67 loss:0.005 grdn:0.150 lr:6.6e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 15:13:28 ts/train.py:232 step:355K smpl:6M ep:29K epch:97.72 loss:0.005 grdn:0.159 lr:6.6e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-06 15:14:17 ts/train.py:232 step:355K smpl:6M ep:29K epch:97.78 loss:0.005 grdn:0.148 lr:6.6e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 15:15:06 ts/train.py:232 step:355K smpl:6M ep:29K epch:97.83 loss:0.005 grdn:0.158 lr:6.6e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-06 15:15:55 ts/train.py:232 step:355K smpl:6M ep:29K epch:97.89 loss:0.005 grdn:0.160 lr:6.6e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 15:16:44 ts/train.py:232 step:356K smpl:6M ep:29K epch:97.94 loss:0.005 grdn:0.162 lr:6.6e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-06 15:17:33 ts/train.py:232 step:356K smpl:6M ep:29K epch:98.00 loss:0.006 grdn:0.168 lr:6.6e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 15:18:22 ts/train.py:232 step:356K smpl:6M ep:29K epch:98.05 loss:0.005 grdn:0.150 lr:6.6e-05 updt_s:0.160 data_s:0.086 +INFO 2025-05-06 15:19:11 ts/train.py:232 step:356K smpl:6M ep:29K epch:98.11 loss:0.005 grdn:0.146 lr:6.6e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 15:20:00 ts/train.py:232 step:356K smpl:6M ep:29K epch:98.16 loss:0.005 grdn:0.152 lr:6.6e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 15:20:48 ts/train.py:232 step:357K smpl:6M ep:29K epch:98.22 loss:0.005 grdn:0.157 lr:6.6e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 15:21:38 ts/train.py:232 step:357K smpl:6M ep:29K epch:98.28 loss:0.005 grdn:0.161 lr:6.6e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 15:22:26 ts/train.py:232 step:357K smpl:6M ep:29K epch:98.33 loss:0.006 grdn:0.160 lr:6.6e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 15:23:20 ts/train.py:232 step:357K smpl:6M ep:30K epch:98.39 loss:0.005 grdn:0.160 lr:6.6e-05 updt_s:0.159 data_s:0.110 +INFO 2025-05-06 15:24:08 ts/train.py:232 step:357K smpl:6M ep:30K epch:98.44 loss:0.006 grdn:0.162 lr:6.6e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 15:24:56 ts/train.py:232 step:358K smpl:6M ep:30K epch:98.50 loss:0.005 grdn:0.156 lr:6.6e-05 updt_s:0.160 data_s:0.078 +INFO 2025-05-06 15:25:44 ts/train.py:232 step:358K smpl:6M ep:30K epch:98.55 loss:0.005 grdn:0.157 lr:6.6e-05 updt_s:0.161 data_s:0.077 +INFO 2025-05-06 15:26:32 ts/train.py:232 step:358K smpl:6M ep:30K epch:98.61 loss:0.005 grdn:0.160 lr:6.6e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 15:27:19 ts/train.py:232 step:358K smpl:6M ep:30K epch:98.66 loss:0.005 grdn:0.152 lr:6.6e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 15:28:08 ts/train.py:232 step:358K smpl:6M ep:30K epch:98.72 loss:0.005 grdn:0.160 lr:6.6e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 15:28:56 ts/train.py:232 step:359K smpl:6M ep:30K epch:98.77 loss:0.005 grdn:0.149 lr:6.6e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 15:29:44 ts/train.py:232 step:359K smpl:6M ep:30K epch:98.83 loss:0.005 grdn:0.157 lr:6.6e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 15:30:32 ts/train.py:232 step:359K smpl:6M ep:30K epch:98.88 loss:0.005 grdn:0.155 lr:6.6e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 15:31:20 ts/train.py:232 step:359K smpl:6M ep:30K epch:98.94 loss:0.005 grdn:0.157 lr:6.6e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 15:32:08 ts/train.py:232 step:359K smpl:6M ep:30K epch:98.99 loss:0.005 grdn:0.146 lr:6.6e-05 updt_s:0.160 data_s:0.080 +INFO 2025-05-06 15:32:56 ts/train.py:232 step:360K smpl:6M ep:30K epch:99.05 loss:0.005 grdn:0.162 lr:6.6e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 15:33:45 ts/train.py:232 step:360K smpl:6M ep:30K epch:99.10 loss:0.005 grdn:0.156 lr:6.6e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 15:34:34 ts/train.py:232 step:360K smpl:6M ep:30K epch:99.16 loss:0.005 grdn:0.151 lr:6.6e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 15:34:34 ts/train.py:241 Checkpoint policy after step 360000 +INFO 2025-05-06 15:35:29 ts/train.py:232 step:360K smpl:6M ep:30K epch:99.21 loss:0.005 grdn:0.158 lr:6.5e-05 updt_s:0.159 data_s:0.098 +INFO 2025-05-06 15:36:17 ts/train.py:232 step:360K smpl:6M ep:30K epch:99.27 loss:0.005 grdn:0.154 lr:6.5e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 15:37:07 ts/train.py:232 step:361K smpl:6M ep:30K epch:99.32 loss:0.005 grdn:0.151 lr:6.5e-05 updt_s:0.160 data_s:0.090 +INFO 2025-05-06 15:37:55 ts/train.py:232 step:361K smpl:6M ep:30K epch:99.38 loss:0.005 grdn:0.161 lr:6.5e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 15:38:44 ts/train.py:232 step:361K smpl:6M ep:30K epch:99.43 loss:0.006 grdn:0.166 lr:6.5e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 15:39:31 ts/train.py:232 step:361K smpl:6M ep:30K epch:99.49 loss:0.005 grdn:0.168 lr:6.5e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 15:40:19 ts/train.py:232 step:361K smpl:6M ep:30K epch:99.54 loss:0.005 grdn:0.155 lr:6.5e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 15:41:07 ts/train.py:232 step:362K smpl:6M ep:30K epch:99.60 loss:0.006 grdn:0.164 lr:6.5e-05 updt_s:0.160 data_s:0.080 +INFO 2025-05-06 15:41:55 ts/train.py:232 step:362K smpl:6M ep:30K epch:99.65 loss:0.006 grdn:0.159 lr:6.5e-05 updt_s:0.160 data_s:0.077 +INFO 2025-05-06 15:42:42 ts/train.py:232 step:362K smpl:6M ep:30K epch:99.71 loss:0.005 grdn:0.155 lr:6.5e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 15:43:30 ts/train.py:232 step:362K smpl:6M ep:30K epch:99.76 loss:0.006 grdn:0.162 lr:6.5e-05 updt_s:0.160 data_s:0.078 +INFO 2025-05-06 15:44:17 ts/train.py:232 step:362K smpl:6M ep:30K epch:99.82 loss:0.005 grdn:0.148 lr:6.5e-05 updt_s:0.160 data_s:0.076 +INFO 2025-05-06 15:45:05 ts/train.py:232 step:363K smpl:6M ep:30K epch:99.87 loss:0.005 grdn:0.152 lr:6.5e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 15:45:53 ts/train.py:232 step:363K smpl:6M ep:30K epch:99.93 loss:0.005 grdn:0.163 lr:6.5e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 15:46:40 ts/train.py:232 step:363K smpl:6M ep:30K epch:99.98 loss:0.006 grdn:0.161 lr:6.5e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 15:47:28 ts/train.py:232 step:363K smpl:6M ep:30K epch:100.04 loss:0.004 grdn:0.141 lr:6.5e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 15:48:15 ts/train.py:232 step:363K smpl:6M ep:30K epch:100.09 loss:0.005 grdn:0.148 lr:6.5e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 15:49:03 ts/train.py:232 step:364K smpl:6M ep:30K epch:100.15 loss:0.005 grdn:0.166 lr:6.5e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 15:49:50 ts/train.py:232 step:364K smpl:6M ep:30K epch:100.20 loss:0.005 grdn:0.161 lr:6.5e-05 updt_s:0.160 data_s:0.077 +INFO 2025-05-06 15:50:38 ts/train.py:232 step:364K smpl:6M ep:30K epch:100.26 loss:0.005 grdn:0.159 lr:6.5e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 15:51:27 ts/train.py:232 step:364K smpl:6M ep:30K epch:100.31 loss:0.005 grdn:0.152 lr:6.5e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 15:52:13 ts/train.py:232 step:364K smpl:6M ep:30K epch:100.37 loss:0.005 grdn:0.150 lr:6.5e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-06 15:52:59 ts/train.py:232 step:365K smpl:6M ep:30K epch:100.42 loss:0.005 grdn:0.151 lr:6.5e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 15:53:46 ts/train.py:232 step:365K smpl:6M ep:30K epch:100.48 loss:0.005 grdn:0.162 lr:6.5e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-06 15:54:32 ts/train.py:232 step:365K smpl:6M ep:30K epch:100.53 loss:0.005 grdn:0.159 lr:6.5e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-06 15:55:18 ts/train.py:232 step:365K smpl:6M ep:30K epch:100.59 loss:0.005 grdn:0.148 lr:6.5e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-06 15:56:03 ts/train.py:232 step:365K smpl:6M ep:30K epch:100.64 loss:0.005 grdn:0.150 lr:6.5e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-06 15:56:49 ts/train.py:232 step:366K smpl:6M ep:30K epch:100.70 loss:0.005 grdn:0.157 lr:6.5e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 15:57:33 ts/train.py:232 step:366K smpl:6M ep:30K epch:100.75 loss:0.005 grdn:0.151 lr:6.5e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-06 15:58:19 ts/train.py:232 step:366K smpl:6M ep:30K epch:100.81 loss:0.005 grdn:0.155 lr:6.5e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-06 15:59:04 ts/train.py:232 step:366K smpl:6M ep:30K epch:100.86 loss:0.005 grdn:0.146 lr:6.4e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-06 15:59:49 ts/train.py:232 step:366K smpl:6M ep:30K epch:100.92 loss:0.005 grdn:0.155 lr:6.4e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-06 16:00:34 ts/train.py:232 step:367K smpl:6M ep:30K epch:100.97 loss:0.005 grdn:0.149 lr:6.4e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-06 16:01:20 ts/train.py:232 step:367K smpl:6M ep:30K epch:101.03 loss:0.005 grdn:0.159 lr:6.4e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-06 16:02:05 ts/train.py:232 step:367K smpl:6M ep:30K epch:101.08 loss:0.005 grdn:0.160 lr:6.4e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 16:02:50 ts/train.py:232 step:367K smpl:6M ep:30K epch:101.14 loss:0.005 grdn:0.167 lr:6.4e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-06 16:03:36 ts/train.py:232 step:367K smpl:6M ep:30K epch:101.19 loss:0.005 grdn:0.148 lr:6.4e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 16:04:25 ts/train.py:232 step:368K smpl:6M ep:30K epch:101.25 loss:0.005 grdn:0.157 lr:6.4e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 16:05:13 ts/train.py:232 step:368K smpl:6M ep:30K epch:101.30 loss:0.005 grdn:0.150 lr:6.4e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 16:06:02 ts/train.py:232 step:368K smpl:6M ep:30K epch:101.36 loss:0.005 grdn:0.146 lr:6.4e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 16:06:51 ts/train.py:232 step:368K smpl:6M ep:30K epch:101.42 loss:0.005 grdn:0.151 lr:6.4e-05 updt_s:0.159 data_s:0.089 +INFO 2025-05-06 16:07:40 ts/train.py:232 step:368K smpl:6M ep:30K epch:101.47 loss:0.005 grdn:0.169 lr:6.4e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-06 16:08:30 ts/train.py:232 step:369K smpl:6M ep:30K epch:101.53 loss:0.006 grdn:0.160 lr:6.4e-05 updt_s:0.159 data_s:0.089 +INFO 2025-05-06 16:09:19 ts/train.py:232 step:369K smpl:6M ep:30K epch:101.58 loss:0.005 grdn:0.146 lr:6.4e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 16:10:08 ts/train.py:232 step:369K smpl:6M ep:30K epch:101.64 loss:0.005 grdn:0.158 lr:6.4e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 16:10:57 ts/train.py:232 step:369K smpl:6M ep:31K epch:101.69 loss:0.005 grdn:0.160 lr:6.4e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 16:11:45 ts/train.py:232 step:369K smpl:6M ep:31K epch:101.75 loss:0.005 grdn:0.148 lr:6.4e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-06 16:12:35 ts/train.py:232 step:370K smpl:6M ep:31K epch:101.80 loss:0.005 grdn:0.145 lr:6.4e-05 updt_s:0.158 data_s:0.092 +INFO 2025-05-06 16:13:25 ts/train.py:232 step:370K smpl:6M ep:31K epch:101.86 loss:0.005 grdn:0.160 lr:6.4e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-06 16:14:13 ts/train.py:232 step:370K smpl:6M ep:31K epch:101.91 loss:0.005 grdn:0.163 lr:6.4e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 16:15:02 ts/train.py:232 step:370K smpl:6M ep:31K epch:101.97 loss:0.005 grdn:0.156 lr:6.4e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 16:15:51 ts/train.py:232 step:370K smpl:6M ep:31K epch:102.02 loss:0.005 grdn:0.148 lr:6.4e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 16:16:40 ts/train.py:232 step:371K smpl:6M ep:31K epch:102.08 loss:0.006 grdn:0.166 lr:6.4e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 16:17:29 ts/train.py:232 step:371K smpl:6M ep:31K epch:102.13 loss:0.005 grdn:0.162 lr:6.4e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-06 16:18:17 ts/train.py:232 step:371K smpl:6M ep:31K epch:102.19 loss:0.006 grdn:0.161 lr:6.4e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-06 16:19:10 ts/train.py:232 step:371K smpl:6M ep:31K epch:102.24 loss:0.005 grdn:0.165 lr:6.4e-05 updt_s:0.158 data_s:0.104 +INFO 2025-05-06 16:20:00 ts/train.py:232 step:371K smpl:6M ep:31K epch:102.30 loss:0.005 grdn:0.153 lr:6.4e-05 updt_s:0.158 data_s:0.089 +INFO 2025-05-06 16:20:50 ts/train.py:232 step:372K smpl:6M ep:31K epch:102.35 loss:0.006 grdn:0.161 lr:6.4e-05 updt_s:0.159 data_s:0.092 +INFO 2025-05-06 16:21:40 ts/train.py:232 step:372K smpl:6M ep:31K epch:102.41 loss:0.005 grdn:0.157 lr:6.4e-05 updt_s:0.159 data_s:0.090 +INFO 2025-05-06 16:22:28 ts/train.py:232 step:372K smpl:6M ep:31K epch:102.46 loss:0.005 grdn:0.160 lr:6.4e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 16:23:18 ts/train.py:232 step:372K smpl:6M ep:31K epch:102.52 loss:0.006 grdn:0.173 lr:6.3e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-06 16:24:09 ts/train.py:232 step:372K smpl:6M ep:31K epch:102.57 loss:0.005 grdn:0.147 lr:6.3e-05 updt_s:0.159 data_s:0.092 +INFO 2025-05-06 16:24:58 ts/train.py:232 step:373K smpl:6M ep:31K epch:102.63 loss:0.005 grdn:0.149 lr:6.3e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 16:25:48 ts/train.py:232 step:373K smpl:6M ep:31K epch:102.68 loss:0.006 grdn:0.163 lr:6.3e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-06 16:26:37 ts/train.py:232 step:373K smpl:6M ep:31K epch:102.74 loss:0.005 grdn:0.148 lr:6.3e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 16:27:26 ts/train.py:232 step:373K smpl:6M ep:31K epch:102.79 loss:0.005 grdn:0.162 lr:6.3e-05 updt_s:0.159 data_s:0.090 +INFO 2025-05-06 16:28:16 ts/train.py:232 step:373K smpl:6M ep:31K epch:102.85 loss:0.005 grdn:0.157 lr:6.3e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 16:29:06 ts/train.py:232 step:374K smpl:6M ep:31K epch:102.90 loss:0.006 grdn:0.163 lr:6.3e-05 updt_s:0.159 data_s:0.092 +INFO 2025-05-06 16:29:56 ts/train.py:232 step:374K smpl:6M ep:31K epch:102.96 loss:0.005 grdn:0.152 lr:6.3e-05 updt_s:0.159 data_s:0.090 +INFO 2025-05-06 16:30:46 ts/train.py:232 step:374K smpl:6M ep:31K epch:103.01 loss:0.005 grdn:0.157 lr:6.3e-05 updt_s:0.159 data_s:0.090 +INFO 2025-05-06 16:31:35 ts/train.py:232 step:374K smpl:6M ep:31K epch:103.07 loss:0.006 grdn:0.171 lr:6.3e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 16:32:25 ts/train.py:232 step:374K smpl:6M ep:31K epch:103.12 loss:0.005 grdn:0.155 lr:6.3e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-06 16:33:15 ts/train.py:232 step:375K smpl:6M ep:31K epch:103.18 loss:0.005 grdn:0.147 lr:6.3e-05 updt_s:0.159 data_s:0.094 +INFO 2025-05-06 16:34:03 ts/train.py:232 step:375K smpl:6M ep:31K epch:103.23 loss:0.006 grdn:0.168 lr:6.3e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 16:34:50 ts/train.py:232 step:375K smpl:6M ep:31K epch:103.29 loss:0.004 grdn:0.132 lr:6.3e-05 updt_s:0.160 data_s:0.076 +INFO 2025-05-06 16:35:37 ts/train.py:232 step:375K smpl:6M ep:31K epch:103.34 loss:0.005 grdn:0.156 lr:6.3e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 16:36:24 ts/train.py:232 step:375K smpl:6M ep:31K epch:103.40 loss:0.005 grdn:0.153 lr:6.3e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 16:37:12 ts/train.py:232 step:376K smpl:6M ep:31K epch:103.45 loss:0.005 grdn:0.158 lr:6.3e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-06 16:37:58 ts/train.py:232 step:376K smpl:6M ep:31K epch:103.51 loss:0.005 grdn:0.147 lr:6.3e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 16:38:45 ts/train.py:232 step:376K smpl:6M ep:31K epch:103.56 loss:0.005 grdn:0.161 lr:6.3e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 16:39:32 ts/train.py:232 step:376K smpl:6M ep:31K epch:103.62 loss:0.005 grdn:0.161 lr:6.3e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 16:40:19 ts/train.py:232 step:376K smpl:6M ep:31K epch:103.67 loss:0.005 grdn:0.159 lr:6.3e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 16:41:07 ts/train.py:232 step:377K smpl:6M ep:31K epch:103.73 loss:0.005 grdn:0.153 lr:6.3e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 16:41:55 ts/train.py:232 step:377K smpl:6M ep:31K epch:103.78 loss:0.005 grdn:0.153 lr:6.3e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-06 16:42:42 ts/train.py:232 step:377K smpl:6M ep:31K epch:103.84 loss:0.005 grdn:0.160 lr:6.3e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 16:43:30 ts/train.py:232 step:377K smpl:6M ep:31K epch:103.89 loss:0.005 grdn:0.160 lr:6.3e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 16:44:17 ts/train.py:232 step:377K smpl:6M ep:31K epch:103.95 loss:0.005 grdn:0.151 lr:6.3e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 16:45:04 ts/train.py:232 step:378K smpl:6M ep:31K epch:104.00 loss:0.005 grdn:0.161 lr:6.3e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 16:45:51 ts/train.py:232 step:378K smpl:6M ep:31K epch:104.06 loss:0.006 grdn:0.163 lr:6.3e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 16:46:38 ts/train.py:232 step:378K smpl:6M ep:31K epch:104.11 loss:0.005 grdn:0.158 lr:6.3e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 16:47:29 ts/train.py:232 step:378K smpl:6M ep:31K epch:104.17 loss:0.005 grdn:0.146 lr:6.2e-05 updt_s:0.159 data_s:0.096 +INFO 2025-05-06 16:48:18 ts/train.py:232 step:378K smpl:6M ep:31K epch:104.22 loss:0.005 grdn:0.158 lr:6.2e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 16:49:08 ts/train.py:232 step:379K smpl:6M ep:31K epch:104.28 loss:0.005 grdn:0.149 lr:6.2e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 16:49:57 ts/train.py:232 step:379K smpl:6M ep:31K epch:104.33 loss:0.005 grdn:0.158 lr:6.2e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 16:50:45 ts/train.py:232 step:379K smpl:6M ep:31K epch:104.39 loss:0.005 grdn:0.166 lr:6.2e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 16:51:32 ts/train.py:232 step:379K smpl:6M ep:31K epch:104.44 loss:0.005 grdn:0.151 lr:6.2e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 16:52:20 ts/train.py:232 step:379K smpl:6M ep:31K epch:104.50 loss:0.005 grdn:0.158 lr:6.2e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 16:53:08 ts/train.py:232 step:380K smpl:6M ep:31K epch:104.56 loss:0.005 grdn:0.154 lr:6.2e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 16:53:56 ts/train.py:232 step:380K smpl:6M ep:31K epch:104.61 loss:0.005 grdn:0.164 lr:6.2e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 16:54:43 ts/train.py:232 step:380K smpl:6M ep:31K epch:104.67 loss:0.005 grdn:0.162 lr:6.2e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 16:55:31 ts/train.py:232 step:380K smpl:6M ep:31K epch:104.72 loss:0.005 grdn:0.154 lr:6.2e-05 updt_s:0.160 data_s:0.080 +INFO 2025-05-06 16:56:20 ts/train.py:232 step:380K smpl:6M ep:31K epch:104.78 loss:0.005 grdn:0.154 lr:6.2e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-06 16:57:08 ts/train.py:232 step:381K smpl:6M ep:31K epch:104.83 loss:0.005 grdn:0.166 lr:6.2e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 16:57:57 ts/train.py:232 step:381K smpl:6M ep:31K epch:104.89 loss:0.006 grdn:0.169 lr:6.2e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 16:58:46 ts/train.py:232 step:381K smpl:6M ep:31K epch:104.94 loss:0.005 grdn:0.150 lr:6.2e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-06 16:59:34 ts/train.py:232 step:381K smpl:6M ep:31K epch:105.00 loss:0.004 grdn:0.148 lr:6.2e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 17:00:22 ts/train.py:232 step:381K smpl:6M ep:32K epch:105.05 loss:0.005 grdn:0.150 lr:6.2e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 17:01:13 ts/train.py:232 step:382K smpl:6M ep:32K epch:105.11 loss:0.005 grdn:0.151 lr:6.2e-05 updt_s:0.158 data_s:0.093 +INFO 2025-05-06 17:02:03 ts/train.py:232 step:382K smpl:6M ep:32K epch:105.16 loss:0.005 grdn:0.155 lr:6.2e-05 updt_s:0.159 data_s:0.090 +INFO 2025-05-06 17:02:53 ts/train.py:232 step:382K smpl:6M ep:32K epch:105.22 loss:0.005 grdn:0.149 lr:6.2e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-06 17:03:42 ts/train.py:232 step:382K smpl:6M ep:32K epch:105.27 loss:0.005 grdn:0.163 lr:6.2e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-06 17:04:31 ts/train.py:232 step:382K smpl:6M ep:32K epch:105.33 loss:0.005 grdn:0.154 lr:6.2e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 17:05:20 ts/train.py:232 step:383K smpl:6M ep:32K epch:105.38 loss:0.004 grdn:0.147 lr:6.2e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 17:06:10 ts/train.py:232 step:383K smpl:6M ep:32K epch:105.44 loss:0.005 grdn:0.161 lr:6.2e-05 updt_s:0.160 data_s:0.088 +INFO 2025-05-06 17:06:59 ts/train.py:232 step:383K smpl:6M ep:32K epch:105.49 loss:0.005 grdn:0.161 lr:6.2e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 17:07:49 ts/train.py:232 step:383K smpl:6M ep:32K epch:105.55 loss:0.005 grdn:0.152 lr:6.2e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 17:08:38 ts/train.py:232 step:383K smpl:6M ep:32K epch:105.60 loss:0.005 grdn:0.153 lr:6.2e-05 updt_s:0.159 data_s:0.089 +INFO 2025-05-06 17:09:27 ts/train.py:232 step:384K smpl:6M ep:32K epch:105.66 loss:0.005 grdn:0.168 lr:6.2e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 17:10:17 ts/train.py:232 step:384K smpl:6M ep:32K epch:105.71 loss:0.005 grdn:0.156 lr:6.2e-05 updt_s:0.160 data_s:0.091 +INFO 2025-05-06 17:11:07 ts/train.py:232 step:384K smpl:6M ep:32K epch:105.77 loss:0.005 grdn:0.159 lr:6.1e-05 updt_s:0.159 data_s:0.089 +INFO 2025-05-06 17:11:55 ts/train.py:232 step:384K smpl:6M ep:32K epch:105.82 loss:0.005 grdn:0.153 lr:6.1e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 17:12:45 ts/train.py:232 step:384K smpl:6M ep:32K epch:105.88 loss:0.004 grdn:0.151 lr:6.1e-05 updt_s:0.160 data_s:0.087 +INFO 2025-05-06 17:13:33 ts/train.py:232 step:385K smpl:6M ep:32K epch:105.93 loss:0.005 grdn:0.153 lr:6.1e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 17:14:22 ts/train.py:232 step:385K smpl:6M ep:32K epch:105.99 loss:0.005 grdn:0.155 lr:6.1e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 17:15:12 ts/train.py:232 step:385K smpl:6M ep:32K epch:106.04 loss:0.005 grdn:0.152 lr:6.1e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-06 17:16:03 ts/train.py:232 step:385K smpl:6M ep:32K epch:106.10 loss:0.005 grdn:0.157 lr:6.1e-05 updt_s:0.158 data_s:0.096 +INFO 2025-05-06 17:16:51 ts/train.py:232 step:385K smpl:6M ep:32K epch:106.15 loss:0.006 grdn:0.166 lr:6.1e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 17:17:40 ts/train.py:232 step:386K smpl:6M ep:32K epch:106.21 loss:0.005 grdn:0.153 lr:6.1e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 17:18:29 ts/train.py:232 step:386K smpl:6M ep:32K epch:106.26 loss:0.005 grdn:0.157 lr:6.1e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-06 17:19:14 ts/train.py:232 step:386K smpl:6M ep:32K epch:106.32 loss:0.005 grdn:0.156 lr:6.1e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 17:20:01 ts/train.py:232 step:386K smpl:6M ep:32K epch:106.37 loss:0.005 grdn:0.155 lr:6.1e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-06 17:20:47 ts/train.py:232 step:386K smpl:6M ep:32K epch:106.43 loss:0.005 grdn:0.151 lr:6.1e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 17:21:33 ts/train.py:232 step:387K smpl:6M ep:32K epch:106.48 loss:0.004 grdn:0.142 lr:6.1e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 17:22:19 ts/train.py:232 step:387K smpl:6M ep:32K epch:106.54 loss:0.005 grdn:0.148 lr:6.1e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 17:23:04 ts/train.py:232 step:387K smpl:6M ep:32K epch:106.59 loss:0.005 grdn:0.163 lr:6.1e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 17:23:51 ts/train.py:232 step:387K smpl:6M ep:32K epch:106.65 loss:0.004 grdn:0.145 lr:6.1e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 17:24:36 ts/train.py:232 step:387K smpl:6M ep:32K epch:106.70 loss:0.004 grdn:0.151 lr:6.1e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-06 17:25:22 ts/train.py:232 step:388K smpl:6M ep:32K epch:106.76 loss:0.005 grdn:0.152 lr:6.1e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 17:26:08 ts/train.py:232 step:388K smpl:6M ep:32K epch:106.81 loss:0.004 grdn:0.144 lr:6.1e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 17:26:54 ts/train.py:232 step:388K smpl:6M ep:32K epch:106.87 loss:0.005 grdn:0.164 lr:6.1e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 17:27:40 ts/train.py:232 step:388K smpl:6M ep:32K epch:106.92 loss:0.005 grdn:0.164 lr:6.1e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-06 17:28:26 ts/train.py:232 step:388K smpl:6M ep:32K epch:106.98 loss:0.004 grdn:0.147 lr:6.1e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 17:29:13 ts/train.py:232 step:389K smpl:6M ep:32K epch:107.03 loss:0.005 grdn:0.151 lr:6.1e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-06 17:29:59 ts/train.py:232 step:389K smpl:6M ep:32K epch:107.09 loss:0.005 grdn:0.156 lr:6.1e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-06 17:30:44 ts/train.py:232 step:389K smpl:6M ep:32K epch:107.14 loss:0.004 grdn:0.142 lr:6.1e-05 updt_s:0.160 data_s:0.063 +INFO 2025-05-06 17:31:28 ts/train.py:232 step:389K smpl:6M ep:32K epch:107.20 loss:0.006 grdn:0.163 lr:6.1e-05 updt_s:0.159 data_s:0.063 +INFO 2025-05-06 17:32:14 ts/train.py:232 step:389K smpl:6M ep:32K epch:107.25 loss:0.005 grdn:0.151 lr:6.1e-05 updt_s:0.160 data_s:0.069 +INFO 2025-05-06 17:33:00 ts/train.py:232 step:390K smpl:6M ep:32K epch:107.31 loss:0.005 grdn:0.157 lr:6.1e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 17:33:45 ts/train.py:232 step:390K smpl:6M ep:32K epch:107.36 loss:0.006 grdn:0.174 lr:6.0e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 17:34:31 ts/train.py:232 step:390K smpl:6M ep:32K epch:107.42 loss:0.005 grdn:0.153 lr:6.0e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 17:34:31 ts/train.py:241 Checkpoint policy after step 390000 +INFO 2025-05-06 17:35:19 ts/train.py:232 step:390K smpl:6M ep:32K epch:107.47 loss:0.005 grdn:0.160 lr:6.0e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-06 17:36:04 ts/train.py:232 step:390K smpl:6M ep:32K epch:107.53 loss:0.004 grdn:0.144 lr:6.0e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-06 17:36:50 ts/train.py:232 step:391K smpl:6M ep:32K epch:107.58 loss:0.005 grdn:0.156 lr:6.0e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 17:37:35 ts/train.py:232 step:391K smpl:6M ep:32K epch:107.64 loss:0.005 grdn:0.153 lr:6.0e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-06 17:38:20 ts/train.py:232 step:391K smpl:6M ep:32K epch:107.69 loss:0.005 grdn:0.159 lr:6.0e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-06 17:39:04 ts/train.py:232 step:391K smpl:6M ep:32K epch:107.75 loss:0.005 grdn:0.153 lr:6.0e-05 updt_s:0.159 data_s:0.063 +INFO 2025-05-06 17:39:50 ts/train.py:232 step:391K smpl:6M ep:32K epch:107.81 loss:0.005 grdn:0.158 lr:6.0e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-06 17:40:35 ts/train.py:232 step:392K smpl:6M ep:32K epch:107.86 loss:0.004 grdn:0.143 lr:6.0e-05 updt_s:0.159 data_s:0.065 +INFO 2025-05-06 17:41:20 ts/train.py:232 step:392K smpl:6M ep:32K epch:107.92 loss:0.005 grdn:0.151 lr:6.0e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-06 17:42:05 ts/train.py:232 step:392K smpl:6M ep:32K epch:107.97 loss:0.005 grdn:0.163 lr:6.0e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 17:42:55 ts/train.py:232 step:392K smpl:6M ep:32K epch:108.03 loss:0.005 grdn:0.160 lr:6.0e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-06 17:43:43 ts/train.py:232 step:392K smpl:6M ep:32K epch:108.08 loss:0.005 grdn:0.159 lr:6.0e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 17:44:32 ts/train.py:232 step:393K smpl:6M ep:32K epch:108.14 loss:0.005 grdn:0.157 lr:6.0e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-06 17:45:22 ts/train.py:232 step:393K smpl:6M ep:32K epch:108.19 loss:0.005 grdn:0.152 lr:6.0e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-06 17:46:11 ts/train.py:232 step:393K smpl:6M ep:32K epch:108.25 loss:0.005 grdn:0.162 lr:6.0e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 17:47:00 ts/train.py:232 step:393K smpl:6M ep:32K epch:108.30 loss:0.005 grdn:0.157 lr:6.0e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 17:47:49 ts/train.py:232 step:393K smpl:6M ep:33K epch:108.36 loss:0.004 grdn:0.151 lr:6.0e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 17:48:38 ts/train.py:232 step:394K smpl:6M ep:33K epch:108.41 loss:0.005 grdn:0.155 lr:6.0e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 17:49:27 ts/train.py:232 step:394K smpl:6M ep:33K epch:108.47 loss:0.005 grdn:0.165 lr:6.0e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-06 17:50:16 ts/train.py:232 step:394K smpl:6M ep:33K epch:108.52 loss:0.005 grdn:0.155 lr:6.0e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 17:51:05 ts/train.py:232 step:394K smpl:6M ep:33K epch:108.58 loss:0.005 grdn:0.155 lr:6.0e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-06 17:51:53 ts/train.py:232 step:394K smpl:6M ep:33K epch:108.63 loss:0.004 grdn:0.144 lr:6.0e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 17:52:42 ts/train.py:232 step:395K smpl:6M ep:33K epch:108.69 loss:0.005 grdn:0.150 lr:6.0e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 17:53:31 ts/train.py:232 step:395K smpl:6M ep:33K epch:108.74 loss:0.005 grdn:0.161 lr:6.0e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 17:54:20 ts/train.py:232 step:395K smpl:6M ep:33K epch:108.80 loss:0.006 grdn:0.169 lr:6.0e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-06 17:55:08 ts/train.py:232 step:395K smpl:6M ep:33K epch:108.85 loss:0.005 grdn:0.158 lr:6.0e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 17:55:57 ts/train.py:232 step:395K smpl:6M ep:33K epch:108.91 loss:0.005 grdn:0.160 lr:6.0e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 17:56:48 ts/train.py:232 step:396K smpl:6M ep:33K epch:108.96 loss:0.005 grdn:0.148 lr:6.0e-05 updt_s:0.158 data_s:0.095 +INFO 2025-05-06 17:57:35 ts/train.py:232 step:396K smpl:6M ep:33K epch:109.02 loss:0.005 grdn:0.157 lr:5.9e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 17:58:23 ts/train.py:232 step:396K smpl:6M ep:33K epch:109.07 loss:0.005 grdn:0.159 lr:5.9e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 17:59:11 ts/train.py:232 step:396K smpl:6M ep:33K epch:109.13 loss:0.005 grdn:0.152 lr:5.9e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 17:59:59 ts/train.py:232 step:396K smpl:6M ep:33K epch:109.18 loss:0.005 grdn:0.159 lr:5.9e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 18:00:47 ts/train.py:232 step:397K smpl:6M ep:33K epch:109.24 loss:0.005 grdn:0.167 lr:5.9e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 18:01:36 ts/train.py:232 step:397K smpl:6M ep:33K epch:109.29 loss:0.004 grdn:0.148 lr:5.9e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 18:02:23 ts/train.py:232 step:397K smpl:6M ep:33K epch:109.35 loss:0.005 grdn:0.156 lr:5.9e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 18:03:11 ts/train.py:232 step:397K smpl:6M ep:33K epch:109.40 loss:0.005 grdn:0.153 lr:5.9e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 18:03:59 ts/train.py:232 step:397K smpl:6M ep:33K epch:109.46 loss:0.005 grdn:0.160 lr:5.9e-05 updt_s:0.160 data_s:0.078 +INFO 2025-05-06 18:04:47 ts/train.py:232 step:398K smpl:6M ep:33K epch:109.51 loss:0.005 grdn:0.160 lr:5.9e-05 updt_s:0.160 data_s:0.078 +INFO 2025-05-06 18:05:36 ts/train.py:232 step:398K smpl:6M ep:33K epch:109.57 loss:0.005 grdn:0.157 lr:5.9e-05 updt_s:0.160 data_s:0.085 +INFO 2025-05-06 18:06:24 ts/train.py:232 step:398K smpl:6M ep:33K epch:109.62 loss:0.004 grdn:0.145 lr:5.9e-05 updt_s:0.160 data_s:0.084 +INFO 2025-05-06 18:07:13 ts/train.py:232 step:398K smpl:6M ep:33K epch:109.68 loss:0.005 grdn:0.153 lr:5.9e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 18:08:02 ts/train.py:232 step:398K smpl:6M ep:33K epch:109.73 loss:0.005 grdn:0.157 lr:5.9e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 18:08:50 ts/train.py:232 step:399K smpl:6M ep:33K epch:109.79 loss:0.005 grdn:0.155 lr:5.9e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 18:09:39 ts/train.py:232 step:399K smpl:6M ep:33K epch:109.84 loss:0.005 grdn:0.158 lr:5.9e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 18:10:27 ts/train.py:232 step:399K smpl:6M ep:33K epch:109.90 loss:0.004 grdn:0.148 lr:5.9e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-06 18:11:23 ts/train.py:232 step:399K smpl:6M ep:33K epch:109.95 loss:0.005 grdn:0.167 lr:5.9e-05 updt_s:0.159 data_s:0.122 +INFO 2025-05-06 18:12:14 ts/train.py:232 step:399K smpl:6M ep:33K epch:110.01 loss:0.005 grdn:0.161 lr:5.9e-05 updt_s:0.160 data_s:0.093 +INFO 2025-05-06 18:13:05 ts/train.py:232 step:400K smpl:6M ep:33K epch:110.06 loss:0.005 grdn:0.154 lr:5.9e-05 updt_s:0.159 data_s:0.096 +INFO 2025-05-06 18:13:55 ts/train.py:232 step:400K smpl:6M ep:33K epch:110.12 loss:0.004 grdn:0.147 lr:5.9e-05 updt_s:0.159 data_s:0.090 +INFO 2025-05-06 18:14:45 ts/train.py:232 step:400K smpl:6M ep:33K epch:110.17 loss:0.004 grdn:0.147 lr:5.9e-05 updt_s:0.159 data_s:0.089 +INFO 2025-05-06 18:15:34 ts/train.py:232 step:400K smpl:6M ep:33K epch:110.23 loss:0.005 grdn:0.154 lr:5.9e-05 updt_s:0.158 data_s:0.089 +INFO 2025-05-06 18:16:25 ts/train.py:232 step:400K smpl:6M ep:33K epch:110.28 loss:0.005 grdn:0.147 lr:5.9e-05 updt_s:0.158 data_s:0.095 +INFO 2025-05-06 18:17:12 ts/train.py:232 step:401K smpl:6M ep:33K epch:110.34 loss:0.005 grdn:0.157 lr:5.9e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 18:18:03 ts/train.py:232 step:401K smpl:6M ep:33K epch:110.39 loss:0.005 grdn:0.152 lr:5.9e-05 updt_s:0.159 data_s:0.093 +INFO 2025-05-06 18:18:52 ts/train.py:232 step:401K smpl:6M ep:33K epch:110.45 loss:0.005 grdn:0.155 lr:5.9e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 18:19:40 ts/train.py:232 step:401K smpl:6M ep:33K epch:110.50 loss:0.004 grdn:0.144 lr:5.9e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 18:20:29 ts/train.py:232 step:401K smpl:6M ep:33K epch:110.56 loss:0.004 grdn:0.153 lr:5.9e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 18:21:19 ts/train.py:232 step:402K smpl:6M ep:33K epch:110.61 loss:0.005 grdn:0.161 lr:5.8e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-06 18:22:07 ts/train.py:232 step:402K smpl:6M ep:33K epch:110.67 loss:0.005 grdn:0.160 lr:5.8e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 18:22:56 ts/train.py:232 step:402K smpl:6M ep:33K epch:110.72 loss:0.005 grdn:0.160 lr:5.8e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 18:23:46 ts/train.py:232 step:402K smpl:6M ep:33K epch:110.78 loss:0.005 grdn:0.159 lr:5.8e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-06 18:24:34 ts/train.py:232 step:402K smpl:6M ep:33K epch:110.83 loss:0.004 grdn:0.148 lr:5.8e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 18:25:22 ts/train.py:232 step:403K smpl:6M ep:33K epch:110.89 loss:0.004 grdn:0.147 lr:5.8e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 18:26:11 ts/train.py:232 step:403K smpl:6M ep:33K epch:110.95 loss:0.004 grdn:0.147 lr:5.8e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 18:26:59 ts/train.py:232 step:403K smpl:6M ep:33K epch:111.00 loss:0.005 grdn:0.159 lr:5.8e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 18:27:47 ts/train.py:232 step:403K smpl:6M ep:33K epch:111.06 loss:0.005 grdn:0.154 lr:5.8e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 18:28:35 ts/train.py:232 step:403K smpl:6M ep:33K epch:111.11 loss:0.005 grdn:0.157 lr:5.8e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 18:29:24 ts/train.py:232 step:404K smpl:6M ep:33K epch:111.17 loss:0.005 grdn:0.160 lr:5.8e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 18:30:11 ts/train.py:232 step:404K smpl:6M ep:33K epch:111.22 loss:0.005 grdn:0.161 lr:5.8e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 18:30:59 ts/train.py:232 step:404K smpl:6M ep:33K epch:111.28 loss:0.004 grdn:0.139 lr:5.8e-05 updt_s:0.160 data_s:0.081 +INFO 2025-05-06 18:31:48 ts/train.py:232 step:404K smpl:6M ep:33K epch:111.33 loss:0.004 grdn:0.144 lr:5.8e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-06 18:32:36 ts/train.py:232 step:404K smpl:6M ep:33K epch:111.39 loss:0.005 grdn:0.160 lr:5.8e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 18:33:25 ts/train.py:232 step:405K smpl:6M ep:33K epch:111.44 loss:0.004 grdn:0.149 lr:5.8e-05 updt_s:0.160 data_s:0.082 +INFO 2025-05-06 18:34:13 ts/train.py:232 step:405K smpl:6M ep:33K epch:111.50 loss:0.005 grdn:0.155 lr:5.8e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 18:35:02 ts/train.py:232 step:405K smpl:6M ep:33K epch:111.55 loss:0.005 grdn:0.157 lr:5.8e-05 updt_s:0.160 data_s:0.083 +INFO 2025-05-06 18:35:50 ts/train.py:232 step:405K smpl:6M ep:33K epch:111.61 loss:0.005 grdn:0.174 lr:5.8e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 18:36:38 ts/train.py:232 step:405K smpl:6M ep:33K epch:111.66 loss:0.005 grdn:0.167 lr:5.8e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 18:37:26 ts/train.py:232 step:406K smpl:6M ep:34K epch:111.72 loss:0.004 grdn:0.150 lr:5.8e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 18:38:13 ts/train.py:232 step:406K smpl:6M ep:34K epch:111.77 loss:0.005 grdn:0.152 lr:5.8e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 18:39:00 ts/train.py:232 step:406K smpl:6M ep:34K epch:111.83 loss:0.005 grdn:0.152 lr:5.8e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 18:39:52 ts/train.py:232 step:406K smpl:6M ep:34K epch:111.88 loss:0.004 grdn:0.152 lr:5.8e-05 updt_s:0.161 data_s:0.097 +INFO 2025-05-06 18:40:40 ts/train.py:232 step:406K smpl:7M ep:34K epch:111.94 loss:0.005 grdn:0.158 lr:5.8e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 18:41:29 ts/train.py:232 step:407K smpl:7M ep:34K epch:111.99 loss:0.005 grdn:0.167 lr:5.8e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 18:42:16 ts/train.py:232 step:407K smpl:7M ep:34K epch:112.05 loss:0.005 grdn:0.151 lr:5.8e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 18:43:02 ts/train.py:232 step:407K smpl:7M ep:34K epch:112.10 loss:0.005 grdn:0.157 lr:5.8e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 18:43:50 ts/train.py:232 step:407K smpl:7M ep:34K epch:112.16 loss:0.005 grdn:0.157 lr:5.8e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 18:44:38 ts/train.py:232 step:407K smpl:7M ep:34K epch:112.21 loss:0.004 grdn:0.141 lr:5.7e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 18:45:25 ts/train.py:232 step:408K smpl:7M ep:34K epch:112.27 loss:0.005 grdn:0.149 lr:5.7e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 18:46:13 ts/train.py:232 step:408K smpl:7M ep:34K epch:112.32 loss:0.005 grdn:0.156 lr:5.7e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 18:47:00 ts/train.py:232 step:408K smpl:7M ep:34K epch:112.38 loss:0.005 grdn:0.160 lr:5.7e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 18:47:46 ts/train.py:232 step:408K smpl:7M ep:34K epch:112.43 loss:0.005 grdn:0.153 lr:5.7e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 18:48:33 ts/train.py:232 step:408K smpl:7M ep:34K epch:112.49 loss:0.005 grdn:0.153 lr:5.7e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 18:49:21 ts/train.py:232 step:409K smpl:7M ep:34K epch:112.54 loss:0.005 grdn:0.157 lr:5.7e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 18:50:08 ts/train.py:232 step:409K smpl:7M ep:34K epch:112.60 loss:0.005 grdn:0.164 lr:5.7e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-06 18:50:55 ts/train.py:232 step:409K smpl:7M ep:34K epch:112.65 loss:0.005 grdn:0.156 lr:5.7e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 18:51:41 ts/train.py:232 step:409K smpl:7M ep:34K epch:112.71 loss:0.005 grdn:0.155 lr:5.7e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 18:52:28 ts/train.py:232 step:409K smpl:7M ep:34K epch:112.76 loss:0.005 grdn:0.152 lr:5.7e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 18:53:18 ts/train.py:232 step:410K smpl:7M ep:34K epch:112.82 loss:0.004 grdn:0.140 lr:5.7e-05 updt_s:0.158 data_s:0.090 +INFO 2025-05-06 18:54:07 ts/train.py:232 step:410K smpl:7M ep:34K epch:112.87 loss:0.005 grdn:0.152 lr:5.7e-05 updt_s:0.158 data_s:0.086 +INFO 2025-05-06 18:54:56 ts/train.py:232 step:410K smpl:7M ep:34K epch:112.93 loss:0.004 grdn:0.152 lr:5.7e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 18:55:44 ts/train.py:232 step:410K smpl:7M ep:34K epch:112.98 loss:0.005 grdn:0.152 lr:5.7e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-06 18:56:32 ts/train.py:232 step:410K smpl:7M ep:34K epch:113.04 loss:0.005 grdn:0.160 lr:5.7e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 18:57:20 ts/train.py:232 step:411K smpl:7M ep:34K epch:113.09 loss:0.005 grdn:0.148 lr:5.7e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 18:58:09 ts/train.py:232 step:411K smpl:7M ep:34K epch:113.15 loss:0.005 grdn:0.167 lr:5.7e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 18:58:58 ts/train.py:232 step:411K smpl:7M ep:34K epch:113.20 loss:0.005 grdn:0.159 lr:5.7e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 18:59:46 ts/train.py:232 step:411K smpl:7M ep:34K epch:113.26 loss:0.005 grdn:0.153 lr:5.7e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-06 19:00:34 ts/train.py:232 step:411K smpl:7M ep:34K epch:113.31 loss:0.005 grdn:0.156 lr:5.7e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 19:01:22 ts/train.py:232 step:412K smpl:7M ep:34K epch:113.37 loss:0.005 grdn:0.160 lr:5.7e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 19:02:10 ts/train.py:232 step:412K smpl:7M ep:34K epch:113.42 loss:0.005 grdn:0.163 lr:5.7e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 19:02:58 ts/train.py:232 step:412K smpl:7M ep:34K epch:113.48 loss:0.004 grdn:0.150 lr:5.7e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 19:03:46 ts/train.py:232 step:412K smpl:7M ep:34K epch:113.53 loss:0.005 grdn:0.156 lr:5.7e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 19:04:35 ts/train.py:232 step:412K smpl:7M ep:34K epch:113.59 loss:0.005 grdn:0.162 lr:5.7e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 19:05:24 ts/train.py:232 step:413K smpl:7M ep:34K epch:113.64 loss:0.005 grdn:0.164 lr:5.7e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 19:06:12 ts/train.py:232 step:413K smpl:7M ep:34K epch:113.70 loss:0.005 grdn:0.161 lr:5.7e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 19:07:00 ts/train.py:232 step:413K smpl:7M ep:34K epch:113.75 loss:0.005 grdn:0.162 lr:5.7e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 19:07:50 ts/train.py:232 step:413K smpl:7M ep:34K epch:113.81 loss:0.004 grdn:0.142 lr:5.6e-05 updt_s:0.159 data_s:0.089 +INFO 2025-05-06 19:08:37 ts/train.py:232 step:413K smpl:7M ep:34K epch:113.86 loss:0.004 grdn:0.147 lr:5.6e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 19:09:24 ts/train.py:232 step:414K smpl:7M ep:34K epch:113.92 loss:0.004 grdn:0.142 lr:5.6e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 19:10:11 ts/train.py:232 step:414K smpl:7M ep:34K epch:113.97 loss:0.005 grdn:0.161 lr:5.6e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 19:10:58 ts/train.py:232 step:414K smpl:7M ep:34K epch:114.03 loss:0.005 grdn:0.155 lr:5.6e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 19:11:45 ts/train.py:232 step:414K smpl:7M ep:34K epch:114.09 loss:0.005 grdn:0.153 lr:5.6e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 19:12:32 ts/train.py:232 step:414K smpl:7M ep:34K epch:114.14 loss:0.005 grdn:0.156 lr:5.6e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 19:13:19 ts/train.py:232 step:415K smpl:7M ep:34K epch:114.20 loss:0.004 grdn:0.149 lr:5.6e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-06 19:14:06 ts/train.py:232 step:415K smpl:7M ep:34K epch:114.25 loss:0.005 grdn:0.161 lr:5.6e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 19:14:53 ts/train.py:232 step:415K smpl:7M ep:34K epch:114.31 loss:0.004 grdn:0.150 lr:5.6e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-06 19:15:40 ts/train.py:232 step:415K smpl:7M ep:34K epch:114.36 loss:0.004 grdn:0.149 lr:5.6e-05 updt_s:0.160 data_s:0.079 +INFO 2025-05-06 19:16:27 ts/train.py:232 step:415K smpl:7M ep:34K epch:114.42 loss:0.005 grdn:0.153 lr:5.6e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 19:17:14 ts/train.py:232 step:416K smpl:7M ep:34K epch:114.47 loss:0.005 grdn:0.163 lr:5.6e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 19:18:01 ts/train.py:232 step:416K smpl:7M ep:34K epch:114.53 loss:0.004 grdn:0.146 lr:5.6e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 19:18:49 ts/train.py:232 step:416K smpl:7M ep:34K epch:114.58 loss:0.005 grdn:0.155 lr:5.6e-05 updt_s:0.160 data_s:0.077 +INFO 2025-05-06 19:19:37 ts/train.py:232 step:416K smpl:7M ep:34K epch:114.64 loss:0.004 grdn:0.150 lr:5.6e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 19:20:23 ts/train.py:232 step:416K smpl:7M ep:34K epch:114.69 loss:0.005 grdn:0.166 lr:5.6e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 19:21:10 ts/train.py:232 step:417K smpl:7M ep:34K epch:114.75 loss:0.005 grdn:0.152 lr:5.6e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 19:21:58 ts/train.py:232 step:417K smpl:7M ep:34K epch:114.80 loss:0.005 grdn:0.153 lr:5.6e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 19:22:46 ts/train.py:232 step:417K smpl:7M ep:34K epch:114.86 loss:0.005 grdn:0.156 lr:5.6e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-06 19:23:34 ts/train.py:232 step:417K smpl:7M ep:34K epch:114.91 loss:0.005 grdn:0.162 lr:5.6e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 19:24:21 ts/train.py:232 step:417K smpl:7M ep:34K epch:114.97 loss:0.004 grdn:0.151 lr:5.6e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 19:25:09 ts/train.py:232 step:418K smpl:7M ep:35K epch:115.02 loss:0.004 grdn:0.150 lr:5.6e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 19:25:58 ts/train.py:232 step:418K smpl:7M ep:35K epch:115.08 loss:0.004 grdn:0.149 lr:5.6e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 19:26:48 ts/train.py:232 step:418K smpl:7M ep:35K epch:115.13 loss:0.005 grdn:0.166 lr:5.6e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 19:27:35 ts/train.py:232 step:418K smpl:7M ep:35K epch:115.19 loss:0.004 grdn:0.143 lr:5.6e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-06 19:28:23 ts/train.py:232 step:418K smpl:7M ep:35K epch:115.24 loss:0.005 grdn:0.158 lr:5.6e-05 updt_s:0.159 data_s:0.081 +INFO 2025-05-06 19:29:10 ts/train.py:232 step:419K smpl:7M ep:35K epch:115.30 loss:0.005 grdn:0.158 lr:5.6e-05 updt_s:0.160 data_s:0.074 +INFO 2025-05-06 19:29:57 ts/train.py:232 step:419K smpl:7M ep:35K epch:115.35 loss:0.005 grdn:0.160 lr:5.5e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 19:30:44 ts/train.py:232 step:419K smpl:7M ep:35K epch:115.41 loss:0.005 grdn:0.151 lr:5.5e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 19:31:32 ts/train.py:232 step:419K smpl:7M ep:35K epch:115.46 loss:0.005 grdn:0.163 lr:5.5e-05 updt_s:0.160 data_s:0.077 +INFO 2025-05-06 19:32:18 ts/train.py:232 step:419K smpl:7M ep:35K epch:115.52 loss:0.004 grdn:0.146 lr:5.5e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 19:33:04 ts/train.py:232 step:420K smpl:7M ep:35K epch:115.57 loss:0.004 grdn:0.154 lr:5.5e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-06 19:33:52 ts/train.py:232 step:420K smpl:7M ep:35K epch:115.63 loss:0.005 grdn:0.154 lr:5.5e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 19:34:38 ts/train.py:232 step:420K smpl:7M ep:35K epch:115.68 loss:0.005 grdn:0.168 lr:5.5e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 19:34:38 ts/train.py:241 Checkpoint policy after step 420000 +INFO 2025-05-06 19:35:30 ts/train.py:232 step:420K smpl:7M ep:35K epch:115.74 loss:0.004 grdn:0.158 lr:5.5e-05 updt_s:0.158 data_s:0.086 +INFO 2025-05-06 19:36:17 ts/train.py:232 step:420K smpl:7M ep:35K epch:115.79 loss:0.004 grdn:0.148 lr:5.5e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 19:37:03 ts/train.py:232 step:421K smpl:7M ep:35K epch:115.85 loss:0.005 grdn:0.157 lr:5.5e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-06 19:37:49 ts/train.py:232 step:421K smpl:7M ep:35K epch:115.90 loss:0.005 grdn:0.155 lr:5.5e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-06 19:38:35 ts/train.py:232 step:421K smpl:7M ep:35K epch:115.96 loss:0.005 grdn:0.162 lr:5.5e-05 updt_s:0.160 data_s:0.068 +INFO 2025-05-06 19:39:20 ts/train.py:232 step:421K smpl:7M ep:35K epch:116.01 loss:0.005 grdn:0.164 lr:5.5e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 19:40:05 ts/train.py:232 step:421K smpl:7M ep:35K epch:116.07 loss:0.004 grdn:0.151 lr:5.5e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 19:40:51 ts/train.py:232 step:422K smpl:7M ep:35K epch:116.12 loss:0.004 grdn:0.145 lr:5.5e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-06 19:41:37 ts/train.py:232 step:422K smpl:7M ep:35K epch:116.18 loss:0.004 grdn:0.158 lr:5.5e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 19:42:22 ts/train.py:232 step:422K smpl:7M ep:35K epch:116.23 loss:0.005 grdn:0.156 lr:5.5e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 19:43:08 ts/train.py:232 step:422K smpl:7M ep:35K epch:116.29 loss:0.004 grdn:0.152 lr:5.5e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-06 19:43:55 ts/train.py:232 step:422K smpl:7M ep:35K epch:116.34 loss:0.005 grdn:0.157 lr:5.5e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-06 19:44:40 ts/train.py:232 step:423K smpl:7M ep:35K epch:116.40 loss:0.004 grdn:0.149 lr:5.5e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 19:45:27 ts/train.py:232 step:423K smpl:7M ep:35K epch:116.45 loss:0.005 grdn:0.152 lr:5.5e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-06 19:46:12 ts/train.py:232 step:423K smpl:7M ep:35K epch:116.51 loss:0.005 grdn:0.153 lr:5.5e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 19:46:59 ts/train.py:232 step:423K smpl:7M ep:35K epch:116.56 loss:0.004 grdn:0.144 lr:5.5e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 19:47:46 ts/train.py:232 step:423K smpl:7M ep:35K epch:116.62 loss:0.005 grdn:0.161 lr:5.5e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-06 19:48:34 ts/train.py:232 step:424K smpl:7M ep:35K epch:116.67 loss:0.004 grdn:0.153 lr:5.5e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-06 19:49:19 ts/train.py:232 step:424K smpl:7M ep:35K epch:116.73 loss:0.005 grdn:0.160 lr:5.5e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-06 19:50:04 ts/train.py:232 step:424K smpl:7M ep:35K epch:116.78 loss:0.005 grdn:0.162 lr:5.5e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-06 19:50:49 ts/train.py:232 step:424K smpl:7M ep:35K epch:116.84 loss:0.005 grdn:0.153 lr:5.5e-05 updt_s:0.158 data_s:0.065 +INFO 2025-05-06 19:51:35 ts/train.py:232 step:424K smpl:7M ep:35K epch:116.89 loss:0.005 grdn:0.152 lr:5.5e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 19:52:20 ts/train.py:232 step:425K smpl:7M ep:35K epch:116.95 loss:0.005 grdn:0.161 lr:5.4e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 19:53:08 ts/train.py:232 step:425K smpl:7M ep:35K epch:117.00 loss:0.004 grdn:0.143 lr:5.4e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-06 19:53:56 ts/train.py:232 step:425K smpl:7M ep:35K epch:117.06 loss:0.004 grdn:0.143 lr:5.4e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-06 19:54:43 ts/train.py:232 step:425K smpl:7M ep:35K epch:117.11 loss:0.004 grdn:0.150 lr:5.4e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-06 19:55:28 ts/train.py:232 step:425K smpl:7M ep:35K epch:117.17 loss:0.004 grdn:0.151 lr:5.4e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 19:56:17 ts/train.py:232 step:426K smpl:7M ep:35K epch:117.22 loss:0.005 grdn:0.168 lr:5.4e-05 updt_s:0.158 data_s:0.086 +INFO 2025-05-06 19:57:05 ts/train.py:232 step:426K smpl:7M ep:35K epch:117.28 loss:0.005 grdn:0.159 lr:5.4e-05 updt_s:0.159 data_s:0.079 +INFO 2025-05-06 19:57:52 ts/train.py:232 step:426K smpl:7M ep:35K epch:117.34 loss:0.004 grdn:0.154 lr:5.4e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 19:58:39 ts/train.py:232 step:426K smpl:7M ep:35K epch:117.39 loss:0.005 grdn:0.174 lr:5.4e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 19:59:27 ts/train.py:232 step:426K smpl:7M ep:35K epch:117.45 loss:0.005 grdn:0.147 lr:5.4e-05 updt_s:0.159 data_s:0.085 +INFO 2025-05-06 20:00:15 ts/train.py:232 step:427K smpl:7M ep:35K epch:117.50 loss:0.005 grdn:0.155 lr:5.4e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-06 20:01:01 ts/train.py:232 step:427K smpl:7M ep:35K epch:117.56 loss:0.004 grdn:0.151 lr:5.4e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 20:01:48 ts/train.py:232 step:427K smpl:7M ep:35K epch:117.61 loss:0.004 grdn:0.145 lr:5.4e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 20:02:37 ts/train.py:232 step:427K smpl:7M ep:35K epch:117.67 loss:0.005 grdn:0.154 lr:5.4e-05 updt_s:0.158 data_s:0.087 +INFO 2025-05-06 20:03:24 ts/train.py:232 step:427K smpl:7M ep:35K epch:117.72 loss:0.005 grdn:0.163 lr:5.4e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 20:04:11 ts/train.py:232 step:428K smpl:7M ep:35K epch:117.78 loss:0.005 grdn:0.156 lr:5.4e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-06 20:04:58 ts/train.py:232 step:428K smpl:7M ep:35K epch:117.83 loss:0.005 grdn:0.159 lr:5.4e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 20:05:44 ts/train.py:232 step:428K smpl:7M ep:35K epch:117.89 loss:0.005 grdn:0.158 lr:5.4e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-06 20:06:30 ts/train.py:232 step:428K smpl:7M ep:35K epch:117.94 loss:0.004 grdn:0.151 lr:5.4e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 20:07:16 ts/train.py:232 step:428K smpl:7M ep:35K epch:118.00 loss:0.004 grdn:0.148 lr:5.4e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:08:02 ts/train.py:232 step:429K smpl:7M ep:35K epch:118.05 loss:0.005 grdn:0.158 lr:5.4e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 20:08:50 ts/train.py:232 step:429K smpl:7M ep:35K epch:118.11 loss:0.004 grdn:0.147 lr:5.4e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-06 20:09:39 ts/train.py:232 step:429K smpl:7M ep:35K epch:118.16 loss:0.004 grdn:0.150 lr:5.4e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-06 20:10:27 ts/train.py:232 step:429K smpl:7M ep:35K epch:118.22 loss:0.004 grdn:0.149 lr:5.4e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 20:11:13 ts/train.py:232 step:429K smpl:7M ep:35K epch:118.27 loss:0.004 grdn:0.148 lr:5.4e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:11:58 ts/train.py:232 step:430K smpl:7M ep:35K epch:118.33 loss:0.005 grdn:0.154 lr:5.4e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 20:12:45 ts/train.py:232 step:430K smpl:7M ep:36K epch:118.38 loss:0.004 grdn:0.151 lr:5.4e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-06 20:13:32 ts/train.py:232 step:430K smpl:7M ep:36K epch:118.44 loss:0.004 grdn:0.149 lr:5.4e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 20:14:17 ts/train.py:232 step:430K smpl:7M ep:36K epch:118.49 loss:0.005 grdn:0.164 lr:5.4e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:15:03 ts/train.py:232 step:430K smpl:7M ep:36K epch:118.55 loss:0.005 grdn:0.157 lr:5.3e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-06 20:15:51 ts/train.py:232 step:431K smpl:7M ep:36K epch:118.60 loss:0.005 grdn:0.159 lr:5.3e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-06 20:16:37 ts/train.py:232 step:431K smpl:7M ep:36K epch:118.66 loss:0.004 grdn:0.145 lr:5.3e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:17:24 ts/train.py:232 step:431K smpl:7M ep:36K epch:118.71 loss:0.004 grdn:0.148 lr:5.3e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-06 20:18:10 ts/train.py:232 step:431K smpl:7M ep:36K epch:118.77 loss:0.004 grdn:0.148 lr:5.3e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:18:54 ts/train.py:232 step:431K smpl:7M ep:36K epch:118.82 loss:0.005 grdn:0.160 lr:5.3e-05 updt_s:0.158 data_s:0.065 +INFO 2025-05-06 20:19:41 ts/train.py:232 step:432K smpl:7M ep:36K epch:118.88 loss:0.004 grdn:0.153 lr:5.3e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 20:20:27 ts/train.py:232 step:432K smpl:7M ep:36K epch:118.93 loss:0.004 grdn:0.164 lr:5.3e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:21:13 ts/train.py:232 step:432K smpl:7M ep:36K epch:118.99 loss:0.004 grdn:0.142 lr:5.3e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-06 20:21:59 ts/train.py:232 step:432K smpl:7M ep:36K epch:119.04 loss:0.005 grdn:0.170 lr:5.3e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-06 20:22:44 ts/train.py:232 step:432K smpl:7M ep:36K epch:119.10 loss:0.004 grdn:0.146 lr:5.3e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:23:30 ts/train.py:232 step:433K smpl:7M ep:36K epch:119.15 loss:0.004 grdn:0.155 lr:5.3e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 20:24:15 ts/train.py:232 step:433K smpl:7M ep:36K epch:119.21 loss:0.004 grdn:0.155 lr:5.3e-05 updt_s:0.160 data_s:0.065 +INFO 2025-05-06 20:25:01 ts/train.py:232 step:433K smpl:7M ep:36K epch:119.26 loss:0.005 grdn:0.164 lr:5.3e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-06 20:25:47 ts/train.py:232 step:433K smpl:7M ep:36K epch:119.32 loss:0.004 grdn:0.141 lr:5.3e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:26:33 ts/train.py:232 step:433K smpl:7M ep:36K epch:119.37 loss:0.004 grdn:0.150 lr:5.3e-05 updt_s:0.159 data_s:0.071 +INFO 2025-05-06 20:27:22 ts/train.py:232 step:434K smpl:7M ep:36K epch:119.43 loss:0.005 grdn:0.152 lr:5.3e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 20:28:09 ts/train.py:232 step:434K smpl:7M ep:36K epch:119.48 loss:0.004 grdn:0.145 lr:5.3e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 20:28:55 ts/train.py:232 step:434K smpl:7M ep:36K epch:119.54 loss:0.004 grdn:0.153 lr:5.3e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:29:42 ts/train.py:232 step:434K smpl:7M ep:36K epch:119.59 loss:0.004 grdn:0.146 lr:5.3e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-06 20:30:28 ts/train.py:232 step:434K smpl:7M ep:36K epch:119.65 loss:0.005 grdn:0.158 lr:5.3e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:31:13 ts/train.py:232 step:435K smpl:7M ep:36K epch:119.70 loss:0.004 grdn:0.155 lr:5.3e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:32:00 ts/train.py:232 step:435K smpl:7M ep:36K epch:119.76 loss:0.004 grdn:0.144 lr:5.3e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 20:32:45 ts/train.py:232 step:435K smpl:7M ep:36K epch:119.81 loss:0.005 grdn:0.164 lr:5.3e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 20:33:30 ts/train.py:232 step:435K smpl:7M ep:36K epch:119.87 loss:0.005 grdn:0.157 lr:5.3e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-06 20:34:16 ts/train.py:232 step:435K smpl:7M ep:36K epch:119.92 loss:0.004 grdn:0.153 lr:5.3e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 20:35:01 ts/train.py:232 step:436K smpl:7M ep:36K epch:119.98 loss:0.004 grdn:0.155 lr:5.3e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:35:47 ts/train.py:232 step:436K smpl:7M ep:36K epch:120.03 loss:0.004 grdn:0.155 lr:5.3e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:36:34 ts/train.py:232 step:436K smpl:7M ep:36K epch:120.09 loss:0.004 grdn:0.152 lr:5.3e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 20:37:20 ts/train.py:232 step:436K smpl:7M ep:36K epch:120.14 loss:0.004 grdn:0.141 lr:5.2e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 20:38:06 ts/train.py:232 step:436K smpl:7M ep:36K epch:120.20 loss:0.005 grdn:0.152 lr:5.2e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 20:38:53 ts/train.py:232 step:437K smpl:7M ep:36K epch:120.25 loss:0.005 grdn:0.153 lr:5.2e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 20:39:40 ts/train.py:232 step:437K smpl:7M ep:36K epch:120.31 loss:0.004 grdn:0.149 lr:5.2e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-06 20:40:26 ts/train.py:232 step:437K smpl:7M ep:36K epch:120.36 loss:0.004 grdn:0.151 lr:5.2e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:41:12 ts/train.py:232 step:437K smpl:7M ep:36K epch:120.42 loss:0.005 grdn:0.154 lr:5.2e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 20:41:57 ts/train.py:232 step:437K smpl:7M ep:36K epch:120.48 loss:0.005 grdn:0.160 lr:5.2e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 20:42:46 ts/train.py:232 step:438K smpl:7M ep:36K epch:120.53 loss:0.004 grdn:0.143 lr:5.2e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-06 20:43:32 ts/train.py:232 step:438K smpl:7M ep:36K epch:120.59 loss:0.005 grdn:0.166 lr:5.2e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 20:44:19 ts/train.py:232 step:438K smpl:7M ep:36K epch:120.64 loss:0.004 grdn:0.144 lr:5.2e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 20:45:05 ts/train.py:232 step:438K smpl:7M ep:36K epch:120.70 loss:0.004 grdn:0.157 lr:5.2e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-06 20:45:50 ts/train.py:232 step:438K smpl:7M ep:36K epch:120.75 loss:0.005 grdn:0.151 lr:5.2e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:46:36 ts/train.py:232 step:439K smpl:7M ep:36K epch:120.81 loss:0.004 grdn:0.154 lr:5.2e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 20:47:21 ts/train.py:232 step:439K smpl:7M ep:36K epch:120.86 loss:0.005 grdn:0.167 lr:5.2e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 20:48:08 ts/train.py:232 step:439K smpl:7M ep:36K epch:120.92 loss:0.004 grdn:0.151 lr:5.2e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 20:48:53 ts/train.py:232 step:439K smpl:7M ep:36K epch:120.97 loss:0.004 grdn:0.156 lr:5.2e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:49:39 ts/train.py:232 step:439K smpl:7M ep:36K epch:121.03 loss:0.005 grdn:0.159 lr:5.2e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:50:25 ts/train.py:232 step:440K smpl:7M ep:36K epch:121.08 loss:0.004 grdn:0.152 lr:5.2e-05 updt_s:0.159 data_s:0.068 +INFO 2025-05-06 20:51:10 ts/train.py:232 step:440K smpl:7M ep:36K epch:121.14 loss:0.004 grdn:0.150 lr:5.2e-05 updt_s:0.159 data_s:0.066 +INFO 2025-05-06 20:51:55 ts/train.py:232 step:440K smpl:7M ep:36K epch:121.19 loss:0.004 grdn:0.152 lr:5.2e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-06 20:52:40 ts/train.py:232 step:440K smpl:7M ep:36K epch:121.25 loss:0.004 grdn:0.151 lr:5.2e-05 updt_s:0.159 data_s:0.064 +INFO 2025-05-06 20:53:26 ts/train.py:232 step:440K smpl:7M ep:36K epch:121.30 loss:0.004 grdn:0.153 lr:5.2e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:54:12 ts/train.py:232 step:441K smpl:7M ep:36K epch:121.36 loss:0.004 grdn:0.153 lr:5.2e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-06 20:54:59 ts/train.py:232 step:441K smpl:7M ep:36K epch:121.41 loss:0.004 grdn:0.144 lr:5.2e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 20:55:45 ts/train.py:232 step:441K smpl:7M ep:36K epch:121.47 loss:0.004 grdn:0.157 lr:5.2e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 20:56:34 ts/train.py:232 step:441K smpl:7M ep:36K epch:121.52 loss:0.004 grdn:0.159 lr:5.2e-05 updt_s:0.158 data_s:0.087 +INFO 2025-05-06 20:57:19 ts/train.py:232 step:441K smpl:7M ep:36K epch:121.58 loss:0.004 grdn:0.155 lr:5.2e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 20:58:07 ts/train.py:232 step:442K smpl:7M ep:36K epch:121.63 loss:0.004 grdn:0.155 lr:5.2e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-06 20:58:53 ts/train.py:232 step:442K smpl:7M ep:37K epch:121.69 loss:0.004 grdn:0.155 lr:5.1e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 20:59:39 ts/train.py:232 step:442K smpl:7M ep:37K epch:121.74 loss:0.004 grdn:0.158 lr:5.1e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 21:00:24 ts/train.py:232 step:442K smpl:7M ep:37K epch:121.80 loss:0.004 grdn:0.149 lr:5.1e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 21:01:12 ts/train.py:232 step:442K smpl:7M ep:37K epch:121.85 loss:0.004 grdn:0.155 lr:5.1e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-06 21:01:58 ts/train.py:232 step:443K smpl:7M ep:37K epch:121.91 loss:0.004 grdn:0.156 lr:5.1e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-06 21:02:45 ts/train.py:232 step:443K smpl:7M ep:37K epch:121.96 loss:0.004 grdn:0.145 lr:5.1e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-06 21:03:31 ts/train.py:232 step:443K smpl:7M ep:37K epch:122.02 loss:0.004 grdn:0.149 lr:5.1e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-06 21:04:18 ts/train.py:232 step:443K smpl:7M ep:37K epch:122.07 loss:0.004 grdn:0.154 lr:5.1e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-06 21:05:05 ts/train.py:232 step:443K smpl:7M ep:37K epch:122.13 loss:0.005 grdn:0.172 lr:5.1e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-06 21:05:52 ts/train.py:232 step:444K smpl:7M ep:37K epch:122.18 loss:0.005 grdn:0.158 lr:5.1e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-06 21:06:38 ts/train.py:232 step:444K smpl:7M ep:37K epch:122.24 loss:0.005 grdn:0.160 lr:5.1e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-06 21:07:24 ts/train.py:232 step:444K smpl:7M ep:37K epch:122.29 loss:0.004 grdn:0.165 lr:5.1e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 21:08:10 ts/train.py:232 step:444K smpl:7M ep:37K epch:122.35 loss:0.004 grdn:0.147 lr:5.1e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 21:08:56 ts/train.py:232 step:444K smpl:7M ep:37K epch:122.40 loss:0.004 grdn:0.154 lr:5.1e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 21:09:44 ts/train.py:232 step:445K smpl:7M ep:37K epch:122.46 loss:0.004 grdn:0.153 lr:5.1e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-06 21:10:29 ts/train.py:232 step:445K smpl:7M ep:37K epch:122.51 loss:0.005 grdn:0.167 lr:5.1e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 21:11:18 ts/train.py:232 step:445K smpl:7M ep:37K epch:122.57 loss:0.005 grdn:0.157 lr:5.1e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-06 21:12:07 ts/train.py:232 step:445K smpl:7M ep:37K epch:122.62 loss:0.004 grdn:0.149 lr:5.1e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-06 21:12:54 ts/train.py:232 step:445K smpl:7M ep:37K epch:122.68 loss:0.005 grdn:0.163 lr:5.1e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-06 21:13:40 ts/train.py:232 step:446K smpl:7M ep:37K epch:122.73 loss:0.004 grdn:0.149 lr:5.1e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-06 21:14:27 ts/train.py:232 step:446K smpl:7M ep:37K epch:122.79 loss:0.004 grdn:0.151 lr:5.1e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-06 21:15:14 ts/train.py:232 step:446K smpl:7M ep:37K epch:122.84 loss:0.004 grdn:0.152 lr:5.1e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-06 21:15:59 ts/train.py:232 step:446K smpl:7M ep:37K epch:122.90 loss:0.004 grdn:0.147 lr:5.1e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-06 21:16:46 ts/train.py:232 step:446K smpl:7M ep:37K epch:122.95 loss:0.005 grdn:0.157 lr:5.1e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-06 21:17:32 ts/train.py:232 step:447K smpl:7M ep:37K epch:123.01 loss:0.004 grdn:0.150 lr:5.1e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-06 21:18:18 ts/train.py:232 step:447K smpl:7M ep:37K epch:123.06 loss:0.004 grdn:0.132 lr:5.1e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-06 21:19:04 ts/train.py:232 step:447K smpl:7M ep:37K epch:123.12 loss:0.004 grdn:0.137 lr:5.1e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-06 21:19:51 ts/train.py:232 step:447K smpl:7M ep:37K epch:123.17 loss:0.005 grdn:0.165 lr:5.1e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-06 21:20:37 ts/train.py:232 step:447K smpl:7M ep:37K epch:123.23 loss:0.004 grdn:0.141 lr:5.1e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 21:21:22 ts/train.py:232 step:448K smpl:7M ep:37K epch:123.28 loss:0.004 grdn:0.157 lr:5.0e-05 updt_s:0.160 data_s:0.067 +INFO 2025-05-06 21:22:10 ts/train.py:232 step:448K smpl:7M ep:37K epch:123.34 loss:0.005 grdn:0.165 lr:5.0e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-06 21:22:57 ts/train.py:232 step:448K smpl:7M ep:37K epch:123.39 loss:0.004 grdn:0.156 lr:5.0e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-06 21:23:47 ts/train.py:232 step:448K smpl:7M ep:37K epch:123.45 loss:0.005 grdn:0.169 lr:5.0e-05 updt_s:0.158 data_s:0.096 +INFO 2025-05-06 21:24:34 ts/train.py:232 step:448K smpl:7M ep:37K epch:123.50 loss:0.004 grdn:0.152 lr:5.0e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-06 21:25:21 ts/train.py:232 step:449K smpl:7M ep:37K epch:123.56 loss:0.004 grdn:0.151 lr:5.0e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 21:26:08 ts/train.py:232 step:449K smpl:7M ep:37K epch:123.62 loss:0.004 grdn:0.162 lr:5.0e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 21:26:55 ts/train.py:232 step:449K smpl:7M ep:37K epch:123.67 loss:0.004 grdn:0.150 lr:5.0e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 21:27:41 ts/train.py:232 step:449K smpl:7M ep:37K epch:123.73 loss:0.004 grdn:0.152 lr:5.0e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 21:28:28 ts/train.py:232 step:449K smpl:7M ep:37K epch:123.78 loss:0.004 grdn:0.154 lr:5.0e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 21:29:14 ts/train.py:232 step:450K smpl:7M ep:37K epch:123.84 loss:0.004 grdn:0.170 lr:5.0e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-06 21:30:01 ts/train.py:232 step:450K smpl:7M ep:37K epch:123.89 loss:0.005 grdn:0.175 lr:5.0e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-06 21:30:47 ts/train.py:232 step:450K smpl:7M ep:37K epch:123.95 loss:0.005 grdn:0.161 lr:5.0e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 21:30:47 ts/train.py:241 Checkpoint policy after step 450000 +INFO 2025-05-06 21:31:36 ts/train.py:232 step:450K smpl:7M ep:37K epch:124.00 loss:0.004 grdn:0.148 lr:5.0e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 21:32:21 ts/train.py:232 step:450K smpl:7M ep:37K epch:124.06 loss:0.004 grdn:0.165 lr:5.0e-05 updt_s:0.160 data_s:0.067 +INFO 2025-05-06 21:33:07 ts/train.py:232 step:451K smpl:7M ep:37K epch:124.11 loss:0.004 grdn:0.143 lr:5.0e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 21:33:52 ts/train.py:232 step:451K smpl:7M ep:37K epch:124.17 loss:0.004 grdn:0.155 lr:5.0e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 21:34:39 ts/train.py:232 step:451K smpl:7M ep:37K epch:124.22 loss:0.005 grdn:0.158 lr:5.0e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 21:35:25 ts/train.py:232 step:451K smpl:7M ep:37K epch:124.28 loss:0.004 grdn:0.152 lr:5.0e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 21:36:11 ts/train.py:232 step:451K smpl:7M ep:37K epch:124.33 loss:0.004 grdn:0.153 lr:5.0e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-06 21:36:59 ts/train.py:232 step:452K smpl:7M ep:37K epch:124.39 loss:0.004 grdn:0.141 lr:5.0e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-06 21:37:45 ts/train.py:232 step:452K smpl:7M ep:37K epch:124.44 loss:0.004 grdn:0.159 lr:5.0e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 21:38:31 ts/train.py:232 step:452K smpl:7M ep:37K epch:124.50 loss:0.004 grdn:0.155 lr:5.0e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 21:39:18 ts/train.py:232 step:452K smpl:7M ep:37K epch:124.55 loss:0.004 grdn:0.158 lr:5.0e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-06 21:40:04 ts/train.py:232 step:452K smpl:7M ep:37K epch:124.61 loss:0.004 grdn:0.149 lr:5.0e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 21:40:51 ts/train.py:232 step:453K smpl:7M ep:37K epch:124.66 loss:0.004 grdn:0.152 lr:5.0e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-06 21:41:37 ts/train.py:232 step:453K smpl:7M ep:37K epch:124.72 loss:0.004 grdn:0.138 lr:5.0e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-06 21:42:24 ts/train.py:232 step:453K smpl:7M ep:37K epch:124.77 loss:0.005 grdn:0.161 lr:5.0e-05 updt_s:0.159 data_s:0.072 +INFO 2025-05-06 21:43:10 ts/train.py:232 step:453K smpl:7M ep:37K epch:124.83 loss:0.004 grdn:0.144 lr:5.0e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-06 21:43:56 ts/train.py:232 step:453K smpl:7M ep:37K epch:124.88 loss:0.004 grdn:0.153 lr:4.9e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 21:44:43 ts/train.py:232 step:454K smpl:7M ep:37K epch:124.94 loss:0.004 grdn:0.159 lr:4.9e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 21:45:31 ts/train.py:232 step:454K smpl:7M ep:37K epch:124.99 loss:0.004 grdn:0.159 lr:4.9e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-06 21:46:17 ts/train.py:232 step:454K smpl:7M ep:38K epch:125.05 loss:0.004 grdn:0.149 lr:4.9e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 21:47:04 ts/train.py:232 step:454K smpl:7M ep:38K epch:125.10 loss:0.004 grdn:0.152 lr:4.9e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 21:47:50 ts/train.py:232 step:454K smpl:7M ep:38K epch:125.16 loss:0.004 grdn:0.139 lr:4.9e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 21:48:37 ts/train.py:232 step:455K smpl:7M ep:38K epch:125.21 loss:0.004 grdn:0.153 lr:4.9e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 21:49:22 ts/train.py:232 step:455K smpl:7M ep:38K epch:125.27 loss:0.005 grdn:0.160 lr:4.9e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 21:50:09 ts/train.py:232 step:455K smpl:7M ep:38K epch:125.32 loss:0.004 grdn:0.146 lr:4.9e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 21:51:00 ts/train.py:232 step:455K smpl:7M ep:38K epch:125.38 loss:0.004 grdn:0.154 lr:4.9e-05 updt_s:0.159 data_s:0.096 +INFO 2025-05-06 21:51:49 ts/train.py:232 step:455K smpl:7M ep:38K epch:125.43 loss:0.004 grdn:0.154 lr:4.9e-05 updt_s:0.159 data_s:0.084 +INFO 2025-05-06 21:52:38 ts/train.py:232 step:456K smpl:7M ep:38K epch:125.49 loss:0.004 grdn:0.141 lr:4.9e-05 updt_s:0.158 data_s:0.088 +INFO 2025-05-06 21:53:27 ts/train.py:232 step:456K smpl:7M ep:38K epch:125.54 loss:0.004 grdn:0.159 lr:4.9e-05 updt_s:0.158 data_s:0.088 +INFO 2025-05-06 21:54:16 ts/train.py:232 step:456K smpl:7M ep:38K epch:125.60 loss:0.004 grdn:0.152 lr:4.9e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 21:55:04 ts/train.py:232 step:456K smpl:7M ep:38K epch:125.65 loss:0.004 grdn:0.154 lr:4.9e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-06 21:55:53 ts/train.py:232 step:456K smpl:7M ep:38K epch:125.71 loss:0.004 grdn:0.142 lr:4.9e-05 updt_s:0.158 data_s:0.086 +INFO 2025-05-06 21:56:41 ts/train.py:232 step:457K smpl:7M ep:38K epch:125.76 loss:0.005 grdn:0.163 lr:4.9e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 21:57:31 ts/train.py:232 step:457K smpl:7M ep:38K epch:125.82 loss:0.004 grdn:0.152 lr:4.9e-05 updt_s:0.159 data_s:0.091 +INFO 2025-05-06 21:58:22 ts/train.py:232 step:457K smpl:7M ep:38K epch:125.87 loss:0.004 grdn:0.152 lr:4.9e-05 updt_s:0.159 data_s:0.090 +INFO 2025-05-06 21:59:11 ts/train.py:232 step:457K smpl:7M ep:38K epch:125.93 loss:0.004 grdn:0.148 lr:4.9e-05 updt_s:0.159 data_s:0.086 +INFO 2025-05-06 21:59:59 ts/train.py:232 step:457K smpl:7M ep:38K epch:125.98 loss:0.004 grdn:0.157 lr:4.9e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 22:00:48 ts/train.py:232 step:458K smpl:7M ep:38K epch:126.04 loss:0.004 grdn:0.154 lr:4.9e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 22:01:36 ts/train.py:232 step:458K smpl:7M ep:38K epch:126.09 loss:0.004 grdn:0.162 lr:4.9e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-06 22:02:25 ts/train.py:232 step:458K smpl:7M ep:38K epch:126.15 loss:0.004 grdn:0.159 lr:4.9e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 22:03:13 ts/train.py:232 step:458K smpl:7M ep:38K epch:126.20 loss:0.004 grdn:0.157 lr:4.9e-05 updt_s:0.159 data_s:0.083 +INFO 2025-05-06 22:04:03 ts/train.py:232 step:458K smpl:7M ep:38K epch:126.26 loss:0.004 grdn:0.149 lr:4.9e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 22:04:53 ts/train.py:232 step:459K smpl:7M ep:38K epch:126.31 loss:0.004 grdn:0.139 lr:4.9e-05 updt_s:0.158 data_s:0.091 +INFO 2025-05-06 22:05:39 ts/train.py:232 step:459K smpl:7M ep:38K epch:126.37 loss:0.004 grdn:0.154 lr:4.9e-05 updt_s:0.159 data_s:0.073 +INFO 2025-05-06 22:06:26 ts/train.py:232 step:459K smpl:7M ep:38K epch:126.42 loss:0.004 grdn:0.164 lr:4.8e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-06 22:07:14 ts/train.py:232 step:459K smpl:7M ep:38K epch:126.48 loss:0.004 grdn:0.148 lr:4.8e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-06 22:08:00 ts/train.py:232 step:459K smpl:7M ep:38K epch:126.53 loss:0.004 grdn:0.146 lr:4.8e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 22:08:46 ts/train.py:232 step:460K smpl:7M ep:38K epch:126.59 loss:0.004 grdn:0.149 lr:4.8e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-06 22:09:32 ts/train.py:232 step:460K smpl:7M ep:38K epch:126.64 loss:0.004 grdn:0.146 lr:4.8e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 22:10:17 ts/train.py:232 step:460K smpl:7M ep:38K epch:126.70 loss:0.004 grdn:0.154 lr:4.8e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-06 22:11:02 ts/train.py:232 step:460K smpl:7M ep:38K epch:126.76 loss:0.004 grdn:0.148 lr:4.8e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 22:11:48 ts/train.py:232 step:460K smpl:7M ep:38K epch:126.81 loss:0.005 grdn:0.172 lr:4.8e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 22:12:33 ts/train.py:232 step:461K smpl:7M ep:38K epch:126.87 loss:0.004 grdn:0.149 lr:4.8e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-06 22:13:19 ts/train.py:232 step:461K smpl:7M ep:38K epch:126.92 loss:0.005 grdn:0.160 lr:4.8e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 22:14:04 ts/train.py:232 step:461K smpl:7M ep:38K epch:126.98 loss:0.004 grdn:0.137 lr:4.8e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 22:14:50 ts/train.py:232 step:461K smpl:7M ep:38K epch:127.03 loss:0.004 grdn:0.145 lr:4.8e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 22:15:35 ts/train.py:232 step:461K smpl:7M ep:38K epch:127.09 loss:0.004 grdn:0.151 lr:4.8e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-06 22:16:21 ts/train.py:232 step:462K smpl:7M ep:38K epch:127.14 loss:0.004 grdn:0.155 lr:4.8e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 22:17:07 ts/train.py:232 step:462K smpl:7M ep:38K epch:127.20 loss:0.004 grdn:0.157 lr:4.8e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 22:17:52 ts/train.py:232 step:462K smpl:7M ep:38K epch:127.25 loss:0.003 grdn:0.136 lr:4.8e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-06 22:18:42 ts/train.py:232 step:462K smpl:7M ep:38K epch:127.31 loss:0.005 grdn:0.164 lr:4.8e-05 updt_s:0.158 data_s:0.092 +INFO 2025-05-06 22:19:31 ts/train.py:232 step:462K smpl:7M ep:38K epch:127.36 loss:0.004 grdn:0.150 lr:4.8e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-06 22:20:20 ts/train.py:232 step:463K smpl:7M ep:38K epch:127.42 loss:0.004 grdn:0.151 lr:4.8e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-06 22:21:09 ts/train.py:232 step:463K smpl:7M ep:38K epch:127.47 loss:0.004 grdn:0.153 lr:4.8e-05 updt_s:0.158 data_s:0.086 +INFO 2025-05-06 22:22:00 ts/train.py:232 step:463K smpl:7M ep:38K epch:127.53 loss:0.005 grdn:0.169 lr:4.8e-05 updt_s:0.157 data_s:0.097 +INFO 2025-05-06 22:22:51 ts/train.py:232 step:463K smpl:7M ep:38K epch:127.58 loss:0.004 grdn:0.158 lr:4.8e-05 updt_s:0.157 data_s:0.100 +INFO 2025-05-06 22:23:43 ts/train.py:232 step:463K smpl:7M ep:38K epch:127.64 loss:0.004 grdn:0.151 lr:4.8e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-06 22:24:35 ts/train.py:232 step:464K smpl:7M ep:38K epch:127.69 loss:0.004 grdn:0.154 lr:4.8e-05 updt_s:0.157 data_s:0.101 +INFO 2025-05-06 22:25:25 ts/train.py:232 step:464K smpl:7M ep:38K epch:127.75 loss:0.004 grdn:0.150 lr:4.8e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-06 22:26:16 ts/train.py:232 step:464K smpl:7M ep:38K epch:127.80 loss:0.004 grdn:0.149 lr:4.8e-05 updt_s:0.157 data_s:0.098 +INFO 2025-05-06 22:27:07 ts/train.py:232 step:464K smpl:7M ep:38K epch:127.86 loss:0.004 grdn:0.151 lr:4.8e-05 updt_s:0.158 data_s:0.097 +INFO 2025-05-06 22:27:58 ts/train.py:232 step:464K smpl:7M ep:38K epch:127.91 loss:0.004 grdn:0.156 lr:4.8e-05 updt_s:0.158 data_s:0.095 +INFO 2025-05-06 22:28:49 ts/train.py:232 step:465K smpl:7M ep:38K epch:127.97 loss:0.004 grdn:0.144 lr:4.8e-05 updt_s:0.157 data_s:0.098 +INFO 2025-05-06 22:29:41 ts/train.py:232 step:465K smpl:7M ep:38K epch:128.02 loss:0.004 grdn:0.156 lr:4.7e-05 updt_s:0.157 data_s:0.100 +INFO 2025-05-06 22:30:32 ts/train.py:232 step:465K smpl:7M ep:38K epch:128.08 loss:0.004 grdn:0.153 lr:4.7e-05 updt_s:0.158 data_s:0.097 +INFO 2025-05-06 22:31:23 ts/train.py:232 step:465K smpl:7M ep:38K epch:128.13 loss:0.004 grdn:0.146 lr:4.7e-05 updt_s:0.158 data_s:0.099 +INFO 2025-05-06 22:32:15 ts/train.py:232 step:465K smpl:7M ep:38K epch:128.19 loss:0.005 grdn:0.162 lr:4.7e-05 updt_s:0.158 data_s:0.098 +INFO 2025-05-06 22:33:07 ts/train.py:232 step:466K smpl:7M ep:38K epch:128.24 loss:0.004 grdn:0.158 lr:4.7e-05 updt_s:0.158 data_s:0.102 +INFO 2025-05-06 22:33:56 ts/train.py:232 step:466K smpl:7M ep:38K epch:128.30 loss:0.004 grdn:0.155 lr:4.7e-05 updt_s:0.159 data_s:0.088 +INFO 2025-05-06 22:34:46 ts/train.py:232 step:466K smpl:7M ep:39K epch:128.35 loss:0.004 grdn:0.146 lr:4.7e-05 updt_s:0.158 data_s:0.090 +INFO 2025-05-06 22:35:36 ts/train.py:232 step:466K smpl:7M ep:39K epch:128.41 loss:0.004 grdn:0.160 lr:4.7e-05 updt_s:0.158 data_s:0.089 +INFO 2025-05-06 22:36:23 ts/train.py:232 step:466K smpl:7M ep:39K epch:128.46 loss:0.004 grdn:0.149 lr:4.7e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-06 22:37:11 ts/train.py:232 step:467K smpl:7M ep:39K epch:128.52 loss:0.004 grdn:0.151 lr:4.7e-05 updt_s:0.159 data_s:0.078 +INFO 2025-05-06 22:37:58 ts/train.py:232 step:467K smpl:7M ep:39K epch:128.57 loss:0.004 grdn:0.154 lr:4.7e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 22:38:44 ts/train.py:232 step:467K smpl:7M ep:39K epch:128.63 loss:0.004 grdn:0.149 lr:4.7e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 22:39:30 ts/train.py:232 step:467K smpl:7M ep:39K epch:128.68 loss:0.004 grdn:0.156 lr:4.7e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 22:40:17 ts/train.py:232 step:467K smpl:7M ep:39K epch:128.74 loss:0.004 grdn:0.152 lr:4.7e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 22:41:03 ts/train.py:232 step:468K smpl:7M ep:39K epch:128.79 loss:0.004 grdn:0.153 lr:4.7e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 22:41:48 ts/train.py:232 step:468K smpl:7M ep:39K epch:128.85 loss:0.004 grdn:0.156 lr:4.7e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 22:42:34 ts/train.py:232 step:468K smpl:7M ep:39K epch:128.90 loss:0.004 grdn:0.156 lr:4.7e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 22:43:20 ts/train.py:232 step:468K smpl:7M ep:39K epch:128.96 loss:0.004 grdn:0.157 lr:4.7e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 22:44:07 ts/train.py:232 step:468K smpl:7M ep:39K epch:129.01 loss:0.004 grdn:0.149 lr:4.7e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 22:44:53 ts/train.py:232 step:469K smpl:7M ep:39K epch:129.07 loss:0.004 grdn:0.152 lr:4.7e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 22:45:39 ts/train.py:232 step:469K smpl:8M ep:39K epch:129.12 loss:0.004 grdn:0.153 lr:4.7e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 22:46:25 ts/train.py:232 step:469K smpl:8M ep:39K epch:129.18 loss:0.004 grdn:0.158 lr:4.7e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 22:47:16 ts/train.py:232 step:469K smpl:8M ep:39K epch:129.23 loss:0.004 grdn:0.147 lr:4.7e-05 updt_s:0.158 data_s:0.096 +INFO 2025-05-06 22:48:05 ts/train.py:232 step:469K smpl:8M ep:39K epch:129.29 loss:0.004 grdn:0.141 lr:4.7e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-06 22:48:53 ts/train.py:232 step:470K smpl:8M ep:39K epch:129.34 loss:0.004 grdn:0.146 lr:4.7e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 22:49:40 ts/train.py:232 step:470K smpl:8M ep:39K epch:129.40 loss:0.004 grdn:0.155 lr:4.7e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-06 22:50:27 ts/train.py:232 step:470K smpl:8M ep:39K epch:129.45 loss:0.004 grdn:0.158 lr:4.7e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 22:51:14 ts/train.py:232 step:470K smpl:8M ep:39K epch:129.51 loss:0.004 grdn:0.149 lr:4.7e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 22:52:00 ts/train.py:232 step:470K smpl:8M ep:39K epch:129.56 loss:0.004 grdn:0.143 lr:4.7e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 22:52:46 ts/train.py:232 step:471K smpl:8M ep:39K epch:129.62 loss:0.004 grdn:0.155 lr:4.6e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 22:53:33 ts/train.py:232 step:471K smpl:8M ep:39K epch:129.67 loss:0.004 grdn:0.153 lr:4.6e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 22:54:18 ts/train.py:232 step:471K smpl:8M ep:39K epch:129.73 loss:0.004 grdn:0.138 lr:4.6e-05 updt_s:0.159 data_s:0.069 +INFO 2025-05-06 22:55:05 ts/train.py:232 step:471K smpl:8M ep:39K epch:129.78 loss:0.004 grdn:0.159 lr:4.6e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 22:55:51 ts/train.py:232 step:471K smpl:8M ep:39K epch:129.84 loss:0.004 grdn:0.152 lr:4.6e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-06 22:56:37 ts/train.py:232 step:472K smpl:8M ep:39K epch:129.89 loss:0.004 grdn:0.151 lr:4.6e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-06 22:57:23 ts/train.py:232 step:472K smpl:8M ep:39K epch:129.95 loss:0.004 grdn:0.158 lr:4.6e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 22:58:11 ts/train.py:232 step:472K smpl:8M ep:39K epch:130.01 loss:0.004 grdn:0.154 lr:4.6e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-06 22:58:59 ts/train.py:232 step:472K smpl:8M ep:39K epch:130.06 loss:0.004 grdn:0.158 lr:4.6e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-06 22:59:45 ts/train.py:232 step:472K smpl:8M ep:39K epch:130.12 loss:0.004 grdn:0.163 lr:4.6e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 23:00:33 ts/train.py:232 step:473K smpl:8M ep:39K epch:130.17 loss:0.004 grdn:0.159 lr:4.6e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-06 23:01:22 ts/train.py:232 step:473K smpl:8M ep:39K epch:130.23 loss:0.004 grdn:0.155 lr:4.6e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-06 23:02:12 ts/train.py:232 step:473K smpl:8M ep:39K epch:130.28 loss:0.004 grdn:0.153 lr:4.6e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-06 23:03:02 ts/train.py:232 step:473K smpl:8M ep:39K epch:130.34 loss:0.004 grdn:0.155 lr:4.6e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-06 23:03:49 ts/train.py:232 step:473K smpl:8M ep:39K epch:130.39 loss:0.004 grdn:0.157 lr:4.6e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 23:04:34 ts/train.py:232 step:474K smpl:8M ep:39K epch:130.45 loss:0.004 grdn:0.145 lr:4.6e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-06 23:05:20 ts/train.py:232 step:474K smpl:8M ep:39K epch:130.50 loss:0.004 grdn:0.158 lr:4.6e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 23:06:08 ts/train.py:232 step:474K smpl:8M ep:39K epch:130.56 loss:0.004 grdn:0.142 lr:4.6e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-06 23:06:53 ts/train.py:232 step:474K smpl:8M ep:39K epch:130.61 loss:0.004 grdn:0.154 lr:4.6e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 23:07:39 ts/train.py:232 step:474K smpl:8M ep:39K epch:130.67 loss:0.004 grdn:0.157 lr:4.6e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 23:08:26 ts/train.py:232 step:475K smpl:8M ep:39K epch:130.72 loss:0.004 grdn:0.150 lr:4.6e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 23:09:13 ts/train.py:232 step:475K smpl:8M ep:39K epch:130.78 loss:0.004 grdn:0.152 lr:4.6e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 23:09:58 ts/train.py:232 step:475K smpl:8M ep:39K epch:130.83 loss:0.004 grdn:0.148 lr:4.6e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 23:10:43 ts/train.py:232 step:475K smpl:8M ep:39K epch:130.89 loss:0.004 grdn:0.154 lr:4.6e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-06 23:11:29 ts/train.py:232 step:475K smpl:8M ep:39K epch:130.94 loss:0.004 grdn:0.163 lr:4.6e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-06 23:12:15 ts/train.py:232 step:476K smpl:8M ep:39K epch:131.00 loss:0.004 grdn:0.157 lr:4.6e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 23:13:03 ts/train.py:232 step:476K smpl:8M ep:39K epch:131.05 loss:0.004 grdn:0.158 lr:4.6e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 23:13:48 ts/train.py:232 step:476K smpl:8M ep:39K epch:131.11 loss:0.004 grdn:0.148 lr:4.6e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-06 23:14:36 ts/train.py:232 step:476K smpl:8M ep:39K epch:131.16 loss:0.004 grdn:0.145 lr:4.5e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-06 23:15:22 ts/train.py:232 step:476K smpl:8M ep:39K epch:131.22 loss:0.004 grdn:0.149 lr:4.5e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 23:16:08 ts/train.py:232 step:477K smpl:8M ep:39K epch:131.27 loss:0.004 grdn:0.158 lr:4.5e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 23:16:54 ts/train.py:232 step:477K smpl:8M ep:39K epch:131.33 loss:0.004 grdn:0.142 lr:4.5e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-06 23:17:39 ts/train.py:232 step:477K smpl:8M ep:39K epch:131.38 loss:0.004 grdn:0.152 lr:4.5e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 23:18:24 ts/train.py:232 step:477K smpl:8M ep:39K epch:131.44 loss:0.004 grdn:0.158 lr:4.5e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-06 23:19:10 ts/train.py:232 step:477K smpl:8M ep:39K epch:131.49 loss:0.004 grdn:0.154 lr:4.5e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 23:19:56 ts/train.py:232 step:478K smpl:8M ep:39K epch:131.55 loss:0.004 grdn:0.156 lr:4.5e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 23:20:42 ts/train.py:232 step:478K smpl:8M ep:39K epch:131.60 loss:0.004 grdn:0.158 lr:4.5e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-06 23:21:27 ts/train.py:232 step:478K smpl:8M ep:39K epch:131.66 loss:0.004 grdn:0.149 lr:4.5e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-06 23:22:13 ts/train.py:232 step:478K smpl:8M ep:40K epch:131.71 loss:0.004 grdn:0.153 lr:4.5e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 23:22:58 ts/train.py:232 step:478K smpl:8M ep:40K epch:131.77 loss:0.004 grdn:0.140 lr:4.5e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-06 23:23:43 ts/train.py:232 step:479K smpl:8M ep:40K epch:131.82 loss:0.004 grdn:0.147 lr:4.5e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-06 23:24:28 ts/train.py:232 step:479K smpl:8M ep:40K epch:131.88 loss:0.004 grdn:0.159 lr:4.5e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 23:25:13 ts/train.py:232 step:479K smpl:8M ep:40K epch:131.93 loss:0.004 grdn:0.152 lr:4.5e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-06 23:25:59 ts/train.py:232 step:479K smpl:8M ep:40K epch:131.99 loss:0.004 grdn:0.152 lr:4.5e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-06 23:26:44 ts/train.py:232 step:479K smpl:8M ep:40K epch:132.04 loss:0.004 grdn:0.140 lr:4.5e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-06 23:27:32 ts/train.py:232 step:480K smpl:8M ep:40K epch:132.10 loss:0.004 grdn:0.154 lr:4.5e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 23:28:21 ts/train.py:232 step:480K smpl:8M ep:40K epch:132.15 loss:0.004 grdn:0.152 lr:4.5e-05 updt_s:0.158 data_s:0.086 +INFO 2025-05-06 23:29:10 ts/train.py:232 step:480K smpl:8M ep:40K epch:132.21 loss:0.003 grdn:0.141 lr:4.5e-05 updt_s:0.158 data_s:0.087 +INFO 2025-05-06 23:29:10 ts/train.py:241 Checkpoint policy after step 480000 +INFO 2025-05-06 23:30:01 ts/train.py:232 step:480K smpl:8M ep:40K epch:132.26 loss:0.004 grdn:0.154 lr:4.5e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-06 23:30:50 ts/train.py:232 step:480K smpl:8M ep:40K epch:132.32 loss:0.004 grdn:0.157 lr:4.5e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-06 23:31:38 ts/train.py:232 step:481K smpl:8M ep:40K epch:132.37 loss:0.004 grdn:0.152 lr:4.5e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-06 23:32:26 ts/train.py:232 step:481K smpl:8M ep:40K epch:132.43 loss:0.004 grdn:0.160 lr:4.5e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 23:33:14 ts/train.py:232 step:481K smpl:8M ep:40K epch:132.48 loss:0.004 grdn:0.153 lr:4.5e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-06 23:34:03 ts/train.py:232 step:481K smpl:8M ep:40K epch:132.54 loss:0.004 grdn:0.150 lr:4.5e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 23:34:52 ts/train.py:232 step:481K smpl:8M ep:40K epch:132.59 loss:0.004 grdn:0.154 lr:4.5e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-06 23:35:40 ts/train.py:232 step:482K smpl:8M ep:40K epch:132.65 loss:0.004 grdn:0.153 lr:4.5e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-06 23:36:29 ts/train.py:232 step:482K smpl:8M ep:40K epch:132.70 loss:0.004 grdn:0.151 lr:4.5e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 23:37:17 ts/train.py:232 step:482K smpl:8M ep:40K epch:132.76 loss:0.004 grdn:0.157 lr:4.4e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-06 23:38:05 ts/train.py:232 step:482K smpl:8M ep:40K epch:132.81 loss:0.004 grdn:0.143 lr:4.4e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-06 23:38:53 ts/train.py:232 step:482K smpl:8M ep:40K epch:132.87 loss:0.004 grdn:0.148 lr:4.4e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-06 23:39:44 ts/train.py:232 step:483K smpl:8M ep:40K epch:132.92 loss:0.004 grdn:0.165 lr:4.4e-05 updt_s:0.157 data_s:0.096 +INFO 2025-05-06 23:40:31 ts/train.py:232 step:483K smpl:8M ep:40K epch:132.98 loss:0.004 grdn:0.150 lr:4.4e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 23:41:18 ts/train.py:232 step:483K smpl:8M ep:40K epch:133.03 loss:0.004 grdn:0.148 lr:4.4e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-06 23:42:10 ts/train.py:232 step:483K smpl:8M ep:40K epch:133.09 loss:0.004 grdn:0.144 lr:4.4e-05 updt_s:0.157 data_s:0.101 +INFO 2025-05-06 23:42:59 ts/train.py:232 step:483K smpl:8M ep:40K epch:133.15 loss:0.004 grdn:0.149 lr:4.4e-05 updt_s:0.158 data_s:0.089 +INFO 2025-05-06 23:43:48 ts/train.py:232 step:484K smpl:8M ep:40K epch:133.20 loss:0.004 grdn:0.156 lr:4.4e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-06 23:44:37 ts/train.py:232 step:484K smpl:8M ep:40K epch:133.26 loss:0.004 grdn:0.147 lr:4.4e-05 updt_s:0.158 data_s:0.088 +INFO 2025-05-06 23:45:24 ts/train.py:232 step:484K smpl:8M ep:40K epch:133.31 loss:0.004 grdn:0.158 lr:4.4e-05 updt_s:0.159 data_s:0.074 +INFO 2025-05-06 23:46:11 ts/train.py:232 step:484K smpl:8M ep:40K epch:133.37 loss:0.004 grdn:0.150 lr:4.4e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-06 23:46:58 ts/train.py:232 step:484K smpl:8M ep:40K epch:133.42 loss:0.004 grdn:0.158 lr:4.4e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-06 23:47:45 ts/train.py:232 step:485K smpl:8M ep:40K epch:133.48 loss:0.004 grdn:0.161 lr:4.4e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-06 23:48:33 ts/train.py:232 step:485K smpl:8M ep:40K epch:133.53 loss:0.005 grdn:0.167 lr:4.4e-05 updt_s:0.159 data_s:0.080 +INFO 2025-05-06 23:49:20 ts/train.py:232 step:485K smpl:8M ep:40K epch:133.59 loss:0.004 grdn:0.149 lr:4.4e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-06 23:50:06 ts/train.py:232 step:485K smpl:8M ep:40K epch:133.64 loss:0.004 grdn:0.150 lr:4.4e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-06 23:50:53 ts/train.py:232 step:485K smpl:8M ep:40K epch:133.70 loss:0.004 grdn:0.162 lr:4.4e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-06 23:51:40 ts/train.py:232 step:486K smpl:8M ep:40K epch:133.75 loss:0.004 grdn:0.145 lr:4.4e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-06 23:52:30 ts/train.py:232 step:486K smpl:8M ep:40K epch:133.81 loss:0.004 grdn:0.142 lr:4.4e-05 updt_s:0.159 data_s:0.087 +INFO 2025-05-06 23:53:17 ts/train.py:232 step:486K smpl:8M ep:40K epch:133.86 loss:0.004 grdn:0.154 lr:4.4e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-06 23:54:02 ts/train.py:232 step:486K smpl:8M ep:40K epch:133.92 loss:0.004 grdn:0.156 lr:4.4e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-06 23:54:48 ts/train.py:232 step:486K smpl:8M ep:40K epch:133.97 loss:0.004 grdn:0.151 lr:4.4e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-06 23:55:38 ts/train.py:232 step:487K smpl:8M ep:40K epch:134.03 loss:0.004 grdn:0.148 lr:4.4e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-06 23:56:27 ts/train.py:232 step:487K smpl:8M ep:40K epch:134.08 loss:0.004 grdn:0.150 lr:4.4e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 23:57:16 ts/train.py:232 step:487K smpl:8M ep:40K epch:134.14 loss:0.003 grdn:0.134 lr:4.4e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-06 23:58:05 ts/train.py:232 step:487K smpl:8M ep:40K epch:134.19 loss:0.004 grdn:0.159 lr:4.4e-05 updt_s:0.158 data_s:0.086 +INFO 2025-05-06 23:58:53 ts/train.py:232 step:487K smpl:8M ep:40K epch:134.25 loss:0.004 grdn:0.159 lr:4.4e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-06 23:59:39 ts/train.py:232 step:488K smpl:8M ep:40K epch:134.30 loss:0.004 grdn:0.154 lr:4.4e-05 updt_s:0.159 data_s:0.075 +INFO 2025-05-07 00:00:28 ts/train.py:232 step:488K smpl:8M ep:40K epch:134.36 loss:0.004 grdn:0.157 lr:4.3e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 00:01:15 ts/train.py:232 step:488K smpl:8M ep:40K epch:134.41 loss:0.004 grdn:0.147 lr:4.3e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 00:02:01 ts/train.py:232 step:488K smpl:8M ep:40K epch:134.47 loss:0.004 grdn:0.152 lr:4.3e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 00:02:48 ts/train.py:232 step:488K smpl:8M ep:40K epch:134.52 loss:0.004 grdn:0.161 lr:4.3e-05 updt_s:0.159 data_s:0.076 +INFO 2025-05-07 00:03:36 ts/train.py:232 step:489K smpl:8M ep:40K epch:134.58 loss:0.004 grdn:0.155 lr:4.3e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 00:04:22 ts/train.py:232 step:489K smpl:8M ep:40K epch:134.63 loss:0.004 grdn:0.151 lr:4.3e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 00:05:10 ts/train.py:232 step:489K smpl:8M ep:40K epch:134.69 loss:0.004 grdn:0.142 lr:4.3e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-07 00:05:56 ts/train.py:232 step:489K smpl:8M ep:40K epch:134.74 loss:0.004 grdn:0.158 lr:4.3e-05 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 00:06:43 ts/train.py:232 step:489K smpl:8M ep:40K epch:134.80 loss:0.004 grdn:0.148 lr:4.3e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 00:07:29 ts/train.py:232 step:490K smpl:8M ep:40K epch:134.85 loss:0.004 grdn:0.149 lr:4.3e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 00:08:17 ts/train.py:232 step:490K smpl:8M ep:40K epch:134.91 loss:0.004 grdn:0.163 lr:4.3e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 00:09:03 ts/train.py:232 step:490K smpl:8M ep:40K epch:134.96 loss:0.004 grdn:0.143 lr:4.3e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 00:09:52 ts/train.py:232 step:490K smpl:8M ep:41K epch:135.02 loss:0.004 grdn:0.152 lr:4.3e-05 updt_s:0.158 data_s:0.089 +INFO 2025-05-07 00:10:41 ts/train.py:232 step:490K smpl:8M ep:41K epch:135.07 loss:0.004 grdn:0.156 lr:4.3e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-07 00:11:29 ts/train.py:232 step:491K smpl:8M ep:41K epch:135.13 loss:0.004 grdn:0.164 lr:4.3e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 00:12:16 ts/train.py:232 step:491K smpl:8M ep:41K epch:135.18 loss:0.004 grdn:0.160 lr:4.3e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-07 00:13:04 ts/train.py:232 step:491K smpl:8M ep:41K epch:135.24 loss:0.004 grdn:0.158 lr:4.3e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-07 00:13:51 ts/train.py:232 step:491K smpl:8M ep:41K epch:135.29 loss:0.004 grdn:0.150 lr:4.3e-05 updt_s:0.159 data_s:0.077 +INFO 2025-05-07 00:14:39 ts/train.py:232 step:491K smpl:8M ep:41K epch:135.35 loss:0.003 grdn:0.142 lr:4.3e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 00:15:28 ts/train.py:232 step:492K smpl:8M ep:41K epch:135.40 loss:0.004 grdn:0.156 lr:4.3e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-07 00:16:17 ts/train.py:232 step:492K smpl:8M ep:41K epch:135.46 loss:0.004 grdn:0.151 lr:4.3e-05 updt_s:0.158 data_s:0.087 +INFO 2025-05-07 00:17:04 ts/train.py:232 step:492K smpl:8M ep:41K epch:135.51 loss:0.005 grdn:0.178 lr:4.3e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 00:17:52 ts/train.py:232 step:492K smpl:8M ep:41K epch:135.57 loss:0.004 grdn:0.159 lr:4.3e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 00:18:40 ts/train.py:232 step:492K smpl:8M ep:41K epch:135.62 loss:0.004 grdn:0.156 lr:4.3e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-07 00:19:28 ts/train.py:232 step:493K smpl:8M ep:41K epch:135.68 loss:0.004 grdn:0.149 lr:4.3e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-07 00:20:17 ts/train.py:232 step:493K smpl:8M ep:41K epch:135.73 loss:0.004 grdn:0.165 lr:4.3e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 00:21:05 ts/train.py:232 step:493K smpl:8M ep:41K epch:135.79 loss:0.004 grdn:0.160 lr:4.3e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-07 00:21:53 ts/train.py:232 step:493K smpl:8M ep:41K epch:135.84 loss:0.004 grdn:0.150 lr:4.3e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 00:22:40 ts/train.py:232 step:493K smpl:8M ep:41K epch:135.90 loss:0.004 grdn:0.164 lr:4.3e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 00:23:30 ts/train.py:232 step:494K smpl:8M ep:41K epch:135.95 loss:0.004 grdn:0.157 lr:4.2e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 00:24:18 ts/train.py:232 step:494K smpl:8M ep:41K epch:136.01 loss:0.003 grdn:0.141 lr:4.2e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-07 00:25:07 ts/train.py:232 step:494K smpl:8M ep:41K epch:136.06 loss:0.003 grdn:0.144 lr:4.2e-05 updt_s:0.159 data_s:0.082 +INFO 2025-05-07 00:25:55 ts/train.py:232 step:494K smpl:8M ep:41K epch:136.12 loss:0.004 grdn:0.157 lr:4.2e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 00:26:42 ts/train.py:232 step:494K smpl:8M ep:41K epch:136.17 loss:0.004 grdn:0.149 lr:4.2e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 00:27:27 ts/train.py:232 step:495K smpl:8M ep:41K epch:136.23 loss:0.004 grdn:0.158 lr:4.2e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 00:28:14 ts/train.py:232 step:495K smpl:8M ep:41K epch:136.29 loss:0.004 grdn:0.151 lr:4.2e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 00:28:59 ts/train.py:232 step:495K smpl:8M ep:41K epch:136.34 loss:0.004 grdn:0.166 lr:4.2e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 00:29:45 ts/train.py:232 step:495K smpl:8M ep:41K epch:136.40 loss:0.004 grdn:0.145 lr:4.2e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 00:30:33 ts/train.py:232 step:495K smpl:8M ep:41K epch:136.45 loss:0.004 grdn:0.152 lr:4.2e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 00:31:19 ts/train.py:232 step:496K smpl:8M ep:41K epch:136.51 loss:0.003 grdn:0.144 lr:4.2e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 00:32:06 ts/train.py:232 step:496K smpl:8M ep:41K epch:136.56 loss:0.004 grdn:0.154 lr:4.2e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 00:32:52 ts/train.py:232 step:496K smpl:8M ep:41K epch:136.62 loss:0.004 grdn:0.152 lr:4.2e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 00:33:38 ts/train.py:232 step:496K smpl:8M ep:41K epch:136.67 loss:0.004 grdn:0.157 lr:4.2e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 00:34:23 ts/train.py:232 step:496K smpl:8M ep:41K epch:136.73 loss:0.004 grdn:0.149 lr:4.2e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 00:35:09 ts/train.py:232 step:497K smpl:8M ep:41K epch:136.78 loss:0.004 grdn:0.154 lr:4.2e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-07 00:35:54 ts/train.py:232 step:497K smpl:8M ep:41K epch:136.84 loss:0.004 grdn:0.159 lr:4.2e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 00:36:40 ts/train.py:232 step:497K smpl:8M ep:41K epch:136.89 loss:0.004 grdn:0.160 lr:4.2e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 00:37:32 ts/train.py:232 step:497K smpl:8M ep:41K epch:136.95 loss:0.004 grdn:0.158 lr:4.2e-05 updt_s:0.158 data_s:0.099 +INFO 2025-05-07 00:38:21 ts/train.py:232 step:497K smpl:8M ep:41K epch:137.00 loss:0.004 grdn:0.158 lr:4.2e-05 updt_s:0.158 data_s:0.088 +INFO 2025-05-07 00:39:09 ts/train.py:232 step:498K smpl:8M ep:41K epch:137.06 loss:0.003 grdn:0.138 lr:4.2e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-07 00:39:58 ts/train.py:232 step:498K smpl:8M ep:41K epch:137.11 loss:0.004 grdn:0.156 lr:4.2e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-07 00:40:44 ts/train.py:232 step:498K smpl:8M ep:41K epch:137.17 loss:0.004 grdn:0.162 lr:4.2e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 00:41:32 ts/train.py:232 step:498K smpl:8M ep:41K epch:137.22 loss:0.003 grdn:0.141 lr:4.2e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-07 00:42:19 ts/train.py:232 step:498K smpl:8M ep:41K epch:137.28 loss:0.004 grdn:0.161 lr:4.2e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 00:43:06 ts/train.py:232 step:499K smpl:8M ep:41K epch:137.33 loss:0.003 grdn:0.148 lr:4.2e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 00:43:53 ts/train.py:232 step:499K smpl:8M ep:41K epch:137.39 loss:0.004 grdn:0.166 lr:4.2e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 00:44:40 ts/train.py:232 step:499K smpl:8M ep:41K epch:137.44 loss:0.004 grdn:0.152 lr:4.2e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 00:45:26 ts/train.py:232 step:499K smpl:8M ep:41K epch:137.50 loss:0.004 grdn:0.146 lr:4.2e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 00:46:13 ts/train.py:232 step:499K smpl:8M ep:41K epch:137.55 loss:0.004 grdn:0.168 lr:4.1e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 00:47:00 ts/train.py:232 step:500K smpl:8M ep:41K epch:137.61 loss:0.004 grdn:0.147 lr:4.1e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 00:47:46 ts/train.py:232 step:500K smpl:8M ep:41K epch:137.66 loss:0.004 grdn:0.151 lr:4.1e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 00:48:33 ts/train.py:232 step:500K smpl:8M ep:41K epch:137.72 loss:0.004 grdn:0.152 lr:4.1e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 00:49:21 ts/train.py:232 step:500K smpl:8M ep:41K epch:137.77 loss:0.004 grdn:0.167 lr:4.1e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-07 00:50:09 ts/train.py:232 step:500K smpl:8M ep:41K epch:137.83 loss:0.004 grdn:0.157 lr:4.1e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 00:50:57 ts/train.py:232 step:501K smpl:8M ep:41K epch:137.88 loss:0.004 grdn:0.159 lr:4.1e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 00:51:43 ts/train.py:232 step:501K smpl:8M ep:41K epch:137.94 loss:0.004 grdn:0.150 lr:4.1e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 00:52:29 ts/train.py:232 step:501K smpl:8M ep:41K epch:137.99 loss:0.004 grdn:0.150 lr:4.1e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 00:53:15 ts/train.py:232 step:501K smpl:8M ep:41K epch:138.05 loss:0.003 grdn:0.147 lr:4.1e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 00:54:00 ts/train.py:232 step:501K smpl:8M ep:41K epch:138.10 loss:0.004 grdn:0.146 lr:4.1e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 00:54:46 ts/train.py:232 step:502K smpl:8M ep:41K epch:138.16 loss:0.004 grdn:0.155 lr:4.1e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 00:55:33 ts/train.py:232 step:502K smpl:8M ep:41K epch:138.21 loss:0.004 grdn:0.156 lr:4.1e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 00:56:19 ts/train.py:232 step:502K smpl:8M ep:41K epch:138.27 loss:0.003 grdn:0.143 lr:4.1e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 00:57:05 ts/train.py:232 step:502K smpl:8M ep:41K epch:138.32 loss:0.004 grdn:0.151 lr:4.1e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 00:57:51 ts/train.py:232 step:502K smpl:8M ep:42K epch:138.38 loss:0.004 grdn:0.151 lr:4.1e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 00:58:36 ts/train.py:232 step:503K smpl:8M ep:42K epch:138.43 loss:0.004 grdn:0.159 lr:4.1e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 00:59:22 ts/train.py:232 step:503K smpl:8M ep:42K epch:138.49 loss:0.004 grdn:0.156 lr:4.1e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 01:00:08 ts/train.py:232 step:503K smpl:8M ep:42K epch:138.54 loss:0.004 grdn:0.160 lr:4.1e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 01:00:54 ts/train.py:232 step:503K smpl:8M ep:42K epch:138.60 loss:0.004 grdn:0.150 lr:4.1e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 01:01:40 ts/train.py:232 step:503K smpl:8M ep:42K epch:138.65 loss:0.004 grdn:0.149 lr:4.1e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 01:02:27 ts/train.py:232 step:504K smpl:8M ep:42K epch:138.71 loss:0.004 grdn:0.169 lr:4.1e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 01:03:14 ts/train.py:232 step:504K smpl:8M ep:42K epch:138.76 loss:0.004 grdn:0.150 lr:4.1e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 01:04:00 ts/train.py:232 step:504K smpl:8M ep:42K epch:138.82 loss:0.004 grdn:0.147 lr:4.1e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 01:04:53 ts/train.py:232 step:504K smpl:8M ep:42K epch:138.87 loss:0.004 grdn:0.152 lr:4.1e-05 updt_s:0.157 data_s:0.107 +INFO 2025-05-07 01:05:42 ts/train.py:232 step:504K smpl:8M ep:42K epch:138.93 loss:0.004 grdn:0.163 lr:4.1e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 01:06:32 ts/train.py:232 step:505K smpl:8M ep:42K epch:138.98 loss:0.004 grdn:0.158 lr:4.1e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 01:07:21 ts/train.py:232 step:505K smpl:8M ep:42K epch:139.04 loss:0.003 grdn:0.146 lr:4.1e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 01:08:09 ts/train.py:232 step:505K smpl:8M ep:42K epch:139.09 loss:0.004 grdn:0.159 lr:4.1e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 01:08:57 ts/train.py:232 step:505K smpl:8M ep:42K epch:139.15 loss:0.004 grdn:0.158 lr:4.0e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 01:09:45 ts/train.py:232 step:505K smpl:8M ep:42K epch:139.20 loss:0.004 grdn:0.151 lr:4.0e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-07 01:10:34 ts/train.py:232 step:506K smpl:8M ep:42K epch:139.26 loss:0.003 grdn:0.140 lr:4.0e-05 updt_s:0.158 data_s:0.087 +INFO 2025-05-07 01:11:23 ts/train.py:232 step:506K smpl:8M ep:42K epch:139.31 loss:0.004 grdn:0.151 lr:4.0e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 01:12:12 ts/train.py:232 step:506K smpl:8M ep:42K epch:139.37 loss:0.004 grdn:0.159 lr:4.0e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 01:13:03 ts/train.py:232 step:506K smpl:8M ep:42K epch:139.43 loss:0.004 grdn:0.155 lr:4.0e-05 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 01:13:51 ts/train.py:232 step:506K smpl:8M ep:42K epch:139.48 loss:0.004 grdn:0.153 lr:4.0e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 01:14:40 ts/train.py:232 step:507K smpl:8M ep:42K epch:139.54 loss:0.004 grdn:0.152 lr:4.0e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 01:15:31 ts/train.py:232 step:507K smpl:8M ep:42K epch:139.59 loss:0.004 grdn:0.156 lr:4.0e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-07 01:16:22 ts/train.py:232 step:507K smpl:8M ep:42K epch:139.65 loss:0.004 grdn:0.156 lr:4.0e-05 updt_s:0.158 data_s:0.096 +INFO 2025-05-07 01:17:11 ts/train.py:232 step:507K smpl:8M ep:42K epch:139.70 loss:0.004 grdn:0.150 lr:4.0e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 01:18:00 ts/train.py:232 step:507K smpl:8M ep:42K epch:139.76 loss:0.004 grdn:0.160 lr:4.0e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 01:18:50 ts/train.py:232 step:508K smpl:8M ep:42K epch:139.81 loss:0.004 grdn:0.156 lr:4.0e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 01:19:36 ts/train.py:232 step:508K smpl:8M ep:42K epch:139.87 loss:0.004 grdn:0.155 lr:4.0e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 01:20:22 ts/train.py:232 step:508K smpl:8M ep:42K epch:139.92 loss:0.003 grdn:0.148 lr:4.0e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 01:21:08 ts/train.py:232 step:508K smpl:8M ep:42K epch:139.98 loss:0.003 grdn:0.145 lr:4.0e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 01:21:53 ts/train.py:232 step:508K smpl:8M ep:42K epch:140.03 loss:0.004 grdn:0.150 lr:4.0e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 01:22:39 ts/train.py:232 step:509K smpl:8M ep:42K epch:140.09 loss:0.004 grdn:0.163 lr:4.0e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 01:23:25 ts/train.py:232 step:509K smpl:8M ep:42K epch:140.14 loss:0.004 grdn:0.156 lr:4.0e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-07 01:24:10 ts/train.py:232 step:509K smpl:8M ep:42K epch:140.20 loss:0.004 grdn:0.154 lr:4.0e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 01:24:56 ts/train.py:232 step:509K smpl:8M ep:42K epch:140.25 loss:0.004 grdn:0.160 lr:4.0e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 01:25:42 ts/train.py:232 step:509K smpl:8M ep:42K epch:140.31 loss:0.004 grdn:0.158 lr:4.0e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 01:26:28 ts/train.py:232 step:510K smpl:8M ep:42K epch:140.36 loss:0.004 grdn:0.160 lr:4.0e-05 updt_s:0.159 data_s:0.070 +INFO 2025-05-07 01:27:14 ts/train.py:232 step:510K smpl:8M ep:42K epch:140.42 loss:0.004 grdn:0.158 lr:4.0e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 01:27:59 ts/train.py:232 step:510K smpl:8M ep:42K epch:140.47 loss:0.004 grdn:0.165 lr:4.0e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 01:27:59 ts/train.py:241 Checkpoint policy after step 510000 +INFO 2025-05-07 01:28:48 ts/train.py:232 step:510K smpl:8M ep:42K epch:140.53 loss:0.004 grdn:0.156 lr:4.0e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 01:29:34 ts/train.py:232 step:510K smpl:8M ep:42K epch:140.58 loss:0.004 grdn:0.164 lr:4.0e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 01:30:20 ts/train.py:232 step:511K smpl:8M ep:42K epch:140.64 loss:0.004 grdn:0.148 lr:4.0e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 01:31:04 ts/train.py:232 step:511K smpl:8M ep:42K epch:140.69 loss:0.003 grdn:0.145 lr:4.0e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 01:31:49 ts/train.py:232 step:511K smpl:8M ep:42K epch:140.75 loss:0.003 grdn:0.141 lr:3.9e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 01:32:39 ts/train.py:232 step:511K smpl:8M ep:42K epch:140.80 loss:0.004 grdn:0.149 lr:3.9e-05 updt_s:0.157 data_s:0.094 +INFO 2025-05-07 01:33:28 ts/train.py:232 step:511K smpl:8M ep:42K epch:140.86 loss:0.003 grdn:0.146 lr:3.9e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 01:34:16 ts/train.py:232 step:512K smpl:8M ep:42K epch:140.91 loss:0.005 grdn:0.175 lr:3.9e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 01:35:05 ts/train.py:232 step:512K smpl:8M ep:42K epch:140.97 loss:0.004 grdn:0.158 lr:3.9e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 01:35:57 ts/train.py:232 step:512K smpl:8M ep:42K epch:141.02 loss:0.004 grdn:0.145 lr:3.9e-05 updt_s:0.157 data_s:0.102 +INFO 2025-05-07 01:36:47 ts/train.py:232 step:512K smpl:8M ep:42K epch:141.08 loss:0.004 grdn:0.154 lr:3.9e-05 updt_s:0.158 data_s:0.091 +INFO 2025-05-07 01:37:38 ts/train.py:232 step:512K smpl:8M ep:42K epch:141.13 loss:0.004 grdn:0.161 lr:3.9e-05 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 01:38:29 ts/train.py:232 step:513K smpl:8M ep:42K epch:141.19 loss:0.003 grdn:0.145 lr:3.9e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-07 01:39:21 ts/train.py:232 step:513K smpl:8M ep:42K epch:141.24 loss:0.003 grdn:0.143 lr:3.9e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-07 01:40:11 ts/train.py:232 step:513K smpl:8M ep:42K epch:141.30 loss:0.004 grdn:0.152 lr:3.9e-05 updt_s:0.157 data_s:0.094 +INFO 2025-05-07 01:41:02 ts/train.py:232 step:513K smpl:8M ep:42K epch:141.35 loss:0.004 grdn:0.157 lr:3.9e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-07 01:41:54 ts/train.py:232 step:513K smpl:8M ep:42K epch:141.41 loss:0.004 grdn:0.163 lr:3.9e-05 updt_s:0.157 data_s:0.100 +INFO 2025-05-07 01:42:45 ts/train.py:232 step:514K smpl:8M ep:42K epch:141.46 loss:0.003 grdn:0.152 lr:3.9e-05 updt_s:0.157 data_s:0.097 +INFO 2025-05-07 01:43:35 ts/train.py:232 step:514K smpl:8M ep:42K epch:141.52 loss:0.003 grdn:0.142 lr:3.9e-05 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 01:44:26 ts/train.py:232 step:514K smpl:8M ep:42K epch:141.57 loss:0.004 grdn:0.155 lr:3.9e-05 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 01:45:17 ts/train.py:232 step:514K smpl:8M ep:42K epch:141.63 loss:0.004 grdn:0.149 lr:3.9e-05 updt_s:0.157 data_s:0.096 +INFO 2025-05-07 01:46:07 ts/train.py:232 step:514K smpl:8M ep:43K epch:141.68 loss:0.004 grdn:0.153 lr:3.9e-05 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 01:47:00 ts/train.py:232 step:515K smpl:8M ep:43K epch:141.74 loss:0.003 grdn:0.151 lr:3.9e-05 updt_s:0.158 data_s:0.103 +INFO 2025-05-07 01:47:47 ts/train.py:232 step:515K smpl:8M ep:43K epch:141.79 loss:0.003 grdn:0.153 lr:3.9e-05 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 01:48:34 ts/train.py:232 step:515K smpl:8M ep:43K epch:141.85 loss:0.004 grdn:0.158 lr:3.9e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 01:49:22 ts/train.py:232 step:515K smpl:8M ep:43K epch:141.90 loss:0.004 grdn:0.155 lr:3.9e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 01:50:09 ts/train.py:232 step:515K smpl:8M ep:43K epch:141.96 loss:0.004 grdn:0.151 lr:3.9e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 01:50:56 ts/train.py:232 step:516K smpl:8M ep:43K epch:142.01 loss:0.004 grdn:0.151 lr:3.9e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 01:51:42 ts/train.py:232 step:516K smpl:8M ep:43K epch:142.07 loss:0.004 grdn:0.157 lr:3.9e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 01:52:28 ts/train.py:232 step:516K smpl:8M ep:43K epch:142.12 loss:0.003 grdn:0.150 lr:3.9e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 01:53:14 ts/train.py:232 step:516K smpl:8M ep:43K epch:142.18 loss:0.004 grdn:0.161 lr:3.9e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 01:53:59 ts/train.py:232 step:516K smpl:8M ep:43K epch:142.23 loss:0.003 grdn:0.146 lr:3.9e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 01:54:45 ts/train.py:232 step:517K smpl:8M ep:43K epch:142.29 loss:0.004 grdn:0.155 lr:3.9e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 01:55:32 ts/train.py:232 step:517K smpl:8M ep:43K epch:142.34 loss:0.003 grdn:0.151 lr:3.8e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 01:56:18 ts/train.py:232 step:517K smpl:8M ep:43K epch:142.40 loss:0.004 grdn:0.154 lr:3.8e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 01:57:07 ts/train.py:232 step:517K smpl:8M ep:43K epch:142.45 loss:0.004 grdn:0.141 lr:3.8e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 01:57:53 ts/train.py:232 step:517K smpl:8M ep:43K epch:142.51 loss:0.003 grdn:0.151 lr:3.8e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 01:58:41 ts/train.py:232 step:518K smpl:8M ep:43K epch:142.56 loss:0.004 grdn:0.156 lr:3.8e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 01:59:26 ts/train.py:232 step:518K smpl:8M ep:43K epch:142.62 loss:0.003 grdn:0.150 lr:3.8e-05 updt_s:0.158 data_s:0.069 +INFO 2025-05-07 02:00:13 ts/train.py:232 step:518K smpl:8M ep:43K epch:142.68 loss:0.003 grdn:0.149 lr:3.8e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 02:01:03 ts/train.py:232 step:518K smpl:8M ep:43K epch:142.73 loss:0.003 grdn:0.153 lr:3.8e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 02:01:51 ts/train.py:232 step:518K smpl:8M ep:43K epch:142.79 loss:0.003 grdn:0.152 lr:3.8e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 02:02:39 ts/train.py:232 step:519K smpl:8M ep:43K epch:142.84 loss:0.004 grdn:0.156 lr:3.8e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 02:03:26 ts/train.py:232 step:519K smpl:8M ep:43K epch:142.90 loss:0.004 grdn:0.152 lr:3.8e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 02:04:13 ts/train.py:232 step:519K smpl:8M ep:43K epch:142.95 loss:0.004 grdn:0.154 lr:3.8e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 02:05:00 ts/train.py:232 step:519K smpl:8M ep:43K epch:143.01 loss:0.004 grdn:0.159 lr:3.8e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 02:05:47 ts/train.py:232 step:519K smpl:8M ep:43K epch:143.06 loss:0.003 grdn:0.150 lr:3.8e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 02:06:35 ts/train.py:232 step:520K smpl:8M ep:43K epch:143.12 loss:0.004 grdn:0.153 lr:3.8e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 02:07:22 ts/train.py:232 step:520K smpl:8M ep:43K epch:143.17 loss:0.003 grdn:0.151 lr:3.8e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 02:08:10 ts/train.py:232 step:520K smpl:8M ep:43K epch:143.23 loss:0.003 grdn:0.150 lr:3.8e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 02:08:58 ts/train.py:232 step:520K smpl:8M ep:43K epch:143.28 loss:0.003 grdn:0.141 lr:3.8e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 02:09:45 ts/train.py:232 step:520K smpl:8M ep:43K epch:143.34 loss:0.003 grdn:0.145 lr:3.8e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 02:10:32 ts/train.py:232 step:521K smpl:8M ep:43K epch:143.39 loss:0.004 grdn:0.159 lr:3.8e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 02:11:20 ts/train.py:232 step:521K smpl:8M ep:43K epch:143.45 loss:0.004 grdn:0.165 lr:3.8e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 02:12:07 ts/train.py:232 step:521K smpl:8M ep:43K epch:143.50 loss:0.003 grdn:0.146 lr:3.8e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 02:12:56 ts/train.py:232 step:521K smpl:8M ep:43K epch:143.56 loss:0.004 grdn:0.152 lr:3.8e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 02:13:43 ts/train.py:232 step:521K smpl:8M ep:43K epch:143.61 loss:0.003 grdn:0.137 lr:3.8e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 02:14:32 ts/train.py:232 step:522K smpl:8M ep:43K epch:143.67 loss:0.004 grdn:0.149 lr:3.8e-05 updt_s:0.158 data_s:0.089 +INFO 2025-05-07 02:15:21 ts/train.py:232 step:522K smpl:8M ep:43K epch:143.72 loss:0.003 grdn:0.146 lr:3.8e-05 updt_s:0.158 data_s:0.084 +INFO 2025-05-07 02:16:09 ts/train.py:232 step:522K smpl:8M ep:43K epch:143.78 loss:0.004 grdn:0.153 lr:3.8e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 02:16:57 ts/train.py:232 step:522K smpl:8M ep:43K epch:143.83 loss:0.003 grdn:0.143 lr:3.8e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 02:17:43 ts/train.py:232 step:522K smpl:8M ep:43K epch:143.89 loss:0.003 grdn:0.154 lr:3.8e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 02:18:29 ts/train.py:232 step:523K smpl:8M ep:43K epch:143.94 loss:0.003 grdn:0.148 lr:3.8e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 02:19:15 ts/train.py:232 step:523K smpl:8M ep:43K epch:144.00 loss:0.004 grdn:0.151 lr:3.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 02:20:00 ts/train.py:232 step:523K smpl:8M ep:43K epch:144.05 loss:0.004 grdn:0.151 lr:3.7e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 02:20:45 ts/train.py:232 step:523K smpl:8M ep:43K epch:144.11 loss:0.003 grdn:0.142 lr:3.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 02:21:30 ts/train.py:232 step:523K smpl:8M ep:43K epch:144.16 loss:0.003 grdn:0.150 lr:3.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 02:22:15 ts/train.py:232 step:524K smpl:8M ep:43K epch:144.22 loss:0.003 grdn:0.143 lr:3.7e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 02:23:00 ts/train.py:232 step:524K smpl:8M ep:43K epch:144.27 loss:0.003 grdn:0.145 lr:3.7e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 02:23:45 ts/train.py:232 step:524K smpl:8M ep:43K epch:144.33 loss:0.003 grdn:0.140 lr:3.7e-05 updt_s:0.159 data_s:0.067 +INFO 2025-05-07 02:24:30 ts/train.py:232 step:524K smpl:8M ep:43K epch:144.38 loss:0.003 grdn:0.149 lr:3.7e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 02:25:16 ts/train.py:232 step:524K smpl:8M ep:43K epch:144.44 loss:0.003 grdn:0.146 lr:3.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 02:26:01 ts/train.py:232 step:525K smpl:8M ep:43K epch:144.49 loss:0.004 grdn:0.165 lr:3.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 02:26:46 ts/train.py:232 step:525K smpl:8M ep:43K epch:144.55 loss:0.003 grdn:0.140 lr:3.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 02:27:31 ts/train.py:232 step:525K smpl:8M ep:43K epch:144.60 loss:0.004 grdn:0.158 lr:3.7e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 02:28:21 ts/train.py:232 step:525K smpl:8M ep:43K epch:144.66 loss:0.004 grdn:0.148 lr:3.7e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 02:29:08 ts/train.py:232 step:525K smpl:8M ep:43K epch:144.71 loss:0.004 grdn:0.154 lr:3.7e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 02:29:56 ts/train.py:232 step:526K smpl:8M ep:43K epch:144.77 loss:0.004 grdn:0.158 lr:3.7e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 02:30:44 ts/train.py:232 step:526K smpl:8M ep:43K epch:144.82 loss:0.004 grdn:0.150 lr:3.7e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 02:31:29 ts/train.py:232 step:526K smpl:8M ep:43K epch:144.88 loss:0.004 grdn:0.163 lr:3.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 02:32:15 ts/train.py:232 step:526K smpl:8M ep:43K epch:144.93 loss:0.003 grdn:0.147 lr:3.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 02:33:01 ts/train.py:232 step:526K smpl:8M ep:43K epch:144.99 loss:0.003 grdn:0.145 lr:3.7e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 02:33:46 ts/train.py:232 step:527K smpl:8M ep:44K epch:145.04 loss:0.004 grdn:0.154 lr:3.7e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 02:34:31 ts/train.py:232 step:527K smpl:8M ep:44K epch:145.10 loss:0.004 grdn:0.151 lr:3.7e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 02:35:16 ts/train.py:232 step:527K smpl:8M ep:44K epch:145.15 loss:0.003 grdn:0.149 lr:3.7e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 02:36:01 ts/train.py:232 step:527K smpl:8M ep:44K epch:145.21 loss:0.004 grdn:0.167 lr:3.7e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 02:36:46 ts/train.py:232 step:527K smpl:8M ep:44K epch:145.26 loss:0.004 grdn:0.155 lr:3.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 02:37:32 ts/train.py:232 step:528K smpl:8M ep:44K epch:145.32 loss:0.003 grdn:0.151 lr:3.7e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 02:38:17 ts/train.py:232 step:528K smpl:8M ep:44K epch:145.37 loss:0.004 grdn:0.156 lr:3.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 02:39:03 ts/train.py:232 step:528K smpl:8M ep:44K epch:145.43 loss:0.003 grdn:0.149 lr:3.7e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 02:39:48 ts/train.py:232 step:528K smpl:8M ep:44K epch:145.48 loss:0.004 grdn:0.154 lr:3.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 02:40:33 ts/train.py:232 step:528K smpl:8M ep:44K epch:145.54 loss:0.004 grdn:0.157 lr:3.7e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 02:41:21 ts/train.py:232 step:529K smpl:8M ep:44K epch:145.59 loss:0.003 grdn:0.144 lr:3.7e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 02:42:09 ts/train.py:232 step:529K smpl:8M ep:44K epch:145.65 loss:0.003 grdn:0.148 lr:3.6e-05 updt_s:0.158 data_s:0.081 +INFO 2025-05-07 02:42:57 ts/train.py:232 step:529K smpl:8M ep:44K epch:145.70 loss:0.003 grdn:0.147 lr:3.6e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 02:43:46 ts/train.py:232 step:529K smpl:8M ep:44K epch:145.76 loss:0.003 grdn:0.146 lr:3.6e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 02:44:32 ts/train.py:232 step:529K smpl:8M ep:44K epch:145.82 loss:0.003 grdn:0.151 lr:3.6e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 02:45:18 ts/train.py:232 step:530K smpl:8M ep:44K epch:145.87 loss:0.004 grdn:0.155 lr:3.6e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 02:46:04 ts/train.py:232 step:530K smpl:8M ep:44K epch:145.93 loss:0.003 grdn:0.144 lr:3.6e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 02:46:51 ts/train.py:232 step:530K smpl:8M ep:44K epch:145.98 loss:0.003 grdn:0.147 lr:3.6e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 02:47:39 ts/train.py:232 step:530K smpl:8M ep:44K epch:146.04 loss:0.004 grdn:0.151 lr:3.6e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 02:48:26 ts/train.py:232 step:530K smpl:8M ep:44K epch:146.09 loss:0.003 grdn:0.146 lr:3.6e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 02:49:13 ts/train.py:232 step:531K smpl:8M ep:44K epch:146.15 loss:0.003 grdn:0.150 lr:3.6e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 02:50:00 ts/train.py:232 step:531K smpl:8M ep:44K epch:146.20 loss:0.003 grdn:0.156 lr:3.6e-05 updt_s:0.158 data_s:0.077 +INFO 2025-05-07 02:50:47 ts/train.py:232 step:531K smpl:8M ep:44K epch:146.26 loss:0.003 grdn:0.147 lr:3.6e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 02:51:34 ts/train.py:232 step:531K smpl:8M ep:44K epch:146.31 loss:0.004 grdn:0.157 lr:3.6e-05 updt_s:0.158 data_s:0.074 +INFO 2025-05-07 02:52:21 ts/train.py:232 step:531K smpl:9M ep:44K epch:146.37 loss:0.003 grdn:0.154 lr:3.6e-05 updt_s:0.158 data_s:0.079 +INFO 2025-05-07 02:53:08 ts/train.py:232 step:532K smpl:9M ep:44K epch:146.42 loss:0.003 grdn:0.150 lr:3.6e-05 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 02:53:54 ts/train.py:232 step:532K smpl:9M ep:44K epch:146.48 loss:0.003 grdn:0.153 lr:3.6e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 02:54:42 ts/train.py:232 step:532K smpl:9M ep:44K epch:146.53 loss:0.003 grdn:0.145 lr:3.6e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 02:55:32 ts/train.py:232 step:532K smpl:9M ep:44K epch:146.59 loss:0.003 grdn:0.146 lr:3.6e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 02:56:19 ts/train.py:232 step:532K smpl:9M ep:44K epch:146.64 loss:0.004 grdn:0.150 lr:3.6e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 02:57:05 ts/train.py:232 step:533K smpl:9M ep:44K epch:146.70 loss:0.004 grdn:0.159 lr:3.6e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 02:57:52 ts/train.py:232 step:533K smpl:9M ep:44K epch:146.75 loss:0.004 grdn:0.151 lr:3.6e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 02:58:38 ts/train.py:232 step:533K smpl:9M ep:44K epch:146.81 loss:0.004 grdn:0.165 lr:3.6e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 02:59:23 ts/train.py:232 step:533K smpl:9M ep:44K epch:146.86 loss:0.004 grdn:0.153 lr:3.6e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 03:00:09 ts/train.py:232 step:533K smpl:9M ep:44K epch:146.92 loss:0.003 grdn:0.141 lr:3.6e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 03:00:55 ts/train.py:232 step:534K smpl:9M ep:44K epch:146.97 loss:0.003 grdn:0.143 lr:3.6e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 03:01:40 ts/train.py:232 step:534K smpl:9M ep:44K epch:147.03 loss:0.003 grdn:0.155 lr:3.6e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 03:02:25 ts/train.py:232 step:534K smpl:9M ep:44K epch:147.08 loss:0.003 grdn:0.142 lr:3.6e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 03:03:10 ts/train.py:232 step:534K smpl:9M ep:44K epch:147.14 loss:0.003 grdn:0.153 lr:3.6e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 03:03:55 ts/train.py:232 step:534K smpl:9M ep:44K epch:147.19 loss:0.003 grdn:0.151 lr:3.6e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 03:04:39 ts/train.py:232 step:535K smpl:9M ep:44K epch:147.25 loss:0.003 grdn:0.145 lr:3.5e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 03:05:24 ts/train.py:232 step:535K smpl:9M ep:44K epch:147.30 loss:0.003 grdn:0.141 lr:3.5e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 03:06:10 ts/train.py:232 step:535K smpl:9M ep:44K epch:147.36 loss:0.004 grdn:0.165 lr:3.5e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 03:06:57 ts/train.py:232 step:535K smpl:9M ep:44K epch:147.41 loss:0.003 grdn:0.152 lr:3.5e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 03:07:43 ts/train.py:232 step:535K smpl:9M ep:44K epch:147.47 loss:0.004 grdn:0.173 lr:3.5e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 03:08:29 ts/train.py:232 step:536K smpl:9M ep:44K epch:147.52 loss:0.003 grdn:0.147 lr:3.5e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 03:09:13 ts/train.py:232 step:536K smpl:9M ep:44K epch:147.58 loss:0.003 grdn:0.145 lr:3.5e-05 updt_s:0.157 data_s:0.061 +INFO 2025-05-07 03:09:58 ts/train.py:232 step:536K smpl:9M ep:44K epch:147.63 loss:0.004 grdn:0.160 lr:3.5e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 03:10:43 ts/train.py:232 step:536K smpl:9M ep:44K epch:147.69 loss:0.003 grdn:0.136 lr:3.5e-05 updt_s:0.158 data_s:0.068 +INFO 2025-05-07 03:11:27 ts/train.py:232 step:536K smpl:9M ep:44K epch:147.74 loss:0.003 grdn:0.144 lr:3.5e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 03:12:13 ts/train.py:232 step:537K smpl:9M ep:44K epch:147.80 loss:0.003 grdn:0.146 lr:3.5e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 03:12:57 ts/train.py:232 step:537K smpl:9M ep:44K epch:147.85 loss:0.003 grdn:0.148 lr:3.5e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 03:13:42 ts/train.py:232 step:537K smpl:9M ep:44K epch:147.91 loss:0.003 grdn:0.150 lr:3.5e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 03:14:27 ts/train.py:232 step:537K smpl:9M ep:44K epch:147.96 loss:0.004 grdn:0.158 lr:3.5e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 03:15:12 ts/train.py:232 step:537K smpl:9M ep:44K epch:148.02 loss:0.003 grdn:0.154 lr:3.5e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 03:15:57 ts/train.py:232 step:538K smpl:9M ep:44K epch:148.07 loss:0.003 grdn:0.153 lr:3.5e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 03:16:41 ts/train.py:232 step:538K smpl:9M ep:44K epch:148.13 loss:0.003 grdn:0.151 lr:3.5e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 03:17:26 ts/train.py:232 step:538K smpl:9M ep:44K epch:148.18 loss:0.003 grdn:0.143 lr:3.5e-05 updt_s:0.158 data_s:0.064 +INFO 2025-05-07 03:18:10 ts/train.py:232 step:538K smpl:9M ep:44K epch:148.24 loss:0.003 grdn:0.140 lr:3.5e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 03:18:55 ts/train.py:232 step:538K smpl:9M ep:44K epch:148.29 loss:0.003 grdn:0.153 lr:3.5e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 03:19:41 ts/train.py:232 step:539K smpl:9M ep:45K epch:148.35 loss:0.004 grdn:0.161 lr:3.5e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 03:20:25 ts/train.py:232 step:539K smpl:9M ep:45K epch:148.40 loss:0.004 grdn:0.158 lr:3.5e-05 updt_s:0.158 data_s:0.066 +INFO 2025-05-07 03:21:10 ts/train.py:232 step:539K smpl:9M ep:45K epch:148.46 loss:0.003 grdn:0.141 lr:3.5e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 03:22:00 ts/train.py:232 step:539K smpl:9M ep:45K epch:148.51 loss:0.004 grdn:0.158 lr:3.5e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 03:22:49 ts/train.py:232 step:539K smpl:9M ep:45K epch:148.57 loss:0.003 grdn:0.145 lr:3.5e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 03:23:37 ts/train.py:232 step:540K smpl:9M ep:45K epch:148.62 loss:0.004 grdn:0.161 lr:3.5e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 03:24:24 ts/train.py:232 step:540K smpl:9M ep:45K epch:148.68 loss:0.004 grdn:0.171 lr:3.5e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 03:25:11 ts/train.py:232 step:540K smpl:9M ep:45K epch:148.73 loss:0.003 grdn:0.153 lr:3.5e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 03:25:11 ts/train.py:241 Checkpoint policy after step 540000 +INFO 2025-05-07 03:26:01 ts/train.py:232 step:540K smpl:9M ep:45K epch:148.79 loss:0.003 grdn:0.152 lr:3.5e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 03:26:48 ts/train.py:232 step:540K smpl:9M ep:45K epch:148.84 loss:0.003 grdn:0.144 lr:3.5e-05 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 03:27:36 ts/train.py:232 step:541K smpl:9M ep:45K epch:148.90 loss:0.003 grdn:0.147 lr:3.4e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 03:28:24 ts/train.py:232 step:541K smpl:9M ep:45K epch:148.96 loss:0.003 grdn:0.146 lr:3.4e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 03:29:12 ts/train.py:232 step:541K smpl:9M ep:45K epch:149.01 loss:0.003 grdn:0.146 lr:3.4e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 03:30:00 ts/train.py:232 step:541K smpl:9M ep:45K epch:149.07 loss:0.003 grdn:0.153 lr:3.4e-05 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 03:30:47 ts/train.py:232 step:541K smpl:9M ep:45K epch:149.12 loss:0.003 grdn:0.155 lr:3.4e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 03:31:34 ts/train.py:232 step:542K smpl:9M ep:45K epch:149.18 loss:0.003 grdn:0.150 lr:3.4e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 03:32:21 ts/train.py:232 step:542K smpl:9M ep:45K epch:149.23 loss:0.003 grdn:0.159 lr:3.4e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 03:33:09 ts/train.py:232 step:542K smpl:9M ep:45K epch:149.29 loss:0.003 grdn:0.156 lr:3.4e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 03:33:57 ts/train.py:232 step:542K smpl:9M ep:45K epch:149.34 loss:0.003 grdn:0.146 lr:3.4e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 03:34:44 ts/train.py:232 step:542K smpl:9M ep:45K epch:149.40 loss:0.003 grdn:0.147 lr:3.4e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 03:35:32 ts/train.py:232 step:543K smpl:9M ep:45K epch:149.45 loss:0.003 grdn:0.155 lr:3.4e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 03:36:18 ts/train.py:232 step:543K smpl:9M ep:45K epch:149.51 loss:0.003 grdn:0.150 lr:3.4e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 03:37:04 ts/train.py:232 step:543K smpl:9M ep:45K epch:149.56 loss:0.003 grdn:0.154 lr:3.4e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 03:37:49 ts/train.py:232 step:543K smpl:9M ep:45K epch:149.62 loss:0.004 grdn:0.156 lr:3.4e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 03:38:33 ts/train.py:232 step:543K smpl:9M ep:45K epch:149.67 loss:0.003 grdn:0.152 lr:3.4e-05 updt_s:0.157 data_s:0.062 +INFO 2025-05-07 03:39:17 ts/train.py:232 step:544K smpl:9M ep:45K epch:149.73 loss:0.003 grdn:0.154 lr:3.4e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 03:40:02 ts/train.py:232 step:544K smpl:9M ep:45K epch:149.78 loss:0.003 grdn:0.142 lr:3.4e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 03:40:46 ts/train.py:232 step:544K smpl:9M ep:45K epch:149.84 loss:0.003 grdn:0.148 lr:3.4e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 03:41:31 ts/train.py:232 step:544K smpl:9M ep:45K epch:149.89 loss:0.003 grdn:0.147 lr:3.4e-05 updt_s:0.158 data_s:0.065 +INFO 2025-05-07 03:42:15 ts/train.py:232 step:544K smpl:9M ep:45K epch:149.95 loss:0.004 grdn:0.157 lr:3.4e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 03:43:00 ts/train.py:232 step:545K smpl:9M ep:45K epch:150.00 loss:0.003 grdn:0.144 lr:3.4e-05 updt_s:0.156 data_s:0.066 +INFO 2025-05-07 03:43:45 ts/train.py:232 step:545K smpl:9M ep:45K epch:150.06 loss:0.003 grdn:0.147 lr:3.4e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 03:44:31 ts/train.py:232 step:545K smpl:9M ep:45K epch:150.11 loss:0.003 grdn:0.149 lr:3.4e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 03:45:15 ts/train.py:232 step:545K smpl:9M ep:45K epch:150.17 loss:0.004 grdn:0.159 lr:3.4e-05 updt_s:0.156 data_s:0.066 +INFO 2025-05-07 03:46:00 ts/train.py:232 step:545K smpl:9M ep:45K epch:150.22 loss:0.004 grdn:0.157 lr:3.4e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 03:46:45 ts/train.py:232 step:546K smpl:9M ep:45K epch:150.28 loss:0.003 grdn:0.158 lr:3.4e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 03:47:30 ts/train.py:232 step:546K smpl:9M ep:45K epch:150.33 loss:0.003 grdn:0.145 lr:3.4e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 03:48:15 ts/train.py:232 step:546K smpl:9M ep:45K epch:150.39 loss:0.003 grdn:0.156 lr:3.4e-05 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 03:49:04 ts/train.py:232 step:546K smpl:9M ep:45K epch:150.44 loss:0.003 grdn:0.151 lr:3.4e-05 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 03:49:52 ts/train.py:232 step:546K smpl:9M ep:45K epch:150.50 loss:0.003 grdn:0.151 lr:3.4e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 03:50:38 ts/train.py:232 step:547K smpl:9M ep:45K epch:150.55 loss:0.003 grdn:0.160 lr:3.4e-05 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 03:51:24 ts/train.py:232 step:547K smpl:9M ep:45K epch:150.61 loss:0.003 grdn:0.151 lr:3.3e-05 updt_s:0.158 data_s:0.072 +INFO 2025-05-07 03:52:10 ts/train.py:232 step:547K smpl:9M ep:45K epch:150.66 loss:0.003 grdn:0.151 lr:3.3e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 03:52:55 ts/train.py:232 step:547K smpl:9M ep:45K epch:150.72 loss:0.003 grdn:0.142 lr:3.3e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 03:53:41 ts/train.py:232 step:547K smpl:9M ep:45K epch:150.77 loss:0.003 grdn:0.157 lr:3.3e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 03:54:26 ts/train.py:232 step:548K smpl:9M ep:45K epch:150.83 loss:0.004 grdn:0.160 lr:3.3e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 03:55:11 ts/train.py:232 step:548K smpl:9M ep:45K epch:150.88 loss:0.003 grdn:0.139 lr:3.3e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 03:55:56 ts/train.py:232 step:548K smpl:9M ep:45K epch:150.94 loss:0.004 grdn:0.164 lr:3.3e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 03:56:41 ts/train.py:232 step:548K smpl:9M ep:45K epch:150.99 loss:0.003 grdn:0.156 lr:3.3e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 03:57:26 ts/train.py:232 step:548K smpl:9M ep:45K epch:151.05 loss:0.003 grdn:0.151 lr:3.3e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 03:58:12 ts/train.py:232 step:549K smpl:9M ep:45K epch:151.10 loss:0.003 grdn:0.141 lr:3.3e-05 updt_s:0.156 data_s:0.070 +INFO 2025-05-07 03:58:57 ts/train.py:232 step:549K smpl:9M ep:45K epch:151.16 loss:0.003 grdn:0.154 lr:3.3e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 03:59:42 ts/train.py:232 step:549K smpl:9M ep:45K epch:151.21 loss:0.004 grdn:0.167 lr:3.3e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 04:00:27 ts/train.py:232 step:549K smpl:9M ep:45K epch:151.27 loss:0.003 grdn:0.148 lr:3.3e-05 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 04:01:12 ts/train.py:232 step:549K smpl:9M ep:45K epch:151.32 loss:0.003 grdn:0.144 lr:3.3e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 04:02:01 ts/train.py:232 step:550K smpl:9M ep:45K epch:151.38 loss:0.003 grdn:0.146 lr:3.3e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 04:02:50 ts/train.py:232 step:550K smpl:9M ep:45K epch:151.43 loss:0.004 grdn:0.158 lr:3.3e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 04:03:38 ts/train.py:232 step:550K smpl:9M ep:45K epch:151.49 loss:0.003 grdn:0.156 lr:3.3e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 04:04:27 ts/train.py:232 step:550K smpl:9M ep:45K epch:151.54 loss:0.003 grdn:0.143 lr:3.3e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 04:05:16 ts/train.py:232 step:550K smpl:9M ep:45K epch:151.60 loss:0.003 grdn:0.151 lr:3.3e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 04:06:07 ts/train.py:232 step:551K smpl:9M ep:45K epch:151.65 loss:0.003 grdn:0.140 lr:3.3e-05 updt_s:0.157 data_s:0.100 +INFO 2025-05-07 04:06:55 ts/train.py:232 step:551K smpl:9M ep:46K epch:151.71 loss:0.003 grdn:0.158 lr:3.3e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 04:07:42 ts/train.py:232 step:551K smpl:9M ep:46K epch:151.76 loss:0.004 grdn:0.166 lr:3.3e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 04:08:30 ts/train.py:232 step:551K smpl:9M ep:46K epch:151.82 loss:0.003 grdn:0.142 lr:3.3e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 04:09:24 ts/train.py:232 step:551K smpl:9M ep:46K epch:151.87 loss:0.003 grdn:0.146 lr:3.3e-05 updt_s:0.157 data_s:0.112 +INFO 2025-05-07 04:10:11 ts/train.py:232 step:552K smpl:9M ep:46K epch:151.93 loss:0.003 grdn:0.155 lr:3.3e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 04:11:00 ts/train.py:232 step:552K smpl:9M ep:46K epch:151.98 loss:0.003 grdn:0.153 lr:3.3e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 04:11:46 ts/train.py:232 step:552K smpl:9M ep:46K epch:152.04 loss:0.003 grdn:0.146 lr:3.3e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 04:12:34 ts/train.py:232 step:552K smpl:9M ep:46K epch:152.10 loss:0.003 grdn:0.162 lr:3.3e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 04:13:22 ts/train.py:232 step:552K smpl:9M ep:46K epch:152.15 loss:0.004 grdn:0.177 lr:3.3e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 04:14:09 ts/train.py:232 step:553K smpl:9M ep:46K epch:152.21 loss:0.003 grdn:0.146 lr:3.3e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 04:14:57 ts/train.py:232 step:553K smpl:9M ep:46K epch:152.26 loss:0.003 grdn:0.158 lr:3.2e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 04:15:45 ts/train.py:232 step:553K smpl:9M ep:46K epch:152.32 loss:0.003 grdn:0.152 lr:3.2e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 04:16:36 ts/train.py:232 step:553K smpl:9M ep:46K epch:152.37 loss:0.003 grdn:0.145 lr:3.2e-05 updt_s:0.156 data_s:0.100 +INFO 2025-05-07 04:17:24 ts/train.py:232 step:553K smpl:9M ep:46K epch:152.43 loss:0.004 grdn:0.161 lr:3.2e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 04:18:11 ts/train.py:232 step:554K smpl:9M ep:46K epch:152.48 loss:0.003 grdn:0.137 lr:3.2e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 04:18:58 ts/train.py:232 step:554K smpl:9M ep:46K epch:152.54 loss:0.003 grdn:0.162 lr:3.2e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 04:19:44 ts/train.py:232 step:554K smpl:9M ep:46K epch:152.59 loss:0.004 grdn:0.164 lr:3.2e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 04:20:31 ts/train.py:232 step:554K smpl:9M ep:46K epch:152.65 loss:0.003 grdn:0.144 lr:3.2e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 04:21:16 ts/train.py:232 step:554K smpl:9M ep:46K epch:152.70 loss:0.003 grdn:0.143 lr:3.2e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 04:22:04 ts/train.py:232 step:555K smpl:9M ep:46K epch:152.76 loss:0.004 grdn:0.160 lr:3.2e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 04:22:51 ts/train.py:232 step:555K smpl:9M ep:46K epch:152.81 loss:0.003 grdn:0.144 lr:3.2e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 04:23:40 ts/train.py:232 step:555K smpl:9M ep:46K epch:152.87 loss:0.003 grdn:0.161 lr:3.2e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 04:24:27 ts/train.py:232 step:555K smpl:9M ep:46K epch:152.92 loss:0.003 grdn:0.165 lr:3.2e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 04:25:15 ts/train.py:232 step:555K smpl:9M ep:46K epch:152.98 loss:0.003 grdn:0.153 lr:3.2e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 04:26:02 ts/train.py:232 step:556K smpl:9M ep:46K epch:153.03 loss:0.003 grdn:0.147 lr:3.2e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 04:26:49 ts/train.py:232 step:556K smpl:9M ep:46K epch:153.09 loss:0.003 grdn:0.156 lr:3.2e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 04:27:36 ts/train.py:232 step:556K smpl:9M ep:46K epch:153.14 loss:0.003 grdn:0.147 lr:3.2e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 04:28:23 ts/train.py:232 step:556K smpl:9M ep:46K epch:153.20 loss:0.004 grdn:0.166 lr:3.2e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 04:29:11 ts/train.py:232 step:556K smpl:9M ep:46K epch:153.25 loss:0.003 grdn:0.150 lr:3.2e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 04:30:01 ts/train.py:232 step:557K smpl:9M ep:46K epch:153.31 loss:0.003 grdn:0.137 lr:3.2e-05 updt_s:0.156 data_s:0.092 +INFO 2025-05-07 04:30:50 ts/train.py:232 step:557K smpl:9M ep:46K epch:153.36 loss:0.003 grdn:0.154 lr:3.2e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 04:31:38 ts/train.py:232 step:557K smpl:9M ep:46K epch:153.42 loss:0.003 grdn:0.139 lr:3.2e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 04:32:28 ts/train.py:232 step:557K smpl:9M ep:46K epch:153.47 loss:0.003 grdn:0.150 lr:3.2e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 04:33:17 ts/train.py:232 step:557K smpl:9M ep:46K epch:153.53 loss:0.004 grdn:0.166 lr:3.2e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 04:34:06 ts/train.py:232 step:558K smpl:9M ep:46K epch:153.58 loss:0.003 grdn:0.149 lr:3.2e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 04:34:55 ts/train.py:232 step:558K smpl:9M ep:46K epch:153.64 loss:0.003 grdn:0.161 lr:3.2e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 04:35:43 ts/train.py:232 step:558K smpl:9M ep:46K epch:153.69 loss:0.004 grdn:0.158 lr:3.2e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 04:36:30 ts/train.py:232 step:558K smpl:9M ep:46K epch:153.75 loss:0.003 grdn:0.149 lr:3.2e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 04:37:19 ts/train.py:232 step:558K smpl:9M ep:46K epch:153.80 loss:0.003 grdn:0.145 lr:3.2e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 04:38:08 ts/train.py:232 step:559K smpl:9M ep:46K epch:153.86 loss:0.003 grdn:0.149 lr:3.2e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 04:38:56 ts/train.py:232 step:559K smpl:9M ep:46K epch:153.91 loss:0.003 grdn:0.163 lr:3.2e-05 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 04:39:44 ts/train.py:232 step:559K smpl:9M ep:46K epch:153.97 loss:0.003 grdn:0.150 lr:3.1e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 04:40:32 ts/train.py:232 step:559K smpl:9M ep:46K epch:154.02 loss:0.003 grdn:0.153 lr:3.1e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 04:41:20 ts/train.py:232 step:559K smpl:9M ep:46K epch:154.08 loss:0.003 grdn:0.143 lr:3.1e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 04:42:08 ts/train.py:232 step:560K smpl:9M ep:46K epch:154.13 loss:0.003 grdn:0.157 lr:3.1e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 04:42:57 ts/train.py:232 step:560K smpl:9M ep:46K epch:154.19 loss:0.003 grdn:0.146 lr:3.1e-05 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 04:43:45 ts/train.py:232 step:560K smpl:9M ep:46K epch:154.24 loss:0.003 grdn:0.149 lr:3.1e-05 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 04:44:34 ts/train.py:232 step:560K smpl:9M ep:46K epch:154.30 loss:0.003 grdn:0.141 lr:3.1e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 04:45:23 ts/train.py:232 step:560K smpl:9M ep:46K epch:154.35 loss:0.003 grdn:0.147 lr:3.1e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 04:46:10 ts/train.py:232 step:561K smpl:9M ep:46K epch:154.41 loss:0.003 grdn:0.140 lr:3.1e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 04:46:58 ts/train.py:232 step:561K smpl:9M ep:46K epch:154.46 loss:0.004 grdn:0.161 lr:3.1e-05 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 04:47:46 ts/train.py:232 step:561K smpl:9M ep:46K epch:154.52 loss:0.003 grdn:0.156 lr:3.1e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 04:48:33 ts/train.py:232 step:561K smpl:9M ep:46K epch:154.57 loss:0.003 grdn:0.147 lr:3.1e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 04:49:21 ts/train.py:232 step:561K smpl:9M ep:46K epch:154.63 loss:0.003 grdn:0.139 lr:3.1e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 04:50:09 ts/train.py:232 step:562K smpl:9M ep:46K epch:154.68 loss:0.003 grdn:0.153 lr:3.1e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 04:50:56 ts/train.py:232 step:562K smpl:9M ep:46K epch:154.74 loss:0.003 grdn:0.152 lr:3.1e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 04:51:44 ts/train.py:232 step:562K smpl:9M ep:46K epch:154.79 loss:0.004 grdn:0.165 lr:3.1e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 04:52:31 ts/train.py:232 step:562K smpl:9M ep:46K epch:154.85 loss:0.003 grdn:0.151 lr:3.1e-05 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 04:53:18 ts/train.py:232 step:562K smpl:9M ep:46K epch:154.90 loss:0.003 grdn:0.150 lr:3.1e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 04:54:06 ts/train.py:232 step:563K smpl:9M ep:46K epch:154.96 loss:0.003 grdn:0.143 lr:3.1e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 04:54:53 ts/train.py:232 step:563K smpl:9M ep:47K epch:155.01 loss:0.003 grdn:0.140 lr:3.1e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 04:55:40 ts/train.py:232 step:563K smpl:9M ep:47K epch:155.07 loss:0.003 grdn:0.154 lr:3.1e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 04:56:28 ts/train.py:232 step:563K smpl:9M ep:47K epch:155.12 loss:0.003 grdn:0.153 lr:3.1e-05 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 04:57:16 ts/train.py:232 step:563K smpl:9M ep:47K epch:155.18 loss:0.003 grdn:0.160 lr:3.1e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 04:58:05 ts/train.py:232 step:564K smpl:9M ep:47K epch:155.23 loss:0.003 grdn:0.146 lr:3.1e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 04:58:52 ts/train.py:232 step:564K smpl:9M ep:47K epch:155.29 loss:0.003 grdn:0.152 lr:3.1e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 04:59:37 ts/train.py:232 step:564K smpl:9M ep:47K epch:155.35 loss:0.003 grdn:0.145 lr:3.1e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 05:00:22 ts/train.py:232 step:564K smpl:9M ep:47K epch:155.40 loss:0.003 grdn:0.149 lr:3.1e-05 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 05:01:08 ts/train.py:232 step:564K smpl:9M ep:47K epch:155.46 loss:0.003 grdn:0.145 lr:3.1e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 05:01:53 ts/train.py:232 step:565K smpl:9M ep:47K epch:155.51 loss:0.003 grdn:0.157 lr:3.1e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 05:02:38 ts/train.py:232 step:565K smpl:9M ep:47K epch:155.57 loss:0.003 grdn:0.156 lr:3.1e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 05:03:24 ts/train.py:232 step:565K smpl:9M ep:47K epch:155.62 loss:0.003 grdn:0.128 lr:3.1e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 05:04:09 ts/train.py:232 step:565K smpl:9M ep:47K epch:155.68 loss:0.003 grdn:0.141 lr:3.0e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 05:04:55 ts/train.py:232 step:565K smpl:9M ep:47K epch:155.73 loss:0.003 grdn:0.147 lr:3.0e-05 updt_s:0.156 data_s:0.074 +INFO 2025-05-07 05:05:43 ts/train.py:232 step:566K smpl:9M ep:47K epch:155.79 loss:0.003 grdn:0.145 lr:3.0e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 05:06:30 ts/train.py:232 step:566K smpl:9M ep:47K epch:155.84 loss:0.003 grdn:0.140 lr:3.0e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 05:07:16 ts/train.py:232 step:566K smpl:9M ep:47K epch:155.90 loss:0.003 grdn:0.138 lr:3.0e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 05:08:01 ts/train.py:232 step:566K smpl:9M ep:47K epch:155.95 loss:0.003 grdn:0.150 lr:3.0e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 05:08:46 ts/train.py:232 step:566K smpl:9M ep:47K epch:156.01 loss:0.003 grdn:0.153 lr:3.0e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 05:09:32 ts/train.py:232 step:567K smpl:9M ep:47K epch:156.06 loss:0.003 grdn:0.152 lr:3.0e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 05:10:17 ts/train.py:232 step:567K smpl:9M ep:47K epch:156.12 loss:0.003 grdn:0.151 lr:3.0e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 05:11:03 ts/train.py:232 step:567K smpl:9M ep:47K epch:156.17 loss:0.003 grdn:0.155 lr:3.0e-05 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 05:11:51 ts/train.py:232 step:567K smpl:9M ep:47K epch:156.23 loss:0.003 grdn:0.150 lr:3.0e-05 updt_s:0.158 data_s:0.083 +INFO 2025-05-07 05:12:38 ts/train.py:232 step:567K smpl:9M ep:47K epch:156.28 loss:0.003 grdn:0.153 lr:3.0e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 05:13:25 ts/train.py:232 step:568K smpl:9M ep:47K epch:156.34 loss:0.003 grdn:0.154 lr:3.0e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 05:14:11 ts/train.py:232 step:568K smpl:9M ep:47K epch:156.39 loss:0.003 grdn:0.154 lr:3.0e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 05:14:57 ts/train.py:232 step:568K smpl:9M ep:47K epch:156.45 loss:0.003 grdn:0.155 lr:3.0e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 05:15:44 ts/train.py:232 step:568K smpl:9M ep:47K epch:156.50 loss:0.003 grdn:0.158 lr:3.0e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 05:16:32 ts/train.py:232 step:568K smpl:9M ep:47K epch:156.56 loss:0.003 grdn:0.154 lr:3.0e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 05:17:20 ts/train.py:232 step:569K smpl:9M ep:47K epch:156.61 loss:0.003 grdn:0.144 lr:3.0e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 05:18:07 ts/train.py:232 step:569K smpl:9M ep:47K epch:156.67 loss:0.003 grdn:0.148 lr:3.0e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 05:18:54 ts/train.py:232 step:569K smpl:9M ep:47K epch:156.72 loss:0.003 grdn:0.152 lr:3.0e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 05:19:41 ts/train.py:232 step:569K smpl:9M ep:47K epch:156.78 loss:0.003 grdn:0.155 lr:3.0e-05 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 05:20:29 ts/train.py:232 step:569K smpl:9M ep:47K epch:156.83 loss:0.004 grdn:0.168 lr:3.0e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 05:21:15 ts/train.py:232 step:570K smpl:9M ep:47K epch:156.89 loss:0.003 grdn:0.149 lr:3.0e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 05:22:02 ts/train.py:232 step:570K smpl:9M ep:47K epch:156.94 loss:0.003 grdn:0.151 lr:3.0e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 05:22:49 ts/train.py:232 step:570K smpl:9M ep:47K epch:157.00 loss:0.003 grdn:0.134 lr:3.0e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 05:22:49 ts/train.py:241 Checkpoint policy after step 570000 +INFO 2025-05-07 05:23:38 ts/train.py:232 step:570K smpl:9M ep:47K epch:157.05 loss:0.003 grdn:0.133 lr:3.0e-05 updt_s:0.156 data_s:0.075 +INFO 2025-05-07 05:24:25 ts/train.py:232 step:570K smpl:9M ep:47K epch:157.11 loss:0.003 grdn:0.147 lr:3.0e-05 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 05:25:14 ts/train.py:232 step:571K smpl:9M ep:47K epch:157.16 loss:0.003 grdn:0.152 lr:3.0e-05 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 05:25:58 ts/train.py:232 step:571K smpl:9M ep:47K epch:157.22 loss:0.003 grdn:0.141 lr:3.0e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 05:26:43 ts/train.py:232 step:571K smpl:9M ep:47K epch:157.27 loss:0.003 grdn:0.150 lr:3.0e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 05:27:27 ts/train.py:232 step:571K smpl:9M ep:47K epch:157.33 loss:0.003 grdn:0.154 lr:3.0e-05 updt_s:0.156 data_s:0.065 +INFO 2025-05-07 05:28:13 ts/train.py:232 step:571K smpl:9M ep:47K epch:157.38 loss:0.003 grdn:0.147 lr:2.9e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 05:28:57 ts/train.py:232 step:572K smpl:9M ep:47K epch:157.44 loss:0.003 grdn:0.154 lr:2.9e-05 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 05:29:41 ts/train.py:232 step:572K smpl:9M ep:47K epch:157.49 loss:0.003 grdn:0.136 lr:2.9e-05 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 05:30:25 ts/train.py:232 step:572K smpl:9M ep:47K epch:157.55 loss:0.003 grdn:0.138 lr:2.9e-05 updt_s:0.156 data_s:0.062 +INFO 2025-05-07 05:31:10 ts/train.py:232 step:572K smpl:9M ep:47K epch:157.60 loss:0.003 grdn:0.144 lr:2.9e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 05:31:54 ts/train.py:232 step:572K smpl:9M ep:47K epch:157.66 loss:0.003 grdn:0.144 lr:2.9e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 05:32:38 ts/train.py:232 step:573K smpl:9M ep:47K epch:157.71 loss:0.003 grdn:0.145 lr:2.9e-05 updt_s:0.157 data_s:0.062 +INFO 2025-05-07 05:33:23 ts/train.py:232 step:573K smpl:9M ep:47K epch:157.77 loss:0.003 grdn:0.153 lr:2.9e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 05:34:07 ts/train.py:232 step:573K smpl:9M ep:47K epch:157.82 loss:0.003 grdn:0.142 lr:2.9e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 05:34:52 ts/train.py:232 step:573K smpl:9M ep:47K epch:157.88 loss:0.003 grdn:0.144 lr:2.9e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 05:35:36 ts/train.py:232 step:573K smpl:9M ep:47K epch:157.93 loss:0.003 grdn:0.148 lr:2.9e-05 updt_s:0.156 data_s:0.066 +INFO 2025-05-07 05:36:21 ts/train.py:232 step:574K smpl:9M ep:47K epch:157.99 loss:0.003 grdn:0.145 lr:2.9e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 05:37:07 ts/train.py:232 step:574K smpl:9M ep:47K epch:158.04 loss:0.003 grdn:0.164 lr:2.9e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 05:37:52 ts/train.py:232 step:574K smpl:9M ep:47K epch:158.10 loss:0.004 grdn:0.166 lr:2.9e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 05:38:42 ts/train.py:232 step:574K smpl:9M ep:47K epch:158.15 loss:0.003 grdn:0.144 lr:2.9e-05 updt_s:0.156 data_s:0.092 +INFO 2025-05-07 05:39:29 ts/train.py:232 step:574K smpl:9M ep:47K epch:158.21 loss:0.003 grdn:0.149 lr:2.9e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 05:40:17 ts/train.py:232 step:575K smpl:9M ep:47K epch:158.26 loss:0.003 grdn:0.145 lr:2.9e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 05:41:05 ts/train.py:232 step:575K smpl:9M ep:47K epch:158.32 loss:0.003 grdn:0.150 lr:2.9e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 05:41:54 ts/train.py:232 step:575K smpl:9M ep:48K epch:158.37 loss:0.003 grdn:0.160 lr:2.9e-05 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 05:42:41 ts/train.py:232 step:575K smpl:9M ep:48K epch:158.43 loss:0.003 grdn:0.152 lr:2.9e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 05:43:30 ts/train.py:232 step:575K smpl:9M ep:48K epch:158.49 loss:0.003 grdn:0.152 lr:2.9e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 05:44:17 ts/train.py:232 step:576K smpl:9M ep:48K epch:158.54 loss:0.003 grdn:0.152 lr:2.9e-05 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 05:45:05 ts/train.py:232 step:576K smpl:9M ep:48K epch:158.60 loss:0.003 grdn:0.140 lr:2.9e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 05:45:52 ts/train.py:232 step:576K smpl:9M ep:48K epch:158.65 loss:0.003 grdn:0.157 lr:2.9e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 05:46:38 ts/train.py:232 step:576K smpl:9M ep:48K epch:158.71 loss:0.003 grdn:0.153 lr:2.9e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 05:47:26 ts/train.py:232 step:576K smpl:9M ep:48K epch:158.76 loss:0.003 grdn:0.153 lr:2.9e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 05:48:14 ts/train.py:232 step:577K smpl:9M ep:48K epch:158.82 loss:0.003 grdn:0.146 lr:2.9e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 05:49:01 ts/train.py:232 step:577K smpl:9M ep:48K epch:158.87 loss:0.003 grdn:0.141 lr:2.9e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 05:49:50 ts/train.py:232 step:577K smpl:9M ep:48K epch:158.93 loss:0.003 grdn:0.150 lr:2.9e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 05:50:37 ts/train.py:232 step:577K smpl:9M ep:48K epch:158.98 loss:0.003 grdn:0.147 lr:2.9e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 05:51:24 ts/train.py:232 step:577K smpl:9M ep:48K epch:159.04 loss:0.003 grdn:0.150 lr:2.9e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 05:52:12 ts/train.py:232 step:578K smpl:9M ep:48K epch:159.09 loss:0.003 grdn:0.162 lr:2.9e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 05:52:56 ts/train.py:232 step:578K smpl:9M ep:48K epch:159.15 loss:0.003 grdn:0.146 lr:2.8e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 05:53:41 ts/train.py:232 step:578K smpl:9M ep:48K epch:159.20 loss:0.003 grdn:0.163 lr:2.8e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 05:54:26 ts/train.py:232 step:578K smpl:9M ep:48K epch:159.26 loss:0.003 grdn:0.160 lr:2.8e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 05:55:10 ts/train.py:232 step:578K smpl:9M ep:48K epch:159.31 loss:0.003 grdn:0.150 lr:2.8e-05 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 05:55:54 ts/train.py:232 step:579K smpl:9M ep:48K epch:159.37 loss:0.003 grdn:0.155 lr:2.8e-05 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 05:56:38 ts/train.py:232 step:579K smpl:9M ep:48K epch:159.42 loss:0.003 grdn:0.137 lr:2.8e-05 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 05:57:23 ts/train.py:232 step:579K smpl:9M ep:48K epch:159.48 loss:0.003 grdn:0.140 lr:2.8e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 05:58:08 ts/train.py:232 step:579K smpl:9M ep:48K epch:159.53 loss:0.003 grdn:0.155 lr:2.8e-05 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 05:58:54 ts/train.py:232 step:579K smpl:9M ep:48K epch:159.59 loss:0.003 grdn:0.160 lr:2.8e-05 updt_s:0.156 data_s:0.076 +INFO 2025-05-07 05:59:39 ts/train.py:232 step:580K smpl:9M ep:48K epch:159.64 loss:0.003 grdn:0.142 lr:2.8e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 06:00:23 ts/train.py:232 step:580K smpl:9M ep:48K epch:159.70 loss:0.003 grdn:0.148 lr:2.8e-05 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 06:01:07 ts/train.py:232 step:580K smpl:9M ep:48K epch:159.75 loss:0.003 grdn:0.148 lr:2.8e-05 updt_s:0.156 data_s:0.063 +INFO 2025-05-07 06:01:52 ts/train.py:232 step:580K smpl:9M ep:48K epch:159.81 loss:0.003 grdn:0.157 lr:2.8e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 06:02:37 ts/train.py:232 step:580K smpl:9M ep:48K epch:159.86 loss:0.003 grdn:0.157 lr:2.8e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 06:03:21 ts/train.py:232 step:581K smpl:9M ep:48K epch:159.92 loss:0.003 grdn:0.147 lr:2.8e-05 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 06:04:06 ts/train.py:232 step:581K smpl:9M ep:48K epch:159.97 loss:0.003 grdn:0.154 lr:2.8e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 06:04:50 ts/train.py:232 step:581K smpl:9M ep:48K epch:160.03 loss:0.003 grdn:0.147 lr:2.8e-05 updt_s:0.156 data_s:0.066 +INFO 2025-05-07 06:05:38 ts/train.py:232 step:581K smpl:9M ep:48K epch:160.08 loss:0.003 grdn:0.143 lr:2.8e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 06:06:24 ts/train.py:232 step:581K smpl:9M ep:48K epch:160.14 loss:0.003 grdn:0.144 lr:2.8e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 06:07:09 ts/train.py:232 step:582K smpl:9M ep:48K epch:160.19 loss:0.003 grdn:0.145 lr:2.8e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 06:07:54 ts/train.py:232 step:582K smpl:9M ep:48K epch:160.25 loss:0.003 grdn:0.147 lr:2.8e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 06:08:38 ts/train.py:232 step:582K smpl:9M ep:48K epch:160.30 loss:0.003 grdn:0.167 lr:2.8e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 06:09:23 ts/train.py:232 step:582K smpl:9M ep:48K epch:160.36 loss:0.003 grdn:0.152 lr:2.8e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 06:10:08 ts/train.py:232 step:582K smpl:9M ep:48K epch:160.41 loss:0.003 grdn:0.153 lr:2.8e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 06:10:52 ts/train.py:232 step:583K smpl:9M ep:48K epch:160.47 loss:0.003 grdn:0.154 lr:2.8e-05 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 06:11:36 ts/train.py:232 step:583K smpl:9M ep:48K epch:160.52 loss:0.003 grdn:0.162 lr:2.8e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 06:12:21 ts/train.py:232 step:583K smpl:9M ep:48K epch:160.58 loss:0.003 grdn:0.144 lr:2.8e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 06:13:05 ts/train.py:232 step:583K smpl:9M ep:48K epch:160.63 loss:0.003 grdn:0.156 lr:2.8e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 06:13:50 ts/train.py:232 step:583K smpl:9M ep:48K epch:160.69 loss:0.003 grdn:0.137 lr:2.8e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 06:14:36 ts/train.py:232 step:584K smpl:9M ep:48K epch:160.74 loss:0.003 grdn:0.167 lr:2.8e-05 updt_s:0.156 data_s:0.074 +INFO 2025-05-07 06:15:20 ts/train.py:232 step:584K smpl:9M ep:48K epch:160.80 loss:0.003 grdn:0.152 lr:2.8e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 06:16:05 ts/train.py:232 step:584K smpl:9M ep:48K epch:160.85 loss:0.003 grdn:0.145 lr:2.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 06:16:50 ts/train.py:232 step:584K smpl:9M ep:48K epch:160.91 loss:0.003 grdn:0.148 lr:2.7e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 06:17:36 ts/train.py:232 step:584K smpl:9M ep:48K epch:160.96 loss:0.003 grdn:0.147 lr:2.7e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 06:18:24 ts/train.py:232 step:585K smpl:9M ep:48K epch:161.02 loss:0.003 grdn:0.168 lr:2.7e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 06:19:09 ts/train.py:232 step:585K smpl:9M ep:48K epch:161.07 loss:0.003 grdn:0.159 lr:2.7e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 06:19:54 ts/train.py:232 step:585K smpl:9M ep:48K epch:161.13 loss:0.003 grdn:0.150 lr:2.7e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 06:20:39 ts/train.py:232 step:585K smpl:9M ep:48K epch:161.18 loss:0.003 grdn:0.159 lr:2.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 06:21:25 ts/train.py:232 step:585K smpl:9M ep:48K epch:161.24 loss:0.003 grdn:0.143 lr:2.7e-05 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 06:22:12 ts/train.py:232 step:586K smpl:9M ep:48K epch:161.29 loss:0.003 grdn:0.151 lr:2.7e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 06:22:58 ts/train.py:232 step:586K smpl:9M ep:48K epch:161.35 loss:0.003 grdn:0.145 lr:2.7e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 06:23:43 ts/train.py:232 step:586K smpl:9M ep:48K epch:161.40 loss:0.003 grdn:0.145 lr:2.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 06:24:30 ts/train.py:232 step:586K smpl:9M ep:48K epch:161.46 loss:0.003 grdn:0.153 lr:2.7e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 06:25:18 ts/train.py:232 step:586K smpl:9M ep:48K epch:161.51 loss:0.003 grdn:0.158 lr:2.7e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 06:26:06 ts/train.py:232 step:587K smpl:9M ep:48K epch:161.57 loss:0.003 grdn:0.161 lr:2.7e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 06:26:53 ts/train.py:232 step:587K smpl:9M ep:48K epch:161.63 loss:0.003 grdn:0.142 lr:2.7e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 06:27:46 ts/train.py:232 step:587K smpl:9M ep:49K epch:161.68 loss:0.003 grdn:0.161 lr:2.7e-05 updt_s:0.156 data_s:0.105 +INFO 2025-05-07 06:28:32 ts/train.py:232 step:587K smpl:9M ep:49K epch:161.74 loss:0.003 grdn:0.159 lr:2.7e-05 updt_s:0.156 data_s:0.074 +INFO 2025-05-07 06:29:18 ts/train.py:232 step:587K smpl:9M ep:49K epch:161.79 loss:0.003 grdn:0.137 lr:2.7e-05 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 06:30:04 ts/train.py:232 step:588K smpl:9M ep:49K epch:161.85 loss:0.003 grdn:0.147 lr:2.7e-05 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 06:30:49 ts/train.py:232 step:588K smpl:9M ep:49K epch:161.90 loss:0.003 grdn:0.147 lr:2.7e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 06:31:35 ts/train.py:232 step:588K smpl:9M ep:49K epch:161.96 loss:0.003 grdn:0.156 lr:2.7e-05 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 06:32:21 ts/train.py:232 step:588K smpl:9M ep:49K epch:162.01 loss:0.003 grdn:0.158 lr:2.7e-05 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 06:33:07 ts/train.py:232 step:588K smpl:9M ep:49K epch:162.07 loss:0.003 grdn:0.140 lr:2.7e-05 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 06:33:52 ts/train.py:232 step:589K smpl:9M ep:49K epch:162.12 loss:0.003 grdn:0.155 lr:2.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 06:34:37 ts/train.py:232 step:589K smpl:9M ep:49K epch:162.18 loss:0.003 grdn:0.142 lr:2.7e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 06:35:22 ts/train.py:232 step:589K smpl:9M ep:49K epch:162.23 loss:0.003 grdn:0.140 lr:2.7e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 06:36:08 ts/train.py:232 step:589K smpl:9M ep:49K epch:162.29 loss:0.003 grdn:0.162 lr:2.7e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 06:36:53 ts/train.py:232 step:589K smpl:9M ep:49K epch:162.34 loss:0.003 grdn:0.145 lr:2.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 06:37:38 ts/train.py:232 step:590K smpl:9M ep:49K epch:162.40 loss:0.003 grdn:0.148 lr:2.7e-05 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 06:38:22 ts/train.py:232 step:590K smpl:9M ep:49K epch:162.45 loss:0.003 grdn:0.151 lr:2.7e-05 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 06:39:06 ts/train.py:232 step:590K smpl:9M ep:49K epch:162.51 loss:0.003 grdn:0.151 lr:2.7e-05 updt_s:0.156 data_s:0.065 +INFO 2025-05-07 06:39:51 ts/train.py:232 step:590K smpl:9M ep:49K epch:162.56 loss:0.003 grdn:0.155 lr:2.7e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 06:40:36 ts/train.py:232 step:590K smpl:9M ep:49K epch:162.62 loss:0.003 grdn:0.145 lr:2.7e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 06:41:20 ts/train.py:232 step:591K smpl:9M ep:49K epch:162.67 loss:0.003 grdn:0.135 lr:2.6e-05 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 06:42:05 ts/train.py:232 step:591K smpl:9M ep:49K epch:162.73 loss:0.003 grdn:0.142 lr:2.6e-05 updt_s:0.156 data_s:0.066 +INFO 2025-05-07 06:42:50 ts/train.py:232 step:591K smpl:9M ep:49K epch:162.78 loss:0.003 grdn:0.151 lr:2.6e-05 updt_s:0.156 data_s:0.070 +INFO 2025-05-07 06:43:35 ts/train.py:232 step:591K smpl:9M ep:49K epch:162.84 loss:0.003 grdn:0.154 lr:2.6e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 06:44:22 ts/train.py:232 step:591K smpl:9M ep:49K epch:162.89 loss:0.003 grdn:0.143 lr:2.6e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 06:45:10 ts/train.py:232 step:592K smpl:9M ep:49K epch:162.95 loss:0.003 grdn:0.144 lr:2.6e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 06:45:56 ts/train.py:232 step:592K smpl:9M ep:49K epch:163.00 loss:0.003 grdn:0.154 lr:2.6e-05 updt_s:0.156 data_s:0.075 +INFO 2025-05-07 06:46:44 ts/train.py:232 step:592K smpl:9M ep:49K epch:163.06 loss:0.003 grdn:0.157 lr:2.6e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 06:47:31 ts/train.py:232 step:592K smpl:9M ep:49K epch:163.11 loss:0.003 grdn:0.147 lr:2.6e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 06:48:17 ts/train.py:232 step:592K smpl:9M ep:49K epch:163.17 loss:0.003 grdn:0.157 lr:2.6e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 06:49:03 ts/train.py:232 step:593K smpl:9M ep:49K epch:163.22 loss:0.003 grdn:0.160 lr:2.6e-05 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 06:49:49 ts/train.py:232 step:593K smpl:9M ep:49K epch:163.28 loss:0.003 grdn:0.153 lr:2.6e-05 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 06:50:34 ts/train.py:232 step:593K smpl:9M ep:49K epch:163.33 loss:0.003 grdn:0.145 lr:2.6e-05 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 06:51:20 ts/train.py:232 step:593K smpl:9M ep:49K epch:163.39 loss:0.003 grdn:0.147 lr:2.6e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 06:52:06 ts/train.py:232 step:593K smpl:9M ep:49K epch:163.44 loss:0.003 grdn:0.163 lr:2.6e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 06:52:51 ts/train.py:232 step:594K smpl:9M ep:49K epch:163.50 loss:0.003 grdn:0.155 lr:2.6e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 06:53:37 ts/train.py:232 step:594K smpl:10M ep:49K epch:163.55 loss:0.002 grdn:0.136 lr:2.6e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 06:54:22 ts/train.py:232 step:594K smpl:10M ep:49K epch:163.61 loss:0.003 grdn:0.141 lr:2.6e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 06:55:08 ts/train.py:232 step:594K smpl:10M ep:49K epch:163.66 loss:0.003 grdn:0.153 lr:2.6e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 06:55:54 ts/train.py:232 step:594K smpl:10M ep:49K epch:163.72 loss:0.003 grdn:0.155 lr:2.6e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 06:56:39 ts/train.py:232 step:595K smpl:10M ep:49K epch:163.77 loss:0.003 grdn:0.138 lr:2.6e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 06:57:24 ts/train.py:232 step:595K smpl:10M ep:49K epch:163.83 loss:0.003 grdn:0.149 lr:2.6e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 06:58:10 ts/train.py:232 step:595K smpl:10M ep:49K epch:163.88 loss:0.003 grdn:0.145 lr:2.6e-05 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 06:58:59 ts/train.py:232 step:595K smpl:10M ep:49K epch:163.94 loss:0.003 grdn:0.148 lr:2.6e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 06:59:48 ts/train.py:232 step:595K smpl:10M ep:49K epch:163.99 loss:0.003 grdn:0.146 lr:2.6e-05 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 07:00:37 ts/train.py:232 step:596K smpl:10M ep:49K epch:164.05 loss:0.003 grdn:0.150 lr:2.6e-05 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 07:01:26 ts/train.py:232 step:596K smpl:10M ep:49K epch:164.10 loss:0.003 grdn:0.152 lr:2.6e-05 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 07:02:14 ts/train.py:232 step:596K smpl:10M ep:49K epch:164.16 loss:0.002 grdn:0.135 lr:2.6e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 07:03:02 ts/train.py:232 step:596K smpl:10M ep:49K epch:164.21 loss:0.003 grdn:0.152 lr:2.6e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 07:03:50 ts/train.py:232 step:596K smpl:10M ep:49K epch:164.27 loss:0.003 grdn:0.149 lr:2.6e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 07:04:38 ts/train.py:232 step:597K smpl:10M ep:49K epch:164.32 loss:0.003 grdn:0.162 lr:2.6e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 07:05:27 ts/train.py:232 step:597K smpl:10M ep:49K epch:164.38 loss:0.003 grdn:0.161 lr:2.6e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 07:06:16 ts/train.py:232 step:597K smpl:10M ep:49K epch:164.43 loss:0.003 grdn:0.151 lr:2.5e-05 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 07:07:03 ts/train.py:232 step:597K smpl:10M ep:49K epch:164.49 loss:0.003 grdn:0.152 lr:2.5e-05 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 07:07:51 ts/train.py:232 step:597K smpl:10M ep:49K epch:164.54 loss:0.003 grdn:0.166 lr:2.5e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 07:08:39 ts/train.py:232 step:598K smpl:10M ep:49K epch:164.60 loss:0.003 grdn:0.152 lr:2.5e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 07:09:27 ts/train.py:232 step:598K smpl:10M ep:49K epch:164.65 loss:0.003 grdn:0.140 lr:2.5e-05 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 07:10:14 ts/train.py:232 step:598K smpl:10M ep:49K epch:164.71 loss:0.003 grdn:0.146 lr:2.5e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 07:11:01 ts/train.py:232 step:598K smpl:10M ep:49K epch:164.77 loss:0.003 grdn:0.147 lr:2.5e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 07:11:50 ts/train.py:232 step:598K smpl:10M ep:49K epch:164.82 loss:0.003 grdn:0.152 lr:2.5e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 07:12:39 ts/train.py:232 step:599K smpl:10M ep:49K epch:164.88 loss:0.003 grdn:0.159 lr:2.5e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 07:13:25 ts/train.py:232 step:599K smpl:10M ep:49K epch:164.93 loss:0.003 grdn:0.143 lr:2.5e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 07:14:11 ts/train.py:232 step:599K smpl:10M ep:49K epch:164.99 loss:0.002 grdn:0.131 lr:2.5e-05 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 07:14:55 ts/train.py:232 step:599K smpl:10M ep:50K epch:165.04 loss:0.003 grdn:0.155 lr:2.5e-05 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 07:15:40 ts/train.py:232 step:599K smpl:10M ep:50K epch:165.10 loss:0.003 grdn:0.152 lr:2.5e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 07:16:24 ts/train.py:232 step:600K smpl:10M ep:50K epch:165.15 loss:0.003 grdn:0.149 lr:2.5e-05 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 07:17:08 ts/train.py:232 step:600K smpl:10M ep:50K epch:165.21 loss:0.003 grdn:0.151 lr:2.5e-05 updt_s:0.156 data_s:0.062 +INFO 2025-05-07 07:17:52 ts/train.py:232 step:600K smpl:10M ep:50K epch:165.26 loss:0.003 grdn:0.140 lr:2.5e-05 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 07:17:52 ts/train.py:241 Checkpoint policy after step 600000 +INFO 2025-05-07 07:18:38 ts/train.py:232 step:600K smpl:10M ep:50K epch:165.32 loss:0.003 grdn:0.161 lr:2.5e-05 updt_s:0.156 data_s:0.062 +INFO 2025-05-07 07:19:24 ts/train.py:232 step:600K smpl:10M ep:50K epch:165.37 loss:0.003 grdn:0.152 lr:2.5e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 07:20:09 ts/train.py:232 step:601K smpl:10M ep:50K epch:165.43 loss:0.003 grdn:0.160 lr:2.5e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 07:20:54 ts/train.py:232 step:601K smpl:10M ep:50K epch:165.48 loss:0.003 grdn:0.152 lr:2.5e-05 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 07:21:39 ts/train.py:232 step:601K smpl:10M ep:50K epch:165.54 loss:0.003 grdn:0.144 lr:2.5e-05 updt_s:0.156 data_s:0.070 +INFO 2025-05-07 07:22:24 ts/train.py:232 step:601K smpl:10M ep:50K epch:165.59 loss:0.002 grdn:0.141 lr:2.5e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 07:23:09 ts/train.py:232 step:601K smpl:10M ep:50K epch:165.65 loss:0.003 grdn:0.142 lr:2.5e-05 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 07:23:55 ts/train.py:232 step:602K smpl:10M ep:50K epch:165.70 loss:0.003 grdn:0.149 lr:2.5e-05 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 07:24:41 ts/train.py:232 step:602K smpl:10M ep:50K epch:165.76 loss:0.003 grdn:0.150 lr:2.5e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 07:25:26 ts/train.py:232 step:602K smpl:10M ep:50K epch:165.81 loss:0.003 grdn:0.157 lr:2.5e-05 updt_s:0.156 data_s:0.070 +INFO 2025-05-07 07:26:13 ts/train.py:232 step:602K smpl:10M ep:50K epch:165.87 loss:0.003 grdn:0.162 lr:2.5e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 07:26:57 ts/train.py:232 step:602K smpl:10M ep:50K epch:165.92 loss:0.003 grdn:0.148 lr:2.5e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 07:27:42 ts/train.py:232 step:603K smpl:10M ep:50K epch:165.98 loss:0.002 grdn:0.139 lr:2.5e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 07:28:28 ts/train.py:232 step:603K smpl:10M ep:50K epch:166.03 loss:0.003 grdn:0.150 lr:2.5e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 07:29:13 ts/train.py:232 step:603K smpl:10M ep:50K epch:166.09 loss:0.003 grdn:0.149 lr:2.5e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 07:29:58 ts/train.py:232 step:603K smpl:10M ep:50K epch:166.14 loss:0.003 grdn:0.146 lr:2.5e-05 updt_s:0.156 data_s:0.065 +INFO 2025-05-07 07:30:42 ts/train.py:232 step:603K smpl:10M ep:50K epch:166.20 loss:0.003 grdn:0.151 lr:2.5e-05 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 07:31:28 ts/train.py:232 step:604K smpl:10M ep:50K epch:166.25 loss:0.003 grdn:0.152 lr:2.4e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 07:32:12 ts/train.py:232 step:604K smpl:10M ep:50K epch:166.31 loss:0.003 grdn:0.142 lr:2.4e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 07:32:58 ts/train.py:232 step:604K smpl:10M ep:50K epch:166.36 loss:0.003 grdn:0.158 lr:2.4e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 07:33:42 ts/train.py:232 step:604K smpl:10M ep:50K epch:166.42 loss:0.003 grdn:0.142 lr:2.4e-05 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 07:34:28 ts/train.py:232 step:604K smpl:10M ep:50K epch:166.47 loss:0.003 grdn:0.156 lr:2.4e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 07:35:14 ts/train.py:232 step:605K smpl:10M ep:50K epch:166.53 loss:0.003 grdn:0.148 lr:2.4e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 07:36:00 ts/train.py:232 step:605K smpl:10M ep:50K epch:166.58 loss:0.002 grdn:0.141 lr:2.4e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 07:36:46 ts/train.py:232 step:605K smpl:10M ep:50K epch:166.64 loss:0.003 grdn:0.156 lr:2.4e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 07:37:32 ts/train.py:232 step:605K smpl:10M ep:50K epch:166.69 loss:0.003 grdn:0.141 lr:2.4e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 07:38:19 ts/train.py:232 step:605K smpl:10M ep:50K epch:166.75 loss:0.003 grdn:0.145 lr:2.4e-05 updt_s:0.156 data_s:0.076 +INFO 2025-05-07 07:39:09 ts/train.py:232 step:606K smpl:10M ep:50K epch:166.80 loss:0.003 grdn:0.154 lr:2.4e-05 updt_s:0.156 data_s:0.095 +INFO 2025-05-07 07:39:59 ts/train.py:232 step:606K smpl:10M ep:50K epch:166.86 loss:0.003 grdn:0.143 lr:2.4e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 07:40:48 ts/train.py:232 step:606K smpl:10M ep:50K epch:166.91 loss:0.002 grdn:0.139 lr:2.4e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 07:41:37 ts/train.py:232 step:606K smpl:10M ep:50K epch:166.97 loss:0.003 grdn:0.146 lr:2.4e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 07:42:26 ts/train.py:232 step:606K smpl:10M ep:50K epch:167.02 loss:0.003 grdn:0.167 lr:2.4e-05 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 07:43:15 ts/train.py:232 step:607K smpl:10M ep:50K epch:167.08 loss:0.003 grdn:0.151 lr:2.4e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 07:44:04 ts/train.py:232 step:607K smpl:10M ep:50K epch:167.13 loss:0.003 grdn:0.157 lr:2.4e-05 updt_s:0.156 data_s:0.091 +INFO 2025-05-07 07:44:54 ts/train.py:232 step:607K smpl:10M ep:50K epch:167.19 loss:0.002 grdn:0.134 lr:2.4e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 07:45:44 ts/train.py:232 step:607K smpl:10M ep:50K epch:167.24 loss:0.003 grdn:0.160 lr:2.4e-05 updt_s:0.156 data_s:0.095 +INFO 2025-05-07 07:46:34 ts/train.py:232 step:607K smpl:10M ep:50K epch:167.30 loss:0.003 grdn:0.157 lr:2.4e-05 updt_s:0.156 data_s:0.093 +INFO 2025-05-07 07:47:24 ts/train.py:232 step:608K smpl:10M ep:50K epch:167.35 loss:0.003 grdn:0.143 lr:2.4e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 07:48:13 ts/train.py:232 step:608K smpl:10M ep:50K epch:167.41 loss:0.002 grdn:0.139 lr:2.4e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 07:49:02 ts/train.py:232 step:608K smpl:10M ep:50K epch:167.46 loss:0.003 grdn:0.156 lr:2.4e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 07:49:52 ts/train.py:232 step:608K smpl:10M ep:50K epch:167.52 loss:0.003 grdn:0.150 lr:2.4e-05 updt_s:0.156 data_s:0.092 +INFO 2025-05-07 07:50:41 ts/train.py:232 step:608K smpl:10M ep:50K epch:167.57 loss:0.002 grdn:0.137 lr:2.4e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 07:51:31 ts/train.py:232 step:609K smpl:10M ep:50K epch:167.63 loss:0.003 grdn:0.150 lr:2.4e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 07:52:20 ts/train.py:232 step:609K smpl:10M ep:50K epch:167.68 loss:0.003 grdn:0.162 lr:2.4e-05 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 07:53:10 ts/train.py:232 step:609K smpl:10M ep:50K epch:167.74 loss:0.003 grdn:0.147 lr:2.4e-05 updt_s:0.156 data_s:0.092 +INFO 2025-05-07 07:53:58 ts/train.py:232 step:609K smpl:10M ep:50K epch:167.79 loss:0.002 grdn:0.134 lr:2.4e-05 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 07:54:45 ts/train.py:232 step:609K smpl:10M ep:50K epch:167.85 loss:0.003 grdn:0.158 lr:2.4e-05 updt_s:0.156 data_s:0.075 +INFO 2025-05-07 07:55:31 ts/train.py:232 step:610K smpl:10M ep:50K epch:167.90 loss:0.003 grdn:0.148 lr:2.4e-05 updt_s:0.156 data_s:0.075 +INFO 2025-05-07 07:56:18 ts/train.py:232 step:610K smpl:10M ep:50K epch:167.96 loss:0.002 grdn:0.135 lr:2.4e-05 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 07:57:04 ts/train.py:232 step:610K smpl:10M ep:50K epch:168.02 loss:0.003 grdn:0.168 lr:2.4e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 07:57:51 ts/train.py:232 step:610K smpl:10M ep:50K epch:168.07 loss:0.002 grdn:0.141 lr:2.4e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 07:58:37 ts/train.py:232 step:610K smpl:10M ep:50K epch:168.13 loss:0.002 grdn:0.145 lr:2.3e-05 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 07:59:23 ts/train.py:232 step:611K smpl:10M ep:50K epch:168.18 loss:0.002 grdn:0.141 lr:2.3e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 08:00:08 ts/train.py:232 step:611K smpl:10M ep:50K epch:168.24 loss:0.003 grdn:0.146 lr:2.3e-05 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 08:00:54 ts/train.py:232 step:611K smpl:10M ep:50K epch:168.29 loss:0.003 grdn:0.153 lr:2.3e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 08:01:44 ts/train.py:232 step:611K smpl:10M ep:51K epch:168.35 loss:0.003 grdn:0.147 lr:2.3e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 08:02:32 ts/train.py:232 step:611K smpl:10M ep:51K epch:168.40 loss:0.003 grdn:0.151 lr:2.3e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 08:03:19 ts/train.py:232 step:612K smpl:10M ep:51K epch:168.46 loss:0.003 grdn:0.156 lr:2.3e-05 updt_s:0.156 data_s:0.075 +INFO 2025-05-07 08:04:06 ts/train.py:232 step:612K smpl:10M ep:51K epch:168.51 loss:0.003 grdn:0.152 lr:2.3e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 08:04:53 ts/train.py:232 step:612K smpl:10M ep:51K epch:168.57 loss:0.003 grdn:0.161 lr:2.3e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 08:05:39 ts/train.py:232 step:612K smpl:10M ep:51K epch:168.62 loss:0.003 grdn:0.161 lr:2.3e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 08:06:24 ts/train.py:232 step:612K smpl:10M ep:51K epch:168.68 loss:0.002 grdn:0.137 lr:2.3e-05 updt_s:0.156 data_s:0.070 +INFO 2025-05-07 08:07:13 ts/train.py:232 step:613K smpl:10M ep:51K epch:168.73 loss:0.003 grdn:0.150 lr:2.3e-05 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 08:08:02 ts/train.py:232 step:613K smpl:10M ep:51K epch:168.79 loss:0.003 grdn:0.152 lr:2.3e-05 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 08:08:51 ts/train.py:232 step:613K smpl:10M ep:51K epch:168.84 loss:0.003 grdn:0.145 lr:2.3e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 08:09:39 ts/train.py:232 step:613K smpl:10M ep:51K epch:168.90 loss:0.002 grdn:0.147 lr:2.3e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 08:10:29 ts/train.py:232 step:613K smpl:10M ep:51K epch:168.95 loss:0.003 grdn:0.162 lr:2.3e-05 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 08:11:19 ts/train.py:232 step:614K smpl:10M ep:51K epch:169.01 loss:0.003 grdn:0.151 lr:2.3e-05 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 08:12:10 ts/train.py:232 step:614K smpl:10M ep:51K epch:169.06 loss:0.002 grdn:0.147 lr:2.3e-05 updt_s:0.156 data_s:0.097 +INFO 2025-05-07 08:13:02 ts/train.py:232 step:614K smpl:10M ep:51K epch:169.12 loss:0.003 grdn:0.151 lr:2.3e-05 updt_s:0.156 data_s:0.099 +INFO 2025-05-07 08:13:51 ts/train.py:232 step:614K smpl:10M ep:51K epch:169.17 loss:0.003 grdn:0.147 lr:2.3e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 08:14:42 ts/train.py:232 step:614K smpl:10M ep:51K epch:169.23 loss:0.002 grdn:0.145 lr:2.3e-05 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 08:15:32 ts/train.py:232 step:615K smpl:10M ep:51K epch:169.28 loss:0.003 grdn:0.162 lr:2.3e-05 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 08:16:24 ts/train.py:232 step:615K smpl:10M ep:51K epch:169.34 loss:0.003 grdn:0.168 lr:2.3e-05 updt_s:0.158 data_s:0.100 +INFO 2025-05-07 08:17:17 ts/train.py:232 step:615K smpl:10M ep:51K epch:169.39 loss:0.003 grdn:0.157 lr:2.3e-05 updt_s:0.156 data_s:0.106 +INFO 2025-05-07 08:18:10 ts/train.py:232 step:615K smpl:10M ep:51K epch:169.45 loss:0.003 grdn:0.155 lr:2.3e-05 updt_s:0.157 data_s:0.108 +INFO 2025-05-07 08:19:03 ts/train.py:232 step:615K smpl:10M ep:51K epch:169.50 loss:0.002 grdn:0.146 lr:2.3e-05 updt_s:0.157 data_s:0.108 +INFO 2025-05-07 08:19:56 ts/train.py:232 step:616K smpl:10M ep:51K epch:169.56 loss:0.002 grdn:0.141 lr:2.3e-05 updt_s:0.157 data_s:0.105 +INFO 2025-05-07 08:20:47 ts/train.py:232 step:616K smpl:10M ep:51K epch:169.61 loss:0.002 grdn:0.134 lr:2.3e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-07 08:21:40 ts/train.py:232 step:616K smpl:10M ep:51K epch:169.67 loss:0.003 grdn:0.150 lr:2.3e-05 updt_s:0.156 data_s:0.107 +INFO 2025-05-07 08:22:33 ts/train.py:232 step:616K smpl:10M ep:51K epch:169.72 loss:0.003 grdn:0.161 lr:2.3e-05 updt_s:0.157 data_s:0.107 +INFO 2025-05-07 08:23:22 ts/train.py:232 step:616K smpl:10M ep:51K epch:169.78 loss:0.003 grdn:0.171 lr:2.3e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 08:24:11 ts/train.py:232 step:617K smpl:10M ep:51K epch:169.83 loss:0.003 grdn:0.160 lr:2.3e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 08:25:02 ts/train.py:232 step:617K smpl:10M ep:51K epch:169.89 loss:0.003 grdn:0.143 lr:2.3e-05 updt_s:0.157 data_s:0.096 +INFO 2025-05-07 08:25:51 ts/train.py:232 step:617K smpl:10M ep:51K epch:169.94 loss:0.003 grdn:0.144 lr:2.3e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 08:26:41 ts/train.py:232 step:617K smpl:10M ep:51K epch:170.00 loss:0.002 grdn:0.141 lr:2.2e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 08:27:30 ts/train.py:232 step:617K smpl:10M ep:51K epch:170.05 loss:0.003 grdn:0.149 lr:2.2e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 08:28:20 ts/train.py:232 step:618K smpl:10M ep:51K epch:170.11 loss:0.003 grdn:0.149 lr:2.2e-05 updt_s:0.158 data_s:0.089 +INFO 2025-05-07 08:29:09 ts/train.py:232 step:618K smpl:10M ep:51K epch:170.16 loss:0.002 grdn:0.134 lr:2.2e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 08:29:58 ts/train.py:232 step:618K smpl:10M ep:51K epch:170.22 loss:0.003 grdn:0.152 lr:2.2e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 08:30:47 ts/train.py:232 step:618K smpl:10M ep:51K epch:170.27 loss:0.003 grdn:0.153 lr:2.2e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 08:31:37 ts/train.py:232 step:618K smpl:10M ep:51K epch:170.33 loss:0.003 grdn:0.168 lr:2.2e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 08:32:28 ts/train.py:232 step:619K smpl:10M ep:51K epch:170.38 loss:0.003 grdn:0.150 lr:2.2e-05 updt_s:0.157 data_s:0.101 +INFO 2025-05-07 08:33:17 ts/train.py:232 step:619K smpl:10M ep:51K epch:170.44 loss:0.002 grdn:0.136 lr:2.2e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 08:34:06 ts/train.py:232 step:619K smpl:10M ep:51K epch:170.49 loss:0.003 grdn:0.148 lr:2.2e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 08:34:54 ts/train.py:232 step:619K smpl:10M ep:51K epch:170.55 loss:0.002 grdn:0.147 lr:2.2e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 08:35:44 ts/train.py:232 step:619K smpl:10M ep:51K epch:170.60 loss:0.003 grdn:0.162 lr:2.2e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 08:36:35 ts/train.py:232 step:620K smpl:10M ep:51K epch:170.66 loss:0.002 grdn:0.147 lr:2.2e-05 updt_s:0.156 data_s:0.101 +INFO 2025-05-07 08:37:25 ts/train.py:232 step:620K smpl:10M ep:51K epch:170.71 loss:0.003 grdn:0.157 lr:2.2e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 08:38:16 ts/train.py:232 step:620K smpl:10M ep:51K epch:170.77 loss:0.003 grdn:0.154 lr:2.2e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-07 08:39:07 ts/train.py:232 step:620K smpl:10M ep:51K epch:170.82 loss:0.003 grdn:0.148 lr:2.2e-05 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 08:39:56 ts/train.py:232 step:620K smpl:10M ep:51K epch:170.88 loss:0.003 grdn:0.157 lr:2.2e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 08:40:45 ts/train.py:232 step:621K smpl:10M ep:51K epch:170.93 loss:0.003 grdn:0.152 lr:2.2e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 08:41:33 ts/train.py:232 step:621K smpl:10M ep:51K epch:170.99 loss:0.002 grdn:0.145 lr:2.2e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 08:42:21 ts/train.py:232 step:621K smpl:10M ep:51K epch:171.04 loss:0.003 grdn:0.155 lr:2.2e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 08:43:09 ts/train.py:232 step:621K smpl:10M ep:51K epch:171.10 loss:0.003 grdn:0.153 lr:2.2e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 08:43:57 ts/train.py:232 step:621K smpl:10M ep:51K epch:171.16 loss:0.002 grdn:0.141 lr:2.2e-05 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 08:44:46 ts/train.py:232 step:622K smpl:10M ep:51K epch:171.21 loss:0.002 grdn:0.138 lr:2.2e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 08:45:34 ts/train.py:232 step:622K smpl:10M ep:51K epch:171.27 loss:0.002 grdn:0.130 lr:2.2e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 08:46:21 ts/train.py:232 step:622K smpl:10M ep:51K epch:171.32 loss:0.003 grdn:0.161 lr:2.2e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 08:47:09 ts/train.py:232 step:622K smpl:10M ep:51K epch:171.38 loss:0.002 grdn:0.140 lr:2.2e-05 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 08:47:57 ts/train.py:232 step:622K smpl:10M ep:51K epch:171.43 loss:0.002 grdn:0.142 lr:2.2e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 08:48:47 ts/train.py:232 step:623K smpl:10M ep:51K epch:171.49 loss:0.003 grdn:0.159 lr:2.2e-05 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 08:49:35 ts/train.py:232 step:623K smpl:10M ep:51K epch:171.54 loss:0.003 grdn:0.154 lr:2.2e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 08:50:23 ts/train.py:232 step:623K smpl:10M ep:51K epch:171.60 loss:0.002 grdn:0.147 lr:2.2e-05 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 08:51:12 ts/train.py:232 step:623K smpl:10M ep:51K epch:171.65 loss:0.003 grdn:0.151 lr:2.2e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 08:51:58 ts/train.py:232 step:623K smpl:10M ep:52K epch:171.71 loss:0.002 grdn:0.152 lr:2.2e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 08:52:44 ts/train.py:232 step:624K smpl:10M ep:52K epch:171.76 loss:0.002 grdn:0.143 lr:2.2e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 08:53:29 ts/train.py:232 step:624K smpl:10M ep:52K epch:171.82 loss:0.003 grdn:0.154 lr:2.2e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 08:54:16 ts/train.py:232 step:624K smpl:10M ep:52K epch:171.87 loss:0.003 grdn:0.160 lr:2.2e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 08:55:02 ts/train.py:232 step:624K smpl:10M ep:52K epch:171.93 loss:0.002 grdn:0.138 lr:2.1e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 08:55:49 ts/train.py:232 step:624K smpl:10M ep:52K epch:171.98 loss:0.002 grdn:0.136 lr:2.1e-05 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 08:56:35 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.04 loss:0.002 grdn:0.149 lr:2.1e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 08:57:21 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.09 loss:0.002 grdn:0.138 lr:2.1e-05 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 08:58:08 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.15 loss:0.003 grdn:0.160 lr:2.1e-05 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 08:58:54 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.20 loss:0.002 grdn:0.137 lr:2.1e-05 updt_s:0.156 data_s:0.074 +INFO 2025-05-07 08:59:40 ts/train.py:232 step:625K smpl:10M ep:52K epch:172.26 loss:0.002 grdn:0.135 lr:2.1e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 09:00:26 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.31 loss:0.002 grdn:0.145 lr:2.1e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 09:01:13 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.37 loss:0.002 grdn:0.149 lr:2.1e-05 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 09:01:59 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.42 loss:0.002 grdn:0.142 lr:2.1e-05 updt_s:0.156 data_s:0.076 +INFO 2025-05-07 09:02:46 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.48 loss:0.003 grdn:0.157 lr:2.1e-05 updt_s:0.156 data_s:0.076 +INFO 2025-05-07 09:03:33 ts/train.py:232 step:626K smpl:10M ep:52K epch:172.53 loss:0.002 grdn:0.135 lr:2.1e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 09:04:24 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.59 loss:0.003 grdn:0.151 lr:2.1e-05 updt_s:0.156 data_s:0.095 +INFO 2025-05-07 09:05:14 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.64 loss:0.003 grdn:0.152 lr:2.1e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 09:06:03 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.70 loss:0.002 grdn:0.132 lr:2.1e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 09:06:53 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.75 loss:0.003 grdn:0.160 lr:2.1e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 09:07:43 ts/train.py:232 step:627K smpl:10M ep:52K epch:172.81 loss:0.002 grdn:0.127 lr:2.1e-05 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 09:08:32 ts/train.py:232 step:628K smpl:10M ep:52K epch:172.86 loss:0.002 grdn:0.139 lr:2.1e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 09:09:21 ts/train.py:232 step:628K smpl:10M ep:52K epch:172.92 loss:0.003 grdn:0.167 lr:2.1e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 09:10:12 ts/train.py:232 step:628K smpl:10M ep:52K epch:172.97 loss:0.002 grdn:0.137 lr:2.1e-05 updt_s:0.157 data_s:0.094 +INFO 2025-05-07 09:11:01 ts/train.py:232 step:628K smpl:10M ep:52K epch:173.03 loss:0.002 grdn:0.143 lr:2.1e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 09:11:50 ts/train.py:232 step:628K smpl:10M ep:52K epch:173.08 loss:0.003 grdn:0.154 lr:2.1e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 09:12:39 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.14 loss:0.003 grdn:0.162 lr:2.1e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 09:13:28 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.19 loss:0.002 grdn:0.142 lr:2.1e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 09:14:17 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.25 loss:0.002 grdn:0.140 lr:2.1e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 09:15:06 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.30 loss:0.002 grdn:0.134 lr:2.1e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 09:15:55 ts/train.py:232 step:629K smpl:10M ep:52K epch:173.36 loss:0.003 grdn:0.155 lr:2.1e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 09:16:43 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.41 loss:0.003 grdn:0.151 lr:2.1e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 09:17:31 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.47 loss:0.003 grdn:0.156 lr:2.1e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 09:18:19 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.52 loss:0.002 grdn:0.140 lr:2.1e-05 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 09:18:19 ts/train.py:241 Checkpoint policy after step 630000 +INFO 2025-05-07 09:19:12 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.58 loss:0.002 grdn:0.143 lr:2.1e-05 updt_s:0.156 data_s:0.096 +INFO 2025-05-07 09:20:00 ts/train.py:232 step:630K smpl:10M ep:52K epch:173.63 loss:0.002 grdn:0.143 lr:2.1e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 09:20:47 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.69 loss:0.002 grdn:0.148 lr:2.1e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 09:21:35 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.74 loss:0.003 grdn:0.145 lr:2.1e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 09:22:23 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.80 loss:0.003 grdn:0.153 lr:2.1e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 09:23:12 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.85 loss:0.003 grdn:0.151 lr:2.0e-05 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 09:23:59 ts/train.py:232 step:631K smpl:10M ep:52K epch:173.91 loss:0.002 grdn:0.141 lr:2.0e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 09:24:48 ts/train.py:232 step:632K smpl:10M ep:52K epch:173.96 loss:0.003 grdn:0.154 lr:2.0e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 09:25:35 ts/train.py:232 step:632K smpl:10M ep:52K epch:174.02 loss:0.002 grdn:0.149 lr:2.0e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 09:26:23 ts/train.py:232 step:632K smpl:10M ep:52K epch:174.07 loss:0.002 grdn:0.135 lr:2.0e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 09:27:11 ts/train.py:232 step:632K smpl:10M ep:52K epch:174.13 loss:0.002 grdn:0.151 lr:2.0e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 09:27:59 ts/train.py:232 step:632K smpl:10M ep:52K epch:174.18 loss:0.002 grdn:0.138 lr:2.0e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 09:28:46 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.24 loss:0.003 grdn:0.148 lr:2.0e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 09:29:34 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.30 loss:0.002 grdn:0.137 lr:2.0e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 09:30:21 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.35 loss:0.003 grdn:0.151 lr:2.0e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 09:31:09 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.41 loss:0.003 grdn:0.159 lr:2.0e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 09:31:58 ts/train.py:232 step:633K smpl:10M ep:52K epch:174.46 loss:0.002 grdn:0.150 lr:2.0e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 09:32:47 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.52 loss:0.003 grdn:0.151 lr:2.0e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 09:33:37 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.57 loss:0.002 grdn:0.146 lr:2.0e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 09:34:26 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.63 loss:0.003 grdn:0.154 lr:2.0e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 09:35:15 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.68 loss:0.003 grdn:0.155 lr:2.0e-05 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 09:36:05 ts/train.py:232 step:634K smpl:10M ep:52K epch:174.74 loss:0.003 grdn:0.150 lr:2.0e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 09:36:54 ts/train.py:232 step:635K smpl:10M ep:52K epch:174.79 loss:0.002 grdn:0.140 lr:2.0e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 09:37:42 ts/train.py:232 step:635K smpl:10M ep:52K epch:174.85 loss:0.002 grdn:0.149 lr:2.0e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 09:38:31 ts/train.py:232 step:635K smpl:10M ep:52K epch:174.90 loss:0.002 grdn:0.140 lr:2.0e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 09:39:20 ts/train.py:232 step:635K smpl:10M ep:52K epch:174.96 loss:0.002 grdn:0.144 lr:2.0e-05 updt_s:0.158 data_s:0.088 +INFO 2025-05-07 09:40:09 ts/train.py:232 step:635K smpl:10M ep:53K epch:175.01 loss:0.002 grdn:0.124 lr:2.0e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 09:40:59 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.07 loss:0.002 grdn:0.153 lr:2.0e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 09:41:48 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.12 loss:0.002 grdn:0.145 lr:2.0e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 09:42:37 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.18 loss:0.002 grdn:0.144 lr:2.0e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 09:43:26 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.23 loss:0.002 grdn:0.154 lr:2.0e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 09:44:15 ts/train.py:232 step:636K smpl:10M ep:53K epch:175.29 loss:0.003 grdn:0.157 lr:2.0e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 09:45:03 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.34 loss:0.002 grdn:0.146 lr:2.0e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 09:45:52 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.40 loss:0.003 grdn:0.154 lr:2.0e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 09:46:41 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.45 loss:0.002 grdn:0.151 lr:2.0e-05 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 09:47:32 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.51 loss:0.002 grdn:0.124 lr:2.0e-05 updt_s:0.156 data_s:0.099 +INFO 2025-05-07 09:48:21 ts/train.py:232 step:637K smpl:10M ep:53K epch:175.56 loss:0.003 grdn:0.158 lr:2.0e-05 updt_s:0.158 data_s:0.089 +INFO 2025-05-07 09:49:11 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.62 loss:0.003 grdn:0.155 lr:2.0e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 09:50:00 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.67 loss:0.003 grdn:0.158 lr:2.0e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 09:50:48 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.73 loss:0.003 grdn:0.154 lr:2.0e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 09:51:39 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.78 loss:0.003 grdn:0.174 lr:1.9e-05 updt_s:0.158 data_s:0.096 +INFO 2025-05-07 09:52:29 ts/train.py:232 step:638K smpl:10M ep:53K epch:175.84 loss:0.002 grdn:0.145 lr:1.9e-05 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 09:53:23 ts/train.py:232 step:639K smpl:10M ep:53K epch:175.89 loss:0.002 grdn:0.137 lr:1.9e-05 updt_s:0.158 data_s:0.109 +INFO 2025-05-07 09:54:12 ts/train.py:232 step:639K smpl:10M ep:53K epch:175.95 loss:0.003 grdn:0.153 lr:1.9e-05 updt_s:0.156 data_s:0.091 +INFO 2025-05-07 09:55:03 ts/train.py:232 step:639K smpl:10M ep:53K epch:176.00 loss:0.002 grdn:0.145 lr:1.9e-05 updt_s:0.156 data_s:0.095 +INFO 2025-05-07 09:55:53 ts/train.py:232 step:639K smpl:10M ep:53K epch:176.06 loss:0.002 grdn:0.147 lr:1.9e-05 updt_s:0.156 data_s:0.096 +INFO 2025-05-07 09:56:42 ts/train.py:232 step:639K smpl:10M ep:53K epch:176.11 loss:0.002 grdn:0.146 lr:1.9e-05 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 09:57:29 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.17 loss:0.002 grdn:0.136 lr:1.9e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 09:58:16 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.22 loss:0.002 grdn:0.141 lr:1.9e-05 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 09:59:03 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.28 loss:0.003 grdn:0.161 lr:1.9e-05 updt_s:0.156 data_s:0.075 +INFO 2025-05-07 09:59:50 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.33 loss:0.002 grdn:0.147 lr:1.9e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 10:00:37 ts/train.py:232 step:640K smpl:10M ep:53K epch:176.39 loss:0.002 grdn:0.145 lr:1.9e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 10:01:29 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.44 loss:0.002 grdn:0.144 lr:1.9e-05 updt_s:0.156 data_s:0.104 +INFO 2025-05-07 10:02:18 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.50 loss:0.002 grdn:0.147 lr:1.9e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 10:03:06 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.55 loss:0.002 grdn:0.141 lr:1.9e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 10:03:54 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.61 loss:0.002 grdn:0.153 lr:1.9e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 10:04:41 ts/train.py:232 step:641K smpl:10M ep:53K epch:176.66 loss:0.003 grdn:0.160 lr:1.9e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 10:05:31 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.72 loss:0.002 grdn:0.140 lr:1.9e-05 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 10:06:20 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.77 loss:0.002 grdn:0.126 lr:1.9e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 10:07:07 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.83 loss:0.002 grdn:0.133 lr:1.9e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 10:07:54 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.88 loss:0.002 grdn:0.153 lr:1.9e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 10:08:42 ts/train.py:232 step:642K smpl:10M ep:53K epch:176.94 loss:0.002 grdn:0.137 lr:1.9e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 10:09:30 ts/train.py:232 step:643K smpl:10M ep:53K epch:176.99 loss:0.002 grdn:0.152 lr:1.9e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 10:10:17 ts/train.py:232 step:643K smpl:10M ep:53K epch:177.05 loss:0.002 grdn:0.150 lr:1.9e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 10:11:05 ts/train.py:232 step:643K smpl:10M ep:53K epch:177.10 loss:0.002 grdn:0.143 lr:1.9e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 10:11:52 ts/train.py:232 step:643K smpl:10M ep:53K epch:177.16 loss:0.002 grdn:0.139 lr:1.9e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 10:12:39 ts/train.py:232 step:643K smpl:10M ep:53K epch:177.21 loss:0.002 grdn:0.136 lr:1.9e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 10:13:27 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.27 loss:0.003 grdn:0.156 lr:1.9e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 10:14:15 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.32 loss:0.002 grdn:0.134 lr:1.9e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 10:15:02 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.38 loss:0.002 grdn:0.149 lr:1.9e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 10:15:50 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.44 loss:0.003 grdn:0.158 lr:1.9e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 10:16:35 ts/train.py:232 step:644K smpl:10M ep:53K epch:177.49 loss:0.003 grdn:0.158 lr:1.9e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 10:17:21 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.55 loss:0.003 grdn:0.152 lr:1.9e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 10:18:08 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.60 loss:0.002 grdn:0.139 lr:1.9e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 10:18:52 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.66 loss:0.002 grdn:0.150 lr:1.9e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 10:19:39 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.71 loss:0.002 grdn:0.140 lr:1.9e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 10:20:23 ts/train.py:232 step:645K smpl:10M ep:53K epch:177.77 loss:0.003 grdn:0.163 lr:1.9e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 10:21:10 ts/train.py:232 step:646K smpl:10M ep:53K epch:177.82 loss:0.002 grdn:0.133 lr:1.8e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 10:21:55 ts/train.py:232 step:646K smpl:10M ep:53K epch:177.88 loss:0.002 grdn:0.146 lr:1.8e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 10:22:40 ts/train.py:232 step:646K smpl:10M ep:53K epch:177.93 loss:0.002 grdn:0.139 lr:1.8e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 10:23:26 ts/train.py:232 step:646K smpl:10M ep:53K epch:177.99 loss:0.003 grdn:0.169 lr:1.8e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 10:24:11 ts/train.py:232 step:646K smpl:10M ep:53K epch:178.04 loss:0.002 grdn:0.152 lr:1.8e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 10:24:56 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.10 loss:0.002 grdn:0.148 lr:1.8e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 10:25:42 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.15 loss:0.002 grdn:0.142 lr:1.8e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 10:26:27 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.21 loss:0.002 grdn:0.146 lr:1.8e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 10:27:13 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.26 loss:0.002 grdn:0.148 lr:1.8e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 10:27:58 ts/train.py:232 step:647K smpl:10M ep:53K epch:178.32 loss:0.002 grdn:0.144 lr:1.8e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 10:28:46 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.37 loss:0.002 grdn:0.144 lr:1.8e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 10:29:35 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.43 loss:0.002 grdn:0.153 lr:1.8e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 10:30:23 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.48 loss:0.002 grdn:0.153 lr:1.8e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 10:31:11 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.54 loss:0.002 grdn:0.147 lr:1.8e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 10:32:01 ts/train.py:232 step:648K smpl:10M ep:54K epch:178.59 loss:0.002 grdn:0.147 lr:1.8e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 10:32:50 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.65 loss:0.002 grdn:0.146 lr:1.8e-05 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 10:33:39 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.70 loss:0.002 grdn:0.142 lr:1.8e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 10:34:28 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.76 loss:0.002 grdn:0.148 lr:1.8e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 10:35:17 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.81 loss:0.002 grdn:0.152 lr:1.8e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 10:36:07 ts/train.py:232 step:649K smpl:10M ep:54K epch:178.87 loss:0.002 grdn:0.142 lr:1.8e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 10:36:57 ts/train.py:232 step:650K smpl:10M ep:54K epch:178.92 loss:0.002 grdn:0.146 lr:1.8e-05 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 10:37:46 ts/train.py:232 step:650K smpl:10M ep:54K epch:178.98 loss:0.002 grdn:0.152 lr:1.8e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-07 10:38:35 ts/train.py:232 step:650K smpl:10M ep:54K epch:179.03 loss:0.002 grdn:0.137 lr:1.8e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 10:39:25 ts/train.py:232 step:650K smpl:10M ep:54K epch:179.09 loss:0.002 grdn:0.142 lr:1.8e-05 updt_s:0.156 data_s:0.092 +INFO 2025-05-07 10:40:14 ts/train.py:232 step:650K smpl:10M ep:54K epch:179.14 loss:0.002 grdn:0.148 lr:1.8e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 10:41:03 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.20 loss:0.002 grdn:0.146 lr:1.8e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 10:41:51 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.25 loss:0.002 grdn:0.142 lr:1.8e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 10:42:40 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.31 loss:0.002 grdn:0.159 lr:1.8e-05 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 10:43:29 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.36 loss:0.002 grdn:0.139 lr:1.8e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 10:44:17 ts/train.py:232 step:651K smpl:10M ep:54K epch:179.42 loss:0.002 grdn:0.151 lr:1.8e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 10:45:05 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.47 loss:0.002 grdn:0.145 lr:1.8e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 10:45:53 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.53 loss:0.002 grdn:0.153 lr:1.8e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 10:46:40 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.58 loss:0.002 grdn:0.150 lr:1.8e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 10:47:28 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.64 loss:0.002 grdn:0.141 lr:1.8e-05 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 10:48:16 ts/train.py:232 step:652K smpl:10M ep:54K epch:179.69 loss:0.002 grdn:0.154 lr:1.8e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 10:49:03 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.75 loss:0.003 grdn:0.163 lr:1.8e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 10:49:50 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.80 loss:0.002 grdn:0.139 lr:1.8e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 10:50:37 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.86 loss:0.002 grdn:0.138 lr:1.7e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 10:51:25 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.91 loss:0.002 grdn:0.152 lr:1.7e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 10:52:12 ts/train.py:232 step:653K smpl:10M ep:54K epch:179.97 loss:0.002 grdn:0.149 lr:1.7e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 10:52:59 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.02 loss:0.002 grdn:0.136 lr:1.7e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 10:53:48 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.08 loss:0.002 grdn:0.143 lr:1.7e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 10:54:35 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.13 loss:0.002 grdn:0.156 lr:1.7e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 10:55:25 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.19 loss:0.002 grdn:0.138 lr:1.7e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 10:56:12 ts/train.py:232 step:654K smpl:10M ep:54K epch:180.24 loss:0.002 grdn:0.124 lr:1.7e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 10:57:01 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.30 loss:0.002 grdn:0.154 lr:1.7e-05 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 10:57:46 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.35 loss:0.002 grdn:0.144 lr:1.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 10:58:30 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.41 loss:0.002 grdn:0.141 lr:1.7e-05 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 10:59:14 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.46 loss:0.002 grdn:0.129 lr:1.7e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 10:59:59 ts/train.py:232 step:655K smpl:10M ep:54K epch:180.52 loss:0.002 grdn:0.147 lr:1.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 11:00:44 ts/train.py:232 step:656K smpl:10M ep:54K epch:180.57 loss:0.003 grdn:0.164 lr:1.7e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 11:01:29 ts/train.py:232 step:656K smpl:10M ep:54K epch:180.63 loss:0.002 grdn:0.149 lr:1.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 11:02:15 ts/train.py:232 step:656K smpl:10M ep:54K epch:180.69 loss:0.002 grdn:0.143 lr:1.7e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 11:02:59 ts/train.py:232 step:656K smpl:10M ep:54K epch:180.74 loss:0.002 grdn:0.136 lr:1.7e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 11:03:45 ts/train.py:232 step:656K smpl:11M ep:54K epch:180.80 loss:0.002 grdn:0.131 lr:1.7e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 11:04:30 ts/train.py:232 step:657K smpl:11M ep:54K epch:180.85 loss:0.002 grdn:0.137 lr:1.7e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 11:05:16 ts/train.py:232 step:657K smpl:11M ep:54K epch:180.91 loss:0.002 grdn:0.153 lr:1.7e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 11:06:01 ts/train.py:232 step:657K smpl:11M ep:54K epch:180.96 loss:0.002 grdn:0.153 lr:1.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 11:06:47 ts/train.py:232 step:657K smpl:11M ep:54K epch:181.02 loss:0.002 grdn:0.159 lr:1.7e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 11:07:32 ts/train.py:232 step:657K smpl:11M ep:54K epch:181.07 loss:0.002 grdn:0.151 lr:1.7e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 11:08:17 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.13 loss:0.002 grdn:0.147 lr:1.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 11:09:06 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.18 loss:0.002 grdn:0.132 lr:1.7e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 11:09:51 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.24 loss:0.002 grdn:0.145 lr:1.7e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 11:10:39 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.29 loss:0.002 grdn:0.142 lr:1.7e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 11:11:25 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.35 loss:0.002 grdn:0.146 lr:1.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 11:12:10 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.40 loss:0.002 grdn:0.154 lr:1.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 11:12:56 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.46 loss:0.002 grdn:0.151 lr:1.7e-05 updt_s:0.157 data_s:0.070 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-internal.log b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-internal.log index 5f62ebc4cb523ae8972361205830e5ed5decd03e..6dd46a936b68234dcfbedfd645937ba693897139 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-internal.log @@ -12,3 +12,4 @@ {"time":"2025-05-06T10:17:58.342619055Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:59259->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T10:36:28.34160936Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:59320->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T11:17:58.342345052Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49666->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T23:06:26.097134902Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb index e521fd9d9c12531df0ea2001070881050579f631..33948994927451d3bb289eb58a369ea8df858fd6 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2c558cb99e14f34416b6e83db68958ee9731a6692645b4ec286ba77dee5cff0d -size 21659648 +oid sha256:123be03fff83191a6b7d8a24991f304a0b3a57b672160aa6bd01491d133dd680 +size 39059456