diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e80f2e67992da94e2ffc19814ba79fa1ffd4d4a3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bae2afbc4b6c8e6ae134c19e5f1904bdc1efd2f6bf2545b97f88b266117f8541 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..5e685a39dc53152fbe671658b7b94efc36625ab5 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 1.6560776687438e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1e052d6b46494953a9f4f7aadedbee2a89109f1b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15834021164d735592e254e2ef435da611cdadf93def6e48c33785726942506b +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..391fae3fc71281908f66980a5d95c3ed01ce824b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f4a153e1f2508caa235c5521905027545b38d0860fbfaffa49ff466f7749d26 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b278d58035ccdcfa8cd70128a420b21e10878aa9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 660000, + "verbose": false, + "_step_count": 660001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.6560776687438e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..1946a814a39401d13bc968ec845f3081adf9bc13 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/660000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 660000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..52b6f107b8535383748ce6ac611971ed4239fd99 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:324420e80305b8508b18a005391ee2d7fbb9589e4254510211f4c6d4f94011fc +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..4001dc6024934e2ec0029ed10ab9c1a4358ee373 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 1.2856394338260691e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c4c41f8b98828b79ff05833b8e92e670f2a3e4f8 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2c0e1b58b922bb137535385f86b8a1316ae95930a99f80a53e06bd7d46fb1bb +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9aa56bd2292584c0b8a730e0c792b3de2a950d59 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:553e00de9ff2e041b8b90bcab643b61965d3e788d662e67a1e4ef31a3aafb7f7 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bfe642da76c15eb5d08bf30c1a297ef1f5d4799c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 690000, + "verbose": false, + "_step_count": 690001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.2856394338260691e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8b5daa23c20b59c1eb7b88487eaeb111856e1464 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/690000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 690000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fc94c61da4e873fc6bc494fea53b2df835749b2e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:186a78dc63ff3b305baf6bad47472346fd843314ba34f1113c98d0e68fdde9c2 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..6926324a229b5f3a741c4acd6c9f9055873ff93e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.55941723863299e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2006b6e0e192be883838df9d62d5217b67e0de7e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a3f7a7d10c8ae4965074fbffa58dc8eab6a3e51e609ad038f2031be4180c281 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e87891dd333a2bd8035068a2d95533c6b80917b7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e554b4b3155a2bedbb9d7ec870733b3a9f28de316f71f74b33a6cbec6133890 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..57f7f32291879d54dd6884b139a3353e9208cf67 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 720000, + "verbose": false, + "_step_count": 720001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.55941723863299e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..721e07e355e18f869a5dd8aa6f83c651ec7c3772 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/720000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 720000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4a03a286fe2febe472f8c728be075cd3cfa6a969 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99fcae9bb918c5d99a5793d0b790a7825748d0c75aded7961e9c00b4e49087d5 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..910528e41cdfbb139e48f5656e7a9c98cbc08d9e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.706007892278027e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..168be755586ba9822ea70387ea531235cf53f05f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37a1d35bd6c42daed442b62da96adabafb88dc0fc9431e7b34f77c398c1fa1fa +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3a9fa2759d582cf91e35f3d18b818b16a6f396b1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b617848aba881da02a8a8ab0bbb22b567feef45165eedd21a973d93686c96ff +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..531a163e77edd5cec217b3d8dfdf81d78aac5b48 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 750000, + "verbose": false, + "_step_count": 750001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.706007892278027e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..362e20eb49f9138dd347449af4f6da372b3c74ac --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/750000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 750000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d0ff3cbe6f3c590313719b660a7438aa4c0adde4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7ade3f2ee8b0a98176553def11d221590060b07b543d702e440d8444709b46c +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9c890340d82de989b405601d8eb6f70d68373 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 4.327463582432278e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a26624c5541ad7462a3363386d89a1bc463ef66e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d1e84ef6c109719c049d88624195cd5ebf7becff460c7150cfb3d0692cc84b5 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c9444168701eefce4efc8c9c71902b464d57ac7d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55b853f170fd27addf7df90ffe016e1ae81c121e3e4e5c795f43705999b336a2 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1fed77bf3648d162e332234925925207b7372ef2 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 780000, + "verbose": false, + "_step_count": 780001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 4.327463582432278e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..e908e4d92cd192332ca7f02b2314283190a92940 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/780000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 780000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d58adffd6cb88e805a61e1d795f7c1df43389b21 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e675457baa7e30700dac529075cb054857e4a3c84e33cdc68cb7059bf508bc6 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..24fd12b096a8b3cc8374a56fa3f1444ddd1e79a7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.449873091292648e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4734e20d7cd65843485e498711fdd9b3f215be1c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bcfa90b034c3d99e233a8ab9205f9197165cebd45404818f47609aebb37b470 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..254af3637ff96390f6cd85f8af51b7a19d59448b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd509ba3fa364c99bae1713e1123aaaf643dc474eefef46249f12bd1da854cee +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ccfa1dee81e2d12eb7db6aab02f6d04e953e7d64 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 810000, + "verbose": false, + "_step_count": 810001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.449873091292648e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dd21436a578c3f7692acb15fcf45406552aea847 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/810000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 810000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..860506cc31fc2ce549cda69edeec3faf618e5b9e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9ec902562e3def9d983090becde990d1290580f8d667f1747fa412dd3c6968b +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..d7d9b557f0d16e9ca21004cc468242db56e55f03 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 1.0938305482918298e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a6bb5f4e8d241a1c75e20a7922eea186a7393d9f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e397479f1864be723eaf6543bca404228181e84ad718da8adaa62fee460f47be +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..95c6b6ae065ce7900087956b479988f6b566afe0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:746df261164d2886cb721e37198f6163bc3d4dabc27d783880fdf90b1668a4ee +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..34b4a3103a1dd0587363f83d1665fd3745f5768f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 840000, + "verbose": false, + "_step_count": 840001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.0938305482918298e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dcc98c000f570a21f87e055135c99c45d8095ae6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/840000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 840000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd059c9958747b7a6bb7f6e5f2246062547e5ba3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01db7fc743a428c8bcb4d3367ff6586ca80e062c37a0fc5d169c2497504e8dcd +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..ff95a438a66d456a3bc8a3c9f97111b5203e78a7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.742095458231675e-07, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cd19f72f91e83e3a0b57b3906a051bd8afb425fd --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e94c9bb5b6b231b9d3fab66c46f9b06a5c525c89dceb1d82763c30099d49724 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d40df9a0737adc65faaf41f6ff294b5fcd1e7d2d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b9df358f34aa001926a2a1bf74098da3f9cb1d5f289d75d80239dfcf7cb4024 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8b59af52612a6931b227893f3a6e8bb90aaf033d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 870000, + "verbose": false, + "_step_count": 870001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.742095458231675e-07 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d7dc2eb64e5995132035fbf3233a84acd6dbdfee --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/870000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 870000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1bcefe86fb1859b2be86fa94d72e3c4953c22855 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/pretrained_model/config.json @@ -0,0 +1,99 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4ec0356c7e4852947910909452f7fa53ed2897e0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06cd86fa1f399dff927bf859aca4bb03f29345584cf00a4afc0d3b4e26b4cb65 +size 369241872 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e136752c4bfa68a0cefe1e46333fc76daeefbd18 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/pretrained_model/train_config.json @@ -0,0 +1,209 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..c27412d579214db5ef0199d94aca48d5fbd9aecc --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 0.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bf1f90fdded4d675c0c71beca6da97e576bc181d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b688105ca288cc9dd6bdfd76157af09f4348cb74dd3e6cf9c14a44e3d17b1293 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..288e863fb427bbe2dcf05ede4e292a195f05170d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ce24577d982fe5c4bbfb09b1c951d42bdf1c6d5fabb4fe1665da178fc2ad39d +size 15708 diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f820026d12b5046df9a66ee511c37ba824ab0033 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 900000, + "verbose": false, + "_step_count": 900001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.0 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..6e08a40d8f4b15183bbf2305222048be81ca55ff --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/checkpoints/900000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 900000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log index 6dd46a936b68234dcfbedfd645937ba693897139..d93f4520937a44899889b6c25a81fecccb483edc 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log @@ -13,3 +13,13 @@ {"time":"2025-05-06T10:36:28.34160936Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:59320->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T11:17:58.342345052Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49666->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T23:06:26.097134902Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} +{"time":"2025-05-07T16:56:18.625880571Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:48120->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-07T23:27:42.933986298Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 172.17.0.4:38206->35.186.228.49:443: read: connection reset by peer"} +{"time":"2025-05-08T03:01:22.548356473Z","level":"INFO","msg":"stream: closing","id":"92tvlwhj"} +{"time":"2025-05-08T03:01:22.548431522Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-05-08T03:01:22.548580182Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-05-08T03:01:41.789898019Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-05-08T03:01:42.069164311Z","level":"INFO","msg":"handler: closed","stream_id":"92tvlwhj"} +{"time":"2025-05-08T03:01:42.06924091Z","level":"INFO","msg":"sender: closed","stream_id":"92tvlwhj"} +{"time":"2025-05-08T03:01:42.06923903Z","level":"INFO","msg":"writer: Close: closed","stream_id":"92tvlwhj"} +{"time":"2025-05-08T03:01:42.06937473Z","level":"INFO","msg":"stream: closed","id":"92tvlwhj"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug.log b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug.log index 6ea2688767a75521b6667fd6ce61f64355ddcb0b..5daad7b257415bdef856df5de33094bbaec2a06e 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/debug.log @@ -20,3 +20,4 @@ config: {'dataset': {'repo_id': 'shylee/so100_pengripA', 'root': '/SSD/LSY/lerob 2025-05-05 13:42:38,419 INFO MainThread:99163 [wandb_run.py:_redirect():2371] Wrapping output streams. 2025-05-05 13:42:38,419 INFO MainThread:99163 [wandb_run.py:_redirect():2394] Redirects installed. 2025-05-05 13:42:38,420 INFO MainThread:99163 [wandb_init.py:init():1056] run started, returning control to user process +2025-05-08 03:01:22,547 INFO MsgRouterThr:99163 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles. diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/config.yaml b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9471d9d7b4643a8da8826d22461602c1dea64f02 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/config.yaml @@ -0,0 +1,189 @@ +_wandb: + value: + cli_version: 0.19.9 + m: [] + python_version: 3.10.17 + t: + "1": + - 1 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 41 + - 49 + - 51 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.10.17 + "5": 0.19.9 + "8": + - 5 + "12": 0.19.9 + "13": linux-x86_64 +batch_size: + value: 16 +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: shylee/so100_pengripA + revision: null + root: /SSD/LSY/lerobot_dataset/pengripA + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4 +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.95 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 0.0001 + type: adam + weight_decay: 1e-06 +output_dir: + value: /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4 +policy: + value: + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + clip_sample_range: 1 + crop_is_random: false + crop_shape: + - 224 + - 224 + device: cuda + diffusion_step_embed_dim: 128 + do_mask_loss_for_padding: false + down_dims: + - 256 + - 512 + - 1024 + drop_n_last_frames: 7 + ema_lambda: 0.5 + freeze_vision_backbone: false + horizon: 16 + kernel_size: 5 + n_action_steps: 16 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler_type: DDIM + normalization_mapping: + ACTION: MIN_MAX + STATE: MIN_MAX + VISUAL: MEAN_STD + num_inference_steps: 10 + num_train_timesteps: 100 + optimizer_betas: + - 0.95 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-06 + pre_resize_shape: + - 224 + - 224 + prediction_type: epsilon + pretrained_backbone_weights: ResNet50_Weights.IMAGENET1K_V1 + scheduler_name: cosine + scheduler_warmup_steps: 500 + spatial_softmax_num_keypoints: 32 + type: diffusion + use_amp: false + use_ema: true + use_film_scale_modulation: true + use_group_norm: false + use_separate_rgb_encoder_per_camera: false + vision_backbone: resnet50 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 30000 +scheduler: + value: + name: cosine + num_warmup_steps: 500 + type: diffuser +seed: + value: 1000 +steps: + value: 900000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/output.log b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/output.log index ef4de12d4fb6130554cfd50f28798cafd9783314..80e427ffc1f0a2da05f9a1818ef22bfcaf518e2d 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/output.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/output.log @@ -4536,3 +4536,1219 @@ INFO 2025-05-07 11:10:39 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.29 l INFO 2025-05-07 11:11:25 ts/train.py:232 step:658K smpl:11M ep:54K epch:181.35 loss:0.002 grdn:0.146 lr:1.7e-05 updt_s:0.157 data_s:0.070 INFO 2025-05-07 11:12:10 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.40 loss:0.002 grdn:0.154 lr:1.7e-05 updt_s:0.157 data_s:0.070 INFO 2025-05-07 11:12:56 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.46 loss:0.002 grdn:0.151 lr:1.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 11:13:41 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.51 loss:0.002 grdn:0.151 lr:1.7e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 11:14:26 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.57 loss:0.002 grdn:0.148 lr:1.7e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 11:15:12 ts/train.py:232 step:659K smpl:11M ep:54K epch:181.62 loss:0.002 grdn:0.156 lr:1.7e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 11:16:00 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.68 loss:0.002 grdn:0.149 lr:1.7e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 11:16:50 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.73 loss:0.002 grdn:0.132 lr:1.7e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 11:17:35 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.79 loss:0.002 grdn:0.130 lr:1.7e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 11:17:35 ts/train.py:241 Checkpoint policy after step 660000 +INFO 2025-05-07 11:18:23 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.84 loss:0.002 grdn:0.152 lr:1.7e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 11:19:10 ts/train.py:232 step:660K smpl:11M ep:55K epch:181.90 loss:0.002 grdn:0.150 lr:1.7e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 11:19:55 ts/train.py:232 step:661K smpl:11M ep:55K epch:181.95 loss:0.002 grdn:0.161 lr:1.6e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 11:20:40 ts/train.py:232 step:661K smpl:11M ep:55K epch:182.01 loss:0.002 grdn:0.159 lr:1.6e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 11:21:25 ts/train.py:232 step:661K smpl:11M ep:55K epch:182.06 loss:0.002 grdn:0.133 lr:1.6e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 11:22:10 ts/train.py:232 step:661K smpl:11M ep:55K epch:182.12 loss:0.002 grdn:0.127 lr:1.6e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 11:22:56 ts/train.py:232 step:661K smpl:11M ep:55K epch:182.17 loss:0.002 grdn:0.145 lr:1.6e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 11:23:44 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.23 loss:0.002 grdn:0.139 lr:1.6e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 11:24:32 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.28 loss:0.002 grdn:0.139 lr:1.6e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 11:25:21 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.34 loss:0.002 grdn:0.163 lr:1.6e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 11:26:09 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.39 loss:0.002 grdn:0.139 lr:1.6e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 11:26:55 ts/train.py:232 step:662K smpl:11M ep:55K epch:182.45 loss:0.002 grdn:0.155 lr:1.6e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 11:27:40 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.50 loss:0.003 grdn:0.171 lr:1.6e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 11:28:26 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.56 loss:0.002 grdn:0.144 lr:1.6e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 11:29:11 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.61 loss:0.002 grdn:0.144 lr:1.6e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 11:29:56 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.67 loss:0.002 grdn:0.162 lr:1.6e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 11:30:42 ts/train.py:232 step:663K smpl:11M ep:55K epch:182.72 loss:0.002 grdn:0.146 lr:1.6e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 11:31:27 ts/train.py:232 step:664K smpl:11M ep:55K epch:182.78 loss:0.002 grdn:0.141 lr:1.6e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 11:32:12 ts/train.py:232 step:664K smpl:11M ep:55K epch:182.83 loss:0.002 grdn:0.150 lr:1.6e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 11:32:57 ts/train.py:232 step:664K smpl:11M ep:55K epch:182.89 loss:0.002 grdn:0.131 lr:1.6e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 11:33:42 ts/train.py:232 step:664K smpl:11M ep:55K epch:182.94 loss:0.002 grdn:0.131 lr:1.6e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 11:34:27 ts/train.py:232 step:664K smpl:11M ep:55K epch:183.00 loss:0.002 grdn:0.146 lr:1.6e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 11:35:12 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.05 loss:0.002 grdn:0.149 lr:1.6e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 11:35:59 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.11 loss:0.002 grdn:0.147 lr:1.6e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 11:36:44 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.16 loss:0.002 grdn:0.147 lr:1.6e-05 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 11:37:34 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.22 loss:0.002 grdn:0.160 lr:1.6e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 11:38:19 ts/train.py:232 step:665K smpl:11M ep:55K epch:183.27 loss:0.002 grdn:0.160 lr:1.6e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 11:39:04 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.33 loss:0.002 grdn:0.147 lr:1.6e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 11:39:50 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.38 loss:0.002 grdn:0.128 lr:1.6e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 11:40:37 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.44 loss:0.002 grdn:0.129 lr:1.6e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 11:41:23 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.49 loss:0.002 grdn:0.148 lr:1.6e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 11:42:10 ts/train.py:232 step:666K smpl:11M ep:55K epch:183.55 loss:0.002 grdn:0.144 lr:1.6e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 11:42:56 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.60 loss:0.002 grdn:0.154 lr:1.6e-05 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 11:43:41 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.66 loss:0.002 grdn:0.145 lr:1.6e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 11:44:25 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.71 loss:0.002 grdn:0.138 lr:1.6e-05 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 11:45:12 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.77 loss:0.002 grdn:0.144 lr:1.6e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 11:45:58 ts/train.py:232 step:667K smpl:11M ep:55K epch:183.83 loss:0.002 grdn:0.152 lr:1.6e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 11:46:45 ts/train.py:232 step:668K smpl:11M ep:55K epch:183.88 loss:0.002 grdn:0.138 lr:1.6e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 11:47:31 ts/train.py:232 step:668K smpl:11M ep:55K epch:183.94 loss:0.002 grdn:0.153 lr:1.6e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 11:48:15 ts/train.py:232 step:668K smpl:11M ep:55K epch:183.99 loss:0.002 grdn:0.154 lr:1.6e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 11:49:00 ts/train.py:232 step:668K smpl:11M ep:55K epch:184.05 loss:0.002 grdn:0.152 lr:1.6e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 11:49:44 ts/train.py:232 step:668K smpl:11M ep:55K epch:184.10 loss:0.002 grdn:0.146 lr:1.5e-05 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 11:50:32 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.16 loss:0.002 grdn:0.139 lr:1.5e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 11:51:21 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.21 loss:0.002 grdn:0.141 lr:1.5e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 11:52:09 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.27 loss:0.002 grdn:0.147 lr:1.5e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 11:52:58 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.32 loss:0.002 grdn:0.127 lr:1.5e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 11:53:46 ts/train.py:232 step:669K smpl:11M ep:55K epch:184.38 loss:0.002 grdn:0.138 lr:1.5e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 11:54:34 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.43 loss:0.002 grdn:0.155 lr:1.5e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 11:55:22 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.49 loss:0.002 grdn:0.160 lr:1.5e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 11:56:11 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.54 loss:0.002 grdn:0.137 lr:1.5e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 11:56:59 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.60 loss:0.002 grdn:0.144 lr:1.5e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 11:57:47 ts/train.py:232 step:670K smpl:11M ep:55K epch:184.65 loss:0.002 grdn:0.162 lr:1.5e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 11:58:35 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.71 loss:0.002 grdn:0.136 lr:1.5e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 11:59:23 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.76 loss:0.002 grdn:0.145 lr:1.5e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 12:00:11 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.82 loss:0.002 grdn:0.138 lr:1.5e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 12:00:59 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.87 loss:0.002 grdn:0.152 lr:1.5e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 12:01:47 ts/train.py:232 step:671K smpl:11M ep:55K epch:184.93 loss:0.002 grdn:0.143 lr:1.5e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 12:02:34 ts/train.py:232 step:672K smpl:11M ep:55K epch:184.98 loss:0.002 grdn:0.152 lr:1.5e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 12:03:22 ts/train.py:232 step:672K smpl:11M ep:56K epch:185.04 loss:0.002 grdn:0.161 lr:1.5e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 12:04:10 ts/train.py:232 step:672K smpl:11M ep:56K epch:185.09 loss:0.002 grdn:0.141 lr:1.5e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 12:05:01 ts/train.py:232 step:672K smpl:11M ep:56K epch:185.15 loss:0.002 grdn:0.133 lr:1.5e-05 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 12:05:49 ts/train.py:232 step:672K smpl:11M ep:56K epch:185.20 loss:0.002 grdn:0.136 lr:1.5e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 12:06:37 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.26 loss:0.002 grdn:0.146 lr:1.5e-05 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 12:07:25 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.31 loss:0.002 grdn:0.139 lr:1.5e-05 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 12:08:13 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.37 loss:0.002 grdn:0.132 lr:1.5e-05 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 12:09:00 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.42 loss:0.002 grdn:0.150 lr:1.5e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 12:09:48 ts/train.py:232 step:673K smpl:11M ep:56K epch:185.48 loss:0.002 grdn:0.155 lr:1.5e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 12:10:35 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.53 loss:0.002 grdn:0.144 lr:1.5e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 12:11:23 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.59 loss:0.002 grdn:0.143 lr:1.5e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 12:12:10 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.64 loss:0.002 grdn:0.135 lr:1.5e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 12:12:58 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.70 loss:0.002 grdn:0.152 lr:1.5e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 12:13:44 ts/train.py:232 step:674K smpl:11M ep:56K epch:185.75 loss:0.002 grdn:0.138 lr:1.5e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 12:14:32 ts/train.py:232 step:675K smpl:11M ep:56K epch:185.81 loss:0.002 grdn:0.145 lr:1.5e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 12:15:19 ts/train.py:232 step:675K smpl:11M ep:56K epch:185.86 loss:0.002 grdn:0.151 lr:1.5e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 12:16:07 ts/train.py:232 step:675K smpl:11M ep:56K epch:185.92 loss:0.002 grdn:0.145 lr:1.5e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 12:16:55 ts/train.py:232 step:675K smpl:11M ep:56K epch:185.97 loss:0.002 grdn:0.126 lr:1.5e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 12:17:43 ts/train.py:232 step:675K smpl:11M ep:56K epch:186.03 loss:0.002 grdn:0.145 lr:1.5e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 12:18:33 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.08 loss:0.002 grdn:0.127 lr:1.5e-05 updt_s:0.156 data_s:0.093 +INFO 2025-05-07 12:19:22 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.14 loss:0.002 grdn:0.144 lr:1.5e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 12:20:10 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.19 loss:0.002 grdn:0.150 lr:1.5e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 12:20:58 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.25 loss:0.002 grdn:0.134 lr:1.5e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 12:21:49 ts/train.py:232 step:676K smpl:11M ep:56K epch:186.30 loss:0.002 grdn:0.131 lr:1.4e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-07 12:22:40 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.36 loss:0.002 grdn:0.150 lr:1.4e-05 updt_s:0.157 data_s:0.098 +INFO 2025-05-07 12:23:33 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.41 loss:0.002 grdn:0.146 lr:1.4e-05 updt_s:0.157 data_s:0.104 +INFO 2025-05-07 12:24:24 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.47 loss:0.002 grdn:0.157 lr:1.4e-05 updt_s:0.156 data_s:0.102 +INFO 2025-05-07 12:25:17 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.52 loss:0.002 grdn:0.141 lr:1.4e-05 updt_s:0.156 data_s:0.105 +INFO 2025-05-07 12:26:08 ts/train.py:232 step:677K smpl:11M ep:56K epch:186.58 loss:0.002 grdn:0.159 lr:1.4e-05 updt_s:0.156 data_s:0.099 +INFO 2025-05-07 12:27:00 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.63 loss:0.002 grdn:0.149 lr:1.4e-05 updt_s:0.156 data_s:0.101 +INFO 2025-05-07 12:27:51 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.69 loss:0.002 grdn:0.137 lr:1.4e-05 updt_s:0.157 data_s:0.097 +INFO 2025-05-07 12:28:42 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.74 loss:0.002 grdn:0.138 lr:1.4e-05 updt_s:0.156 data_s:0.100 +INFO 2025-05-07 12:29:33 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.80 loss:0.002 grdn:0.149 lr:1.4e-05 updt_s:0.156 data_s:0.097 +INFO 2025-05-07 12:30:23 ts/train.py:232 step:678K smpl:11M ep:56K epch:186.85 loss:0.002 grdn:0.135 lr:1.4e-05 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 12:31:13 ts/train.py:232 step:679K smpl:11M ep:56K epch:186.91 loss:0.002 grdn:0.146 lr:1.4e-05 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 12:32:04 ts/train.py:232 step:679K smpl:11M ep:56K epch:186.97 loss:0.002 grdn:0.151 lr:1.4e-05 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 12:32:55 ts/train.py:232 step:679K smpl:11M ep:56K epch:187.02 loss:0.002 grdn:0.124 lr:1.4e-05 updt_s:0.156 data_s:0.098 +INFO 2025-05-07 12:33:42 ts/train.py:232 step:679K smpl:11M ep:56K epch:187.08 loss:0.002 grdn:0.137 lr:1.4e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 12:34:27 ts/train.py:232 step:679K smpl:11M ep:56K epch:187.13 loss:0.002 grdn:0.154 lr:1.4e-05 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 12:35:12 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.19 loss:0.002 grdn:0.145 lr:1.4e-05 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 12:35:58 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.24 loss:0.002 grdn:0.139 lr:1.4e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 12:36:43 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.30 loss:0.002 grdn:0.148 lr:1.4e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 12:37:28 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.35 loss:0.002 grdn:0.150 lr:1.4e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 12:38:14 ts/train.py:232 step:680K smpl:11M ep:56K epch:187.41 loss:0.002 grdn:0.145 lr:1.4e-05 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 12:38:59 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.46 loss:0.002 grdn:0.167 lr:1.4e-05 updt_s:0.156 data_s:0.070 +INFO 2025-05-07 12:39:45 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.52 loss:0.002 grdn:0.142 lr:1.4e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 12:40:32 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.57 loss:0.002 grdn:0.155 lr:1.4e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 12:41:16 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.63 loss:0.002 grdn:0.149 lr:1.4e-05 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 12:42:01 ts/train.py:232 step:681K smpl:11M ep:56K epch:187.68 loss:0.002 grdn:0.142 lr:1.4e-05 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 12:42:46 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.74 loss:0.002 grdn:0.143 lr:1.4e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 12:43:33 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.79 loss:0.002 grdn:0.153 lr:1.4e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 12:44:18 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.85 loss:0.002 grdn:0.132 lr:1.4e-05 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 12:45:03 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.90 loss:0.002 grdn:0.150 lr:1.4e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 12:45:49 ts/train.py:232 step:682K smpl:11M ep:56K epch:187.96 loss:0.002 grdn:0.138 lr:1.4e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 12:46:37 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.01 loss:0.002 grdn:0.159 lr:1.4e-05 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 12:47:24 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.07 loss:0.002 grdn:0.156 lr:1.4e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 12:48:10 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.12 loss:0.002 grdn:0.146 lr:1.4e-05 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 12:48:56 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.18 loss:0.002 grdn:0.146 lr:1.4e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 12:49:43 ts/train.py:232 step:683K smpl:11M ep:56K epch:188.23 loss:0.002 grdn:0.144 lr:1.4e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 12:50:30 ts/train.py:232 step:684K smpl:11M ep:56K epch:188.29 loss:0.002 grdn:0.145 lr:1.4e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 12:51:17 ts/train.py:232 step:684K smpl:11M ep:57K epch:188.34 loss:0.002 grdn:0.156 lr:1.4e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 12:52:05 ts/train.py:232 step:684K smpl:11M ep:57K epch:188.40 loss:0.002 grdn:0.143 lr:1.4e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 12:52:54 ts/train.py:232 step:684K smpl:11M ep:57K epch:188.45 loss:0.002 grdn:0.140 lr:1.4e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 12:53:42 ts/train.py:232 step:684K smpl:11M ep:57K epch:188.51 loss:0.002 grdn:0.129 lr:1.4e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 12:54:31 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.56 loss:0.002 grdn:0.142 lr:1.4e-05 updt_s:0.158 data_s:0.089 +INFO 2025-05-07 12:55:18 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.62 loss:0.002 grdn:0.142 lr:1.3e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 12:56:04 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.67 loss:0.002 grdn:0.126 lr:1.3e-05 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 12:56:50 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.73 loss:0.002 grdn:0.142 lr:1.3e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 12:57:36 ts/train.py:232 step:685K smpl:11M ep:57K epch:188.78 loss:0.002 grdn:0.142 lr:1.3e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 12:58:23 ts/train.py:232 step:686K smpl:11M ep:57K epch:188.84 loss:0.002 grdn:0.137 lr:1.3e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 12:59:09 ts/train.py:232 step:686K smpl:11M ep:57K epch:188.89 loss:0.002 grdn:0.136 lr:1.3e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 12:59:56 ts/train.py:232 step:686K smpl:11M ep:57K epch:188.95 loss:0.002 grdn:0.159 lr:1.3e-05 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 13:00:48 ts/train.py:232 step:686K smpl:11M ep:57K epch:189.00 loss:0.002 grdn:0.148 lr:1.3e-05 updt_s:0.156 data_s:0.107 +INFO 2025-05-07 13:01:38 ts/train.py:232 step:686K smpl:11M ep:57K epch:189.06 loss:0.002 grdn:0.133 lr:1.3e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 13:02:28 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.11 loss:0.002 grdn:0.148 lr:1.3e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 13:03:17 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.17 loss:0.002 grdn:0.153 lr:1.3e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 13:04:03 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.22 loss:0.002 grdn:0.147 lr:1.3e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 13:04:49 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.28 loss:0.002 grdn:0.148 lr:1.3e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 13:05:36 ts/train.py:232 step:687K smpl:11M ep:57K epch:189.33 loss:0.002 grdn:0.155 lr:1.3e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 13:06:23 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.39 loss:0.002 grdn:0.145 lr:1.3e-05 updt_s:0.158 data_s:0.075 +INFO 2025-05-07 13:07:09 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.44 loss:0.002 grdn:0.136 lr:1.3e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 13:07:55 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.50 loss:0.002 grdn:0.136 lr:1.3e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 13:08:42 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.55 loss:0.002 grdn:0.152 lr:1.3e-05 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 13:09:30 ts/train.py:232 step:688K smpl:11M ep:57K epch:189.61 loss:0.002 grdn:0.128 lr:1.3e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 13:10:17 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.66 loss:0.002 grdn:0.134 lr:1.3e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 13:11:04 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.72 loss:0.002 grdn:0.131 lr:1.3e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 13:11:50 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.77 loss:0.002 grdn:0.149 lr:1.3e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 13:12:38 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.83 loss:0.002 grdn:0.132 lr:1.3e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 13:13:24 ts/train.py:232 step:689K smpl:11M ep:57K epch:189.88 loss:0.002 grdn:0.143 lr:1.3e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 13:14:13 ts/train.py:232 step:690K smpl:11M ep:57K epch:189.94 loss:0.002 grdn:0.142 lr:1.3e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 13:15:00 ts/train.py:232 step:690K smpl:11M ep:57K epch:189.99 loss:0.002 grdn:0.150 lr:1.3e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 13:15:49 ts/train.py:232 step:690K smpl:11M ep:57K epch:190.05 loss:0.002 grdn:0.139 lr:1.3e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 13:15:49 ts/train.py:241 Checkpoint policy after step 690000 +INFO 2025-05-07 13:16:40 ts/train.py:232 step:690K smpl:11M ep:57K epch:190.11 loss:0.002 grdn:0.164 lr:1.3e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 13:17:26 ts/train.py:232 step:690K smpl:11M ep:57K epch:190.16 loss:0.002 grdn:0.153 lr:1.3e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 13:18:17 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.22 loss:0.002 grdn:0.131 lr:1.3e-05 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 13:19:03 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.27 loss:0.002 grdn:0.150 lr:1.3e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 13:19:50 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.33 loss:0.002 grdn:0.153 lr:1.3e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 13:20:38 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.38 loss:0.002 grdn:0.167 lr:1.3e-05 updt_s:0.158 data_s:0.085 +INFO 2025-05-07 13:21:25 ts/train.py:232 step:691K smpl:11M ep:57K epch:190.44 loss:0.002 grdn:0.157 lr:1.3e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 13:22:11 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.49 loss:0.002 grdn:0.136 lr:1.3e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 13:22:56 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.55 loss:0.002 grdn:0.144 lr:1.3e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 13:23:42 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.60 loss:0.002 grdn:0.148 lr:1.3e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 13:24:28 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.66 loss:0.002 grdn:0.156 lr:1.3e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 13:25:14 ts/train.py:232 step:692K smpl:11M ep:57K epch:190.71 loss:0.002 grdn:0.159 lr:1.3e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 13:25:59 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.77 loss:0.002 grdn:0.143 lr:1.3e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 13:26:46 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.82 loss:0.002 grdn:0.153 lr:1.3e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 13:27:33 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.88 loss:0.002 grdn:0.147 lr:1.3e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 13:28:25 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.93 loss:0.002 grdn:0.141 lr:1.2e-05 updt_s:0.157 data_s:0.101 +INFO 2025-05-07 13:29:13 ts/train.py:232 step:693K smpl:11M ep:57K epch:190.99 loss:0.002 grdn:0.153 lr:1.2e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 13:30:03 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.04 loss:0.002 grdn:0.147 lr:1.2e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 13:30:52 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.10 loss:0.002 grdn:0.156 lr:1.2e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 13:31:40 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.15 loss:0.002 grdn:0.140 lr:1.2e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 13:32:28 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.21 loss:0.002 grdn:0.127 lr:1.2e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 13:33:19 ts/train.py:232 step:694K smpl:11M ep:57K epch:191.26 loss:0.002 grdn:0.139 lr:1.2e-05 updt_s:0.156 data_s:0.095 +INFO 2025-05-07 13:34:08 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.32 loss:0.002 grdn:0.142 lr:1.2e-05 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 13:34:57 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.37 loss:0.002 grdn:0.151 lr:1.2e-05 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 13:35:46 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.43 loss:0.002 grdn:0.141 lr:1.2e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 13:36:36 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.48 loss:0.002 grdn:0.130 lr:1.2e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 13:37:25 ts/train.py:232 step:695K smpl:11M ep:57K epch:191.54 loss:0.002 grdn:0.139 lr:1.2e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 13:38:13 ts/train.py:232 step:696K smpl:11M ep:57K epch:191.59 loss:0.002 grdn:0.140 lr:1.2e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 13:39:03 ts/train.py:232 step:696K smpl:11M ep:57K epch:191.65 loss:0.002 grdn:0.131 lr:1.2e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 13:39:52 ts/train.py:232 step:696K smpl:11M ep:58K epch:191.70 loss:0.002 grdn:0.143 lr:1.2e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 13:40:40 ts/train.py:232 step:696K smpl:11M ep:58K epch:191.76 loss:0.002 grdn:0.149 lr:1.2e-05 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 13:41:29 ts/train.py:232 step:696K smpl:11M ep:58K epch:191.81 loss:0.002 grdn:0.156 lr:1.2e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 13:42:21 ts/train.py:232 step:697K smpl:11M ep:58K epch:191.87 loss:0.002 grdn:0.131 lr:1.2e-05 updt_s:0.156 data_s:0.099 +INFO 2025-05-07 13:43:10 ts/train.py:232 step:697K smpl:11M ep:58K epch:191.92 loss:0.002 grdn:0.136 lr:1.2e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 13:44:02 ts/train.py:232 step:697K smpl:11M ep:58K epch:191.98 loss:0.002 grdn:0.150 lr:1.2e-05 updt_s:0.157 data_s:0.100 +INFO 2025-05-07 13:44:52 ts/train.py:232 step:697K smpl:11M ep:58K epch:192.03 loss:0.002 grdn:0.157 lr:1.2e-05 updt_s:0.156 data_s:0.096 +INFO 2025-05-07 13:45:42 ts/train.py:232 step:697K smpl:11M ep:58K epch:192.09 loss:0.002 grdn:0.148 lr:1.2e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 13:46:31 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.14 loss:0.002 grdn:0.140 lr:1.2e-05 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 13:47:19 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.20 loss:0.002 grdn:0.130 lr:1.2e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 13:48:07 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.25 loss:0.002 grdn:0.140 lr:1.2e-05 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 13:48:55 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.31 loss:0.002 grdn:0.125 lr:1.2e-05 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 13:49:42 ts/train.py:232 step:698K smpl:11M ep:58K epch:192.36 loss:0.002 grdn:0.139 lr:1.2e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 13:50:30 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.42 loss:0.002 grdn:0.139 lr:1.2e-05 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 13:51:17 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.47 loss:0.002 grdn:0.155 lr:1.2e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 13:52:04 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.53 loss:0.002 grdn:0.147 lr:1.2e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 13:52:50 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.58 loss:0.002 grdn:0.127 lr:1.2e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 13:53:37 ts/train.py:232 step:699K smpl:11M ep:58K epch:192.64 loss:0.002 grdn:0.138 lr:1.2e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 13:54:24 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.69 loss:0.002 grdn:0.148 lr:1.2e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 13:55:13 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.75 loss:0.002 grdn:0.143 lr:1.2e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 13:56:00 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.80 loss:0.002 grdn:0.130 lr:1.2e-05 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 13:56:53 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.86 loss:0.002 grdn:0.147 lr:1.2e-05 updt_s:0.157 data_s:0.105 +INFO 2025-05-07 13:57:42 ts/train.py:232 step:700K smpl:11M ep:58K epch:192.91 loss:0.002 grdn:0.151 lr:1.2e-05 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 13:58:32 ts/train.py:232 step:701K smpl:11M ep:58K epch:192.97 loss:0.002 grdn:0.136 lr:1.2e-05 updt_s:0.156 data_s:0.095 +INFO 2025-05-07 13:59:22 ts/train.py:232 step:701K smpl:11M ep:58K epch:193.02 loss:0.002 grdn:0.144 lr:1.2e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 14:00:13 ts/train.py:232 step:701K smpl:11M ep:58K epch:193.08 loss:0.002 grdn:0.136 lr:1.2e-05 updt_s:0.157 data_s:0.097 +INFO 2025-05-07 14:01:03 ts/train.py:232 step:701K smpl:11M ep:58K epch:193.13 loss:0.002 grdn:0.141 lr:1.2e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 14:01:52 ts/train.py:232 step:701K smpl:11M ep:58K epch:193.19 loss:0.002 grdn:0.134 lr:1.2e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 14:02:42 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.24 loss:0.002 grdn:0.149 lr:1.2e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 14:03:31 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.30 loss:0.002 grdn:0.133 lr:1.2e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 14:04:21 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.36 loss:0.002 grdn:0.137 lr:1.1e-05 updt_s:0.157 data_s:0.094 +INFO 2025-05-07 14:05:11 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.41 loss:0.002 grdn:0.150 lr:1.1e-05 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 14:06:02 ts/train.py:232 step:702K smpl:11M ep:58K epch:193.47 loss:0.002 grdn:0.129 lr:1.1e-05 updt_s:0.157 data_s:0.094 +INFO 2025-05-07 14:06:51 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.52 loss:0.002 grdn:0.160 lr:1.1e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 14:07:40 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.58 loss:0.002 grdn:0.146 lr:1.1e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 14:08:31 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.63 loss:0.002 grdn:0.133 lr:1.1e-05 updt_s:0.157 data_s:0.096 +INFO 2025-05-07 14:09:21 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.69 loss:0.002 grdn:0.129 lr:1.1e-05 updt_s:0.158 data_s:0.093 +INFO 2025-05-07 14:10:11 ts/train.py:232 step:703K smpl:11M ep:58K epch:193.74 loss:0.002 grdn:0.134 lr:1.1e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 14:11:02 ts/train.py:232 step:704K smpl:11M ep:58K epch:193.80 loss:0.002 grdn:0.142 lr:1.1e-05 updt_s:0.156 data_s:0.099 +INFO 2025-05-07 14:11:52 ts/train.py:232 step:704K smpl:11M ep:58K epch:193.85 loss:0.002 grdn:0.158 lr:1.1e-05 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 14:12:42 ts/train.py:232 step:704K smpl:11M ep:58K epch:193.91 loss:0.002 grdn:0.148 lr:1.1e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 14:13:31 ts/train.py:232 step:704K smpl:11M ep:58K epch:193.96 loss:0.002 grdn:0.149 lr:1.1e-05 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 14:14:20 ts/train.py:232 step:704K smpl:11M ep:58K epch:194.02 loss:0.002 grdn:0.152 lr:1.1e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 14:15:09 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.07 loss:0.002 grdn:0.136 lr:1.1e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 14:16:00 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.13 loss:0.002 grdn:0.142 lr:1.1e-05 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 14:16:53 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.18 loss:0.002 grdn:0.137 lr:1.1e-05 updt_s:0.158 data_s:0.108 +INFO 2025-05-07 14:17:45 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.24 loss:0.002 grdn:0.148 lr:1.1e-05 updt_s:0.157 data_s:0.103 +INFO 2025-05-07 14:18:36 ts/train.py:232 step:705K smpl:11M ep:58K epch:194.29 loss:0.002 grdn:0.148 lr:1.1e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-07 14:19:28 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.35 loss:0.002 grdn:0.151 lr:1.1e-05 updt_s:0.157 data_s:0.099 +INFO 2025-05-07 14:20:17 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.40 loss:0.002 grdn:0.140 lr:1.1e-05 updt_s:0.158 data_s:0.091 +INFO 2025-05-07 14:21:07 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.46 loss:0.002 grdn:0.150 lr:1.1e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 14:21:57 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.51 loss:0.002 grdn:0.149 lr:1.1e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 14:22:47 ts/train.py:232 step:706K smpl:11M ep:58K epch:194.57 loss:0.002 grdn:0.147 lr:1.1e-05 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 14:23:36 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.62 loss:0.002 grdn:0.161 lr:1.1e-05 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 14:24:27 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.68 loss:0.002 grdn:0.129 lr:1.1e-05 updt_s:0.157 data_s:0.096 +INFO 2025-05-07 14:25:16 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.73 loss:0.002 grdn:0.138 lr:1.1e-05 updt_s:0.156 data_s:0.092 +INFO 2025-05-07 14:26:12 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.79 loss:0.002 grdn:0.121 lr:1.1e-05 updt_s:0.158 data_s:0.119 +INFO 2025-05-07 14:27:03 ts/train.py:232 step:707K smpl:11M ep:58K epch:194.84 loss:0.002 grdn:0.143 lr:1.1e-05 updt_s:0.157 data_s:0.097 +INFO 2025-05-07 14:27:55 ts/train.py:232 step:708K smpl:11M ep:58K epch:194.90 loss:0.002 grdn:0.136 lr:1.1e-05 updt_s:0.158 data_s:0.103 +INFO 2025-05-07 14:28:45 ts/train.py:232 step:708K smpl:11M ep:58K epch:194.95 loss:0.002 grdn:0.144 lr:1.1e-05 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 14:29:32 ts/train.py:232 step:708K smpl:11M ep:59K epch:195.01 loss:0.002 grdn:0.159 lr:1.1e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 14:30:20 ts/train.py:232 step:708K smpl:11M ep:59K epch:195.06 loss:0.002 grdn:0.138 lr:1.1e-05 updt_s:0.158 data_s:0.080 +INFO 2025-05-07 14:31:08 ts/train.py:232 step:708K smpl:11M ep:59K epch:195.12 loss:0.002 grdn:0.148 lr:1.1e-05 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 14:31:55 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.17 loss:0.002 grdn:0.142 lr:1.1e-05 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 14:32:42 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.23 loss:0.002 grdn:0.137 lr:1.1e-05 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 14:33:30 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.28 loss:0.002 grdn:0.128 lr:1.1e-05 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 14:34:16 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.34 loss:0.002 grdn:0.143 lr:1.1e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 14:35:04 ts/train.py:232 step:709K smpl:11M ep:59K epch:195.39 loss:0.002 grdn:0.134 lr:1.1e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 14:35:51 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.45 loss:0.002 grdn:0.141 lr:1.1e-05 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 14:36:37 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.50 loss:0.002 grdn:0.149 lr:1.1e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 14:37:22 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.56 loss:0.002 grdn:0.128 lr:1.1e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 14:38:09 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.61 loss:0.002 grdn:0.131 lr:1.1e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 14:38:56 ts/train.py:232 step:710K smpl:11M ep:59K epch:195.67 loss:0.002 grdn:0.140 lr:1.1e-05 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 14:39:46 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.72 loss:0.002 grdn:0.129 lr:1.1e-05 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 14:40:34 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.78 loss:0.002 grdn:0.133 lr:1.1e-05 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 14:41:21 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.83 loss:0.002 grdn:0.143 lr:1.1e-05 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 14:42:06 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.89 loss:0.002 grdn:0.149 lr:1.0e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 14:42:52 ts/train.py:232 step:711K smpl:11M ep:59K epch:195.94 loss:0.002 grdn:0.131 lr:1.0e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 14:43:37 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.00 loss:0.002 grdn:0.144 lr:1.0e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 14:44:22 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.05 loss:0.002 grdn:0.143 lr:1.0e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 14:45:07 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.11 loss:0.002 grdn:0.154 lr:1.0e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 14:45:53 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.16 loss:0.002 grdn:0.154 lr:1.0e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 14:46:38 ts/train.py:232 step:712K smpl:11M ep:59K epch:196.22 loss:0.002 grdn:0.147 lr:1.0e-05 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 14:47:23 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.27 loss:0.002 grdn:0.128 lr:1.0e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 14:48:09 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.33 loss:0.001 grdn:0.124 lr:1.0e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 14:48:54 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.38 loss:0.002 grdn:0.142 lr:1.0e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 14:49:40 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.44 loss:0.002 grdn:0.144 lr:1.0e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 14:50:26 ts/train.py:232 step:713K smpl:11M ep:59K epch:196.50 loss:0.002 grdn:0.153 lr:1.0e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 14:51:12 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.55 loss:0.002 grdn:0.137 lr:1.0e-05 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 14:51:58 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.61 loss:0.002 grdn:0.146 lr:1.0e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 14:52:43 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.66 loss:0.002 grdn:0.137 lr:1.0e-05 updt_s:0.156 data_s:0.070 +INFO 2025-05-07 14:53:34 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.72 loss:0.002 grdn:0.135 lr:1.0e-05 updt_s:0.157 data_s:0.098 +INFO 2025-05-07 14:54:23 ts/train.py:232 step:714K smpl:11M ep:59K epch:196.77 loss:0.002 grdn:0.144 lr:1.0e-05 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 14:55:09 ts/train.py:232 step:715K smpl:11M ep:59K epch:196.83 loss:0.002 grdn:0.135 lr:1.0e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 14:55:54 ts/train.py:232 step:715K smpl:11M ep:59K epch:196.88 loss:0.002 grdn:0.137 lr:1.0e-05 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 14:56:40 ts/train.py:232 step:715K smpl:11M ep:59K epch:196.94 loss:0.002 grdn:0.139 lr:1.0e-05 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 14:57:26 ts/train.py:232 step:715K smpl:11M ep:59K epch:196.99 loss:0.001 grdn:0.122 lr:1.0e-05 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 14:58:11 ts/train.py:232 step:715K smpl:11M ep:59K epch:197.05 loss:0.002 grdn:0.152 lr:1.0e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 14:58:56 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.10 loss:0.002 grdn:0.149 lr:1.0e-05 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 14:59:41 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.16 loss:0.002 grdn:0.137 lr:1.0e-05 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 15:00:26 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.21 loss:0.002 grdn:0.133 lr:1.0e-05 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 15:01:13 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.27 loss:0.002 grdn:0.139 lr:1.0e-05 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 15:02:00 ts/train.py:232 step:716K smpl:11M ep:59K epch:197.32 loss:0.002 grdn:0.149 lr:9.9e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 15:02:46 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.38 loss:0.002 grdn:0.138 lr:9.9e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 15:03:32 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.43 loss:0.002 grdn:0.139 lr:9.9e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 15:04:17 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.49 loss:0.002 grdn:0.138 lr:9.9e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 15:05:03 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.54 loss:0.002 grdn:0.140 lr:9.9e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 15:05:51 ts/train.py:232 step:717K smpl:11M ep:59K epch:197.60 loss:0.002 grdn:0.138 lr:9.8e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 15:06:42 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.65 loss:0.002 grdn:0.159 lr:9.8e-06 updt_s:0.156 data_s:0.100 +INFO 2025-05-07 15:07:31 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.71 loss:0.001 grdn:0.128 lr:9.8e-06 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 15:08:20 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.76 loss:0.002 grdn:0.127 lr:9.8e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 15:09:07 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.82 loss:0.002 grdn:0.138 lr:9.8e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 15:09:56 ts/train.py:232 step:718K smpl:11M ep:59K epch:197.87 loss:0.002 grdn:0.133 lr:9.7e-06 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 15:10:44 ts/train.py:232 step:719K smpl:11M ep:59K epch:197.93 loss:0.002 grdn:0.152 lr:9.7e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 15:11:32 ts/train.py:232 step:719K smpl:12M ep:59K epch:197.98 loss:0.002 grdn:0.143 lr:9.7e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 15:12:20 ts/train.py:232 step:719K smpl:12M ep:59K epch:198.04 loss:0.002 grdn:0.143 lr:9.7e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 15:13:07 ts/train.py:232 step:719K smpl:12M ep:59K epch:198.09 loss:0.002 grdn:0.150 lr:9.7e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 15:13:55 ts/train.py:232 step:719K smpl:12M ep:59K epch:198.15 loss:0.002 grdn:0.143 lr:9.6e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 15:14:42 ts/train.py:232 step:720K smpl:12M ep:59K epch:198.20 loss:0.002 grdn:0.177 lr:9.6e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 15:15:31 ts/train.py:232 step:720K smpl:12M ep:59K epch:198.26 loss:0.002 grdn:0.134 lr:9.6e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 15:16:19 ts/train.py:232 step:720K smpl:12M ep:59K epch:198.31 loss:0.002 grdn:0.145 lr:9.6e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 15:16:19 ts/train.py:241 Checkpoint policy after step 720000 +INFO 2025-05-07 15:17:09 ts/train.py:232 step:720K smpl:12M ep:60K epch:198.37 loss:0.002 grdn:0.151 lr:9.5e-06 updt_s:0.158 data_s:0.079 +INFO 2025-05-07 15:17:57 ts/train.py:232 step:720K smpl:12M ep:60K epch:198.42 loss:0.002 grdn:0.142 lr:9.5e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 15:18:45 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.48 loss:0.002 grdn:0.134 lr:9.5e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 15:19:33 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.53 loss:0.002 grdn:0.130 lr:9.5e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 15:20:21 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.59 loss:0.001 grdn:0.130 lr:9.5e-06 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 15:21:07 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.64 loss:0.002 grdn:0.132 lr:9.4e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 15:21:54 ts/train.py:232 step:721K smpl:12M ep:60K epch:198.70 loss:0.001 grdn:0.131 lr:9.4e-06 updt_s:0.156 data_s:0.076 +INFO 2025-05-07 15:22:39 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.75 loss:0.002 grdn:0.138 lr:9.4e-06 updt_s:0.156 data_s:0.066 +INFO 2025-05-07 15:23:28 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.81 loss:0.002 grdn:0.143 lr:9.4e-06 updt_s:0.156 data_s:0.090 +INFO 2025-05-07 15:24:14 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.86 loss:0.002 grdn:0.128 lr:9.4e-06 updt_s:0.156 data_s:0.075 +INFO 2025-05-07 15:24:59 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.92 loss:0.002 grdn:0.143 lr:9.3e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 15:25:45 ts/train.py:232 step:722K smpl:12M ep:60K epch:198.97 loss:0.002 grdn:0.166 lr:9.3e-06 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 15:26:30 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.03 loss:0.002 grdn:0.146 lr:9.3e-06 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 15:27:16 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.08 loss:0.002 grdn:0.145 lr:9.3e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 15:28:02 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.14 loss:0.002 grdn:0.139 lr:9.3e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 15:28:47 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.19 loss:0.001 grdn:0.114 lr:9.2e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 15:29:31 ts/train.py:232 step:723K smpl:12M ep:60K epch:199.25 loss:0.002 grdn:0.141 lr:9.2e-06 updt_s:0.156 data_s:0.063 +INFO 2025-05-07 15:30:16 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.30 loss:0.002 grdn:0.134 lr:9.2e-06 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 15:31:01 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.36 loss:0.001 grdn:0.120 lr:9.2e-06 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 15:31:48 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.41 loss:0.002 grdn:0.135 lr:9.2e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 15:32:33 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.47 loss:0.002 grdn:0.140 lr:9.1e-06 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 15:33:17 ts/train.py:232 step:724K smpl:12M ep:60K epch:199.52 loss:0.002 grdn:0.148 lr:9.1e-06 updt_s:0.156 data_s:0.065 +INFO 2025-05-07 15:34:04 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.58 loss:0.002 grdn:0.142 lr:9.1e-06 updt_s:0.156 data_s:0.076 +INFO 2025-05-07 15:34:49 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.64 loss:0.002 grdn:0.137 lr:9.1e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 15:35:36 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.69 loss:0.002 grdn:0.134 lr:9.1e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 15:36:21 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.75 loss:0.002 grdn:0.161 lr:9.0e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 15:37:07 ts/train.py:232 step:725K smpl:12M ep:60K epch:199.80 loss:0.002 grdn:0.131 lr:9.0e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 15:37:54 ts/train.py:232 step:726K smpl:12M ep:60K epch:199.86 loss:0.002 grdn:0.147 lr:9.0e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 15:38:40 ts/train.py:232 step:726K smpl:12M ep:60K epch:199.91 loss:0.002 grdn:0.142 lr:9.0e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 15:39:25 ts/train.py:232 step:726K smpl:12M ep:60K epch:199.97 loss:0.002 grdn:0.152 lr:9.0e-06 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 15:40:10 ts/train.py:232 step:726K smpl:12M ep:60K epch:200.02 loss:0.002 grdn:0.133 lr:8.9e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 15:40:55 ts/train.py:232 step:726K smpl:12M ep:60K epch:200.08 loss:0.001 grdn:0.127 lr:8.9e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 15:41:40 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.13 loss:0.001 grdn:0.130 lr:8.9e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 15:42:25 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.19 loss:0.002 grdn:0.146 lr:8.9e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 15:43:11 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.24 loss:0.002 grdn:0.141 lr:8.9e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 15:43:56 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.30 loss:0.001 grdn:0.125 lr:8.8e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 15:44:41 ts/train.py:232 step:727K smpl:12M ep:60K epch:200.35 loss:0.002 grdn:0.134 lr:8.8e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 15:45:27 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.41 loss:0.002 grdn:0.148 lr:8.8e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 15:46:14 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.46 loss:0.002 grdn:0.137 lr:8.8e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 15:47:04 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.52 loss:0.002 grdn:0.144 lr:8.8e-06 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 15:47:54 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.57 loss:0.002 grdn:0.141 lr:8.7e-06 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 15:48:41 ts/train.py:232 step:728K smpl:12M ep:60K epch:200.63 loss:0.002 grdn:0.137 lr:8.7e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 15:49:30 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.68 loss:0.002 grdn:0.136 lr:8.7e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 15:50:17 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.74 loss:0.002 grdn:0.145 lr:8.7e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 15:51:04 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.79 loss:0.001 grdn:0.121 lr:8.7e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 15:51:51 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.85 loss:0.002 grdn:0.138 lr:8.6e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 15:52:38 ts/train.py:232 step:729K smpl:12M ep:60K epch:200.90 loss:0.002 grdn:0.140 lr:8.6e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 15:53:25 ts/train.py:232 step:730K smpl:12M ep:60K epch:200.96 loss:0.002 grdn:0.134 lr:8.6e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 15:54:11 ts/train.py:232 step:730K smpl:12M ep:60K epch:201.01 loss:0.001 grdn:0.125 lr:8.6e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 15:54:58 ts/train.py:232 step:730K smpl:12M ep:60K epch:201.07 loss:0.002 grdn:0.136 lr:8.6e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 15:55:45 ts/train.py:232 step:730K smpl:12M ep:60K epch:201.12 loss:0.002 grdn:0.151 lr:8.5e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 15:56:33 ts/train.py:232 step:730K smpl:12M ep:60K epch:201.18 loss:0.002 grdn:0.146 lr:8.5e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 15:57:20 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.23 loss:0.002 grdn:0.154 lr:8.5e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 15:58:08 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.29 loss:0.001 grdn:0.136 lr:8.5e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 15:58:56 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.34 loss:0.002 grdn:0.141 lr:8.5e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 15:59:43 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.40 loss:0.002 grdn:0.134 lr:8.5e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 16:00:32 ts/train.py:232 step:731K smpl:12M ep:60K epch:201.45 loss:0.002 grdn:0.133 lr:8.4e-06 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 16:01:21 ts/train.py:232 step:732K smpl:12M ep:60K epch:201.51 loss:0.002 grdn:0.139 lr:8.4e-06 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 16:02:06 ts/train.py:232 step:732K smpl:12M ep:60K epch:201.56 loss:0.002 grdn:0.134 lr:8.4e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 16:02:51 ts/train.py:232 step:732K smpl:12M ep:60K epch:201.62 loss:0.002 grdn:0.140 lr:8.4e-06 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 16:03:37 ts/train.py:232 step:732K smpl:12M ep:61K epch:201.67 loss:0.002 grdn:0.141 lr:8.4e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 16:04:22 ts/train.py:232 step:732K smpl:12M ep:61K epch:201.73 loss:0.002 grdn:0.145 lr:8.3e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 16:05:07 ts/train.py:232 step:733K smpl:12M ep:61K epch:201.78 loss:0.002 grdn:0.144 lr:8.3e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 16:05:52 ts/train.py:232 step:733K smpl:12M ep:61K epch:201.84 loss:0.001 grdn:0.128 lr:8.3e-06 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 16:06:37 ts/train.py:232 step:733K smpl:12M ep:61K epch:201.89 loss:0.002 grdn:0.139 lr:8.3e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 16:07:22 ts/train.py:232 step:733K smpl:12M ep:61K epch:201.95 loss:0.002 grdn:0.139 lr:8.3e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 16:08:06 ts/train.py:232 step:733K smpl:12M ep:61K epch:202.00 loss:0.002 grdn:0.134 lr:8.2e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 16:08:51 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.06 loss:0.002 grdn:0.140 lr:8.2e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 16:09:36 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.11 loss:0.002 grdn:0.171 lr:8.2e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 16:10:21 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.17 loss:0.002 grdn:0.147 lr:8.2e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 16:11:08 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.22 loss:0.002 grdn:0.152 lr:8.2e-06 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 16:11:57 ts/train.py:232 step:734K smpl:12M ep:61K epch:202.28 loss:0.002 grdn:0.145 lr:8.1e-06 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 16:12:43 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.33 loss:0.002 grdn:0.139 lr:8.1e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 16:13:27 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.39 loss:0.002 grdn:0.145 lr:8.1e-06 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 16:14:14 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.44 loss:0.001 grdn:0.129 lr:8.1e-06 updt_s:0.156 data_s:0.076 +INFO 2025-05-07 16:15:04 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.50 loss:0.002 grdn:0.138 lr:8.1e-06 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 16:15:51 ts/train.py:232 step:735K smpl:12M ep:61K epch:202.55 loss:0.002 grdn:0.133 lr:8.0e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 16:16:39 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.61 loss:0.002 grdn:0.128 lr:8.0e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 16:17:27 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.66 loss:0.001 grdn:0.115 lr:8.0e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 16:18:14 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.72 loss:0.002 grdn:0.133 lr:8.0e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 16:19:01 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.78 loss:0.002 grdn:0.137 lr:8.0e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 16:19:47 ts/train.py:232 step:736K smpl:12M ep:61K epch:202.83 loss:0.002 grdn:0.157 lr:8.0e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 16:20:34 ts/train.py:232 step:737K smpl:12M ep:61K epch:202.89 loss:0.002 grdn:0.138 lr:7.9e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 16:21:21 ts/train.py:232 step:737K smpl:12M ep:61K epch:202.94 loss:0.002 grdn:0.134 lr:7.9e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 16:22:08 ts/train.py:232 step:737K smpl:12M ep:61K epch:203.00 loss:0.002 grdn:0.135 lr:7.9e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 16:22:56 ts/train.py:232 step:737K smpl:12M ep:61K epch:203.05 loss:0.002 grdn:0.132 lr:7.9e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 16:23:43 ts/train.py:232 step:737K smpl:12M ep:61K epch:203.11 loss:0.001 grdn:0.127 lr:7.9e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 16:24:30 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.16 loss:0.002 grdn:0.139 lr:7.8e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 16:25:17 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.22 loss:0.001 grdn:0.125 lr:7.8e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 16:26:05 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.27 loss:0.001 grdn:0.129 lr:7.8e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 16:26:51 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.33 loss:0.002 grdn:0.148 lr:7.8e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 16:27:37 ts/train.py:232 step:738K smpl:12M ep:61K epch:203.38 loss:0.002 grdn:0.145 lr:7.8e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 16:29:51 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.44 loss:0.002 grdn:0.128 lr:7.7e-06 updt_s:0.156 data_s:0.513 +INFO 2025-05-07 16:30:36 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.49 loss:0.002 grdn:0.150 lr:7.7e-06 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 16:31:21 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.55 loss:0.002 grdn:0.136 lr:7.7e-06 updt_s:0.156 data_s:0.065 +INFO 2025-05-07 16:32:08 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.60 loss:0.002 grdn:0.143 lr:7.7e-06 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 16:32:52 ts/train.py:232 step:739K smpl:12M ep:61K epch:203.66 loss:0.001 grdn:0.135 lr:7.7e-06 updt_s:0.156 data_s:0.065 +INFO 2025-05-07 16:33:36 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.71 loss:0.001 grdn:0.129 lr:7.7e-06 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 16:34:22 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.77 loss:0.002 grdn:0.155 lr:7.6e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 16:35:06 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.82 loss:0.002 grdn:0.130 lr:7.6e-06 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 16:35:51 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.88 loss:0.002 grdn:0.148 lr:7.6e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 16:36:36 ts/train.py:232 step:740K smpl:12M ep:61K epch:203.93 loss:0.002 grdn:0.149 lr:7.6e-06 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 16:37:21 ts/train.py:232 step:741K smpl:12M ep:61K epch:203.99 loss:0.002 grdn:0.137 lr:7.6e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 16:38:06 ts/train.py:232 step:741K smpl:12M ep:61K epch:204.04 loss:0.002 grdn:0.141 lr:7.5e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 16:38:51 ts/train.py:232 step:741K smpl:12M ep:61K epch:204.10 loss:0.001 grdn:0.127 lr:7.5e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 16:39:36 ts/train.py:232 step:741K smpl:12M ep:61K epch:204.15 loss:0.002 grdn:0.133 lr:7.5e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 16:40:21 ts/train.py:232 step:741K smpl:12M ep:61K epch:204.21 loss:0.001 grdn:0.127 lr:7.5e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 16:41:06 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.26 loss:0.002 grdn:0.139 lr:7.5e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 16:41:51 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.32 loss:0.001 grdn:0.133 lr:7.4e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 16:42:37 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.37 loss:0.001 grdn:0.115 lr:7.4e-06 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 16:43:29 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.43 loss:0.002 grdn:0.130 lr:7.4e-06 updt_s:0.157 data_s:0.102 +INFO 2025-05-07 16:44:17 ts/train.py:232 step:742K smpl:12M ep:61K epch:204.48 loss:0.002 grdn:0.151 lr:7.4e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 16:45:07 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.54 loss:0.001 grdn:0.128 lr:7.4e-06 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 16:45:55 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.59 loss:0.002 grdn:0.140 lr:7.4e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 16:46:42 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.65 loss:0.002 grdn:0.136 lr:7.3e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 16:47:31 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.70 loss:0.002 grdn:0.133 lr:7.3e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 16:48:20 ts/train.py:232 step:743K smpl:12M ep:61K epch:204.76 loss:0.001 grdn:0.134 lr:7.3e-06 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 16:49:07 ts/train.py:232 step:744K smpl:12M ep:61K epch:204.81 loss:0.001 grdn:0.128 lr:7.3e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 16:49:56 ts/train.py:232 step:744K smpl:12M ep:61K epch:204.87 loss:0.001 grdn:0.139 lr:7.3e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 16:50:44 ts/train.py:232 step:744K smpl:12M ep:61K epch:204.92 loss:0.001 grdn:0.131 lr:7.2e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 16:51:35 ts/train.py:232 step:744K smpl:12M ep:61K epch:204.98 loss:0.002 grdn:0.144 lr:7.2e-06 updt_s:0.158 data_s:0.094 +INFO 2025-05-07 16:52:23 ts/train.py:232 step:744K smpl:12M ep:62K epch:205.03 loss:0.001 grdn:0.124 lr:7.2e-06 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 16:53:11 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.09 loss:0.001 grdn:0.134 lr:7.2e-06 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 16:53:59 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.14 loss:0.002 grdn:0.138 lr:7.2e-06 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 16:54:47 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.20 loss:0.002 grdn:0.144 lr:7.2e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 16:55:34 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.25 loss:0.001 grdn:0.128 lr:7.1e-06 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 16:56:23 ts/train.py:232 step:745K smpl:12M ep:62K epch:205.31 loss:0.001 grdn:0.134 lr:7.1e-06 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 16:57:12 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.36 loss:0.002 grdn:0.146 lr:7.1e-06 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 16:58:00 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.42 loss:0.001 grdn:0.126 lr:7.1e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 16:58:47 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.47 loss:0.002 grdn:0.142 lr:7.1e-06 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 16:59:34 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.53 loss:0.002 grdn:0.147 lr:7.1e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 17:00:22 ts/train.py:232 step:746K smpl:12M ep:62K epch:205.58 loss:0.001 grdn:0.125 lr:7.0e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 17:01:10 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.64 loss:0.001 grdn:0.117 lr:7.0e-06 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 17:01:57 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.69 loss:0.002 grdn:0.137 lr:7.0e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 17:02:44 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.75 loss:0.001 grdn:0.123 lr:7.0e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 17:03:32 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.80 loss:0.001 grdn:0.123 lr:7.0e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 17:04:20 ts/train.py:232 step:747K smpl:12M ep:62K epch:205.86 loss:0.001 grdn:0.136 lr:6.9e-06 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 17:05:08 ts/train.py:232 step:748K smpl:12M ep:62K epch:205.91 loss:0.001 grdn:0.137 lr:6.9e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 17:05:56 ts/train.py:232 step:748K smpl:12M ep:62K epch:205.97 loss:0.001 grdn:0.117 lr:6.9e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 17:06:44 ts/train.py:232 step:748K smpl:12M ep:62K epch:206.03 loss:0.001 grdn:0.128 lr:6.9e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 17:07:32 ts/train.py:232 step:748K smpl:12M ep:62K epch:206.08 loss:0.001 grdn:0.131 lr:6.9e-06 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 17:08:20 ts/train.py:232 step:748K smpl:12M ep:62K epch:206.14 loss:0.002 grdn:0.145 lr:6.9e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 17:09:07 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.19 loss:0.002 grdn:0.141 lr:6.8e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 17:09:55 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.25 loss:0.002 grdn:0.145 lr:6.8e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 17:10:43 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.30 loss:0.002 grdn:0.138 lr:6.8e-06 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 17:11:33 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.36 loss:0.001 grdn:0.122 lr:6.8e-06 updt_s:0.158 data_s:0.094 +INFO 2025-05-07 17:12:20 ts/train.py:232 step:749K smpl:12M ep:62K epch:206.41 loss:0.001 grdn:0.125 lr:6.8e-06 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 17:13:09 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.47 loss:0.001 grdn:0.130 lr:6.7e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 17:13:56 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.52 loss:0.002 grdn:0.139 lr:6.7e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 17:14:46 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.58 loss:0.002 grdn:0.128 lr:6.7e-06 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 17:14:46 ts/train.py:241 Checkpoint policy after step 750000 +INFO 2025-05-07 17:15:37 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.63 loss:0.002 grdn:0.139 lr:6.7e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 17:16:25 ts/train.py:232 step:750K smpl:12M ep:62K epch:206.69 loss:0.002 grdn:0.145 lr:6.7e-06 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 17:17:13 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.74 loss:0.002 grdn:0.139 lr:6.7e-06 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 17:18:01 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.80 loss:0.002 grdn:0.150 lr:6.6e-06 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 17:18:49 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.85 loss:0.001 grdn:0.116 lr:6.6e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 17:19:36 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.91 loss:0.001 grdn:0.126 lr:6.6e-06 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 17:20:24 ts/train.py:232 step:751K smpl:12M ep:62K epch:206.96 loss:0.001 grdn:0.124 lr:6.6e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 17:21:12 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.02 loss:0.001 grdn:0.124 lr:6.6e-06 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 17:22:00 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.07 loss:0.001 grdn:0.133 lr:6.6e-06 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 17:22:47 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.13 loss:0.001 grdn:0.129 lr:6.5e-06 updt_s:0.156 data_s:0.080 +INFO 2025-05-07 17:23:34 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.18 loss:0.002 grdn:0.142 lr:6.5e-06 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 17:24:22 ts/train.py:232 step:752K smpl:12M ep:62K epch:207.24 loss:0.002 grdn:0.139 lr:6.5e-06 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 17:25:12 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.29 loss:0.001 grdn:0.107 lr:6.5e-06 updt_s:0.156 data_s:0.097 +INFO 2025-05-07 17:26:01 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.35 loss:0.002 grdn:0.139 lr:6.5e-06 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 17:26:49 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.40 loss:0.001 grdn:0.131 lr:6.5e-06 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 17:27:39 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.46 loss:0.002 grdn:0.146 lr:6.4e-06 updt_s:0.156 data_s:0.092 +INFO 2025-05-07 17:28:28 ts/train.py:232 step:753K smpl:12M ep:62K epch:207.51 loss:0.002 grdn:0.130 lr:6.4e-06 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 17:29:18 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.57 loss:0.001 grdn:0.130 lr:6.4e-06 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 17:30:07 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.62 loss:0.001 grdn:0.122 lr:6.4e-06 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 17:30:55 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.68 loss:0.001 grdn:0.123 lr:6.4e-06 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 17:31:45 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.73 loss:0.001 grdn:0.130 lr:6.4e-06 updt_s:0.156 data_s:0.093 +INFO 2025-05-07 17:32:35 ts/train.py:232 step:754K smpl:12M ep:62K epch:207.79 loss:0.001 grdn:0.127 lr:6.3e-06 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 17:33:24 ts/train.py:232 step:755K smpl:12M ep:62K epch:207.84 loss:0.002 grdn:0.141 lr:6.3e-06 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 17:34:13 ts/train.py:232 step:755K smpl:12M ep:62K epch:207.90 loss:0.001 grdn:0.113 lr:6.3e-06 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 17:35:02 ts/train.py:232 step:755K smpl:12M ep:62K epch:207.95 loss:0.001 grdn:0.136 lr:6.3e-06 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 17:35:51 ts/train.py:232 step:755K smpl:12M ep:62K epch:208.01 loss:0.001 grdn:0.127 lr:6.3e-06 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 17:36:40 ts/train.py:232 step:755K smpl:12M ep:62K epch:208.06 loss:0.001 grdn:0.131 lr:6.3e-06 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 17:37:29 ts/train.py:232 step:756K smpl:12M ep:62K epch:208.12 loss:0.002 grdn:0.140 lr:6.2e-06 updt_s:0.156 data_s:0.091 +INFO 2025-05-07 17:38:18 ts/train.py:232 step:756K smpl:12M ep:62K epch:208.17 loss:0.001 grdn:0.124 lr:6.2e-06 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 17:39:08 ts/train.py:232 step:756K smpl:12M ep:62K epch:208.23 loss:0.001 grdn:0.130 lr:6.2e-06 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 17:39:59 ts/train.py:232 step:756K smpl:12M ep:62K epch:208.28 loss:0.001 grdn:0.124 lr:6.2e-06 updt_s:0.157 data_s:0.097 +INFO 2025-05-07 17:40:48 ts/train.py:232 step:756K smpl:12M ep:63K epch:208.34 loss:0.001 grdn:0.117 lr:6.2e-06 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 17:41:35 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.39 loss:0.001 grdn:0.126 lr:6.1e-06 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 17:42:22 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.45 loss:0.002 grdn:0.144 lr:6.1e-06 updt_s:0.156 data_s:0.075 +INFO 2025-05-07 17:43:09 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.50 loss:0.001 grdn:0.132 lr:6.1e-06 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 17:43:56 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.56 loss:0.001 grdn:0.128 lr:6.1e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 17:44:44 ts/train.py:232 step:757K smpl:12M ep:63K epch:208.61 loss:0.001 grdn:0.122 lr:6.1e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 17:45:31 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.67 loss:0.001 grdn:0.119 lr:6.1e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 17:46:18 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.72 loss:0.001 grdn:0.115 lr:6.0e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 17:47:06 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.78 loss:0.001 grdn:0.133 lr:6.0e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 17:47:54 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.83 loss:0.001 grdn:0.131 lr:6.0e-06 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 17:48:43 ts/train.py:232 step:758K smpl:12M ep:63K epch:208.89 loss:0.001 grdn:0.123 lr:6.0e-06 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 17:49:32 ts/train.py:232 step:759K smpl:12M ep:63K epch:208.94 loss:0.001 grdn:0.125 lr:6.0e-06 updt_s:0.158 data_s:0.088 +INFO 2025-05-07 17:50:20 ts/train.py:232 step:759K smpl:12M ep:63K epch:209.00 loss:0.001 grdn:0.130 lr:6.0e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 17:51:08 ts/train.py:232 step:759K smpl:12M ep:63K epch:209.05 loss:0.002 grdn:0.136 lr:5.9e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 17:51:57 ts/train.py:232 step:759K smpl:12M ep:63K epch:209.11 loss:0.002 grdn:0.143 lr:5.9e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 17:52:46 ts/train.py:232 step:759K smpl:12M ep:63K epch:209.17 loss:0.002 grdn:0.142 lr:5.9e-06 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 17:53:35 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.22 loss:0.001 grdn:0.124 lr:5.9e-06 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 17:54:22 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.28 loss:0.001 grdn:0.134 lr:5.9e-06 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 17:55:11 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.33 loss:0.001 grdn:0.139 lr:5.9e-06 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 17:55:59 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.39 loss:0.001 grdn:0.136 lr:5.9e-06 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 17:56:48 ts/train.py:232 step:760K smpl:12M ep:63K epch:209.44 loss:0.001 grdn:0.121 lr:5.8e-06 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 17:57:36 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.50 loss:0.002 grdn:0.146 lr:5.8e-06 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 17:58:23 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.55 loss:0.001 grdn:0.129 lr:5.8e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 17:59:11 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.61 loss:0.001 grdn:0.127 lr:5.8e-06 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 18:00:00 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.66 loss:0.001 grdn:0.124 lr:5.8e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 18:00:48 ts/train.py:232 step:761K smpl:12M ep:63K epch:209.72 loss:0.001 grdn:0.128 lr:5.8e-06 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 18:01:36 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.77 loss:0.001 grdn:0.130 lr:5.7e-06 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 18:02:25 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.83 loss:0.001 grdn:0.106 lr:5.7e-06 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 18:03:14 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.88 loss:0.001 grdn:0.132 lr:5.7e-06 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 18:04:02 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.94 loss:0.001 grdn:0.130 lr:5.7e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 18:04:50 ts/train.py:232 step:762K smpl:12M ep:63K epch:209.99 loss:0.001 grdn:0.136 lr:5.7e-06 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 18:05:38 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.05 loss:0.001 grdn:0.138 lr:5.7e-06 updt_s:0.156 data_s:0.081 +INFO 2025-05-07 18:06:27 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.10 loss:0.001 grdn:0.130 lr:5.6e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 18:07:14 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.16 loss:0.002 grdn:0.131 lr:5.6e-06 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 18:08:05 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.21 loss:0.001 grdn:0.126 lr:5.6e-06 updt_s:0.157 data_s:0.096 +INFO 2025-05-07 18:08:53 ts/train.py:232 step:763K smpl:12M ep:63K epch:210.27 loss:0.001 grdn:0.135 lr:5.6e-06 updt_s:0.158 data_s:0.083 +INFO 2025-05-07 18:09:43 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.32 loss:0.001 grdn:0.129 lr:5.6e-06 updt_s:0.157 data_s:0.092 +INFO 2025-05-07 18:10:32 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.38 loss:0.001 grdn:0.129 lr:5.6e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 18:11:21 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.43 loss:0.001 grdn:0.120 lr:5.5e-06 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 18:12:09 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.49 loss:0.001 grdn:0.132 lr:5.5e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 18:12:58 ts/train.py:232 step:764K smpl:12M ep:63K epch:210.54 loss:0.001 grdn:0.129 lr:5.5e-06 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 18:13:48 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.60 loss:0.002 grdn:0.145 lr:5.5e-06 updt_s:0.157 data_s:0.094 +INFO 2025-05-07 18:14:37 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.65 loss:0.001 grdn:0.123 lr:5.5e-06 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 18:15:27 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.71 loss:0.001 grdn:0.119 lr:5.5e-06 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 18:16:16 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.76 loss:0.001 grdn:0.114 lr:5.4e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 18:17:06 ts/train.py:232 step:765K smpl:12M ep:63K epch:210.82 loss:0.001 grdn:0.117 lr:5.4e-06 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 18:17:55 ts/train.py:232 step:766K smpl:12M ep:63K epch:210.87 loss:0.002 grdn:0.136 lr:5.4e-06 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 18:18:44 ts/train.py:232 step:766K smpl:12M ep:63K epch:210.93 loss:0.001 grdn:0.121 lr:5.4e-06 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 18:19:32 ts/train.py:232 step:766K smpl:12M ep:63K epch:210.98 loss:0.001 grdn:0.136 lr:5.4e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 18:20:21 ts/train.py:232 step:766K smpl:12M ep:63K epch:211.04 loss:0.001 grdn:0.124 lr:5.4e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 18:21:10 ts/train.py:232 step:766K smpl:12M ep:63K epch:211.09 loss:0.001 grdn:0.131 lr:5.4e-06 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 18:21:59 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.15 loss:0.002 grdn:0.149 lr:5.3e-06 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 18:22:43 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.20 loss:0.001 grdn:0.129 lr:5.3e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 18:23:28 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.26 loss:0.001 grdn:0.135 lr:5.3e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 18:24:11 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.31 loss:0.001 grdn:0.128 lr:5.3e-06 updt_s:0.157 data_s:0.059 +INFO 2025-05-07 18:24:55 ts/train.py:232 step:767K smpl:12M ep:63K epch:211.37 loss:0.001 grdn:0.125 lr:5.3e-06 updt_s:0.157 data_s:0.061 +INFO 2025-05-07 18:25:38 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.42 loss:0.001 grdn:0.119 lr:5.3e-06 updt_s:0.157 data_s:0.060 +INFO 2025-05-07 18:26:22 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.48 loss:0.001 grdn:0.120 lr:5.2e-06 updt_s:0.157 data_s:0.060 +INFO 2025-05-07 18:27:06 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.53 loss:0.001 grdn:0.136 lr:5.2e-06 updt_s:0.157 data_s:0.061 +INFO 2025-05-07 18:27:49 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.59 loss:0.001 grdn:0.123 lr:5.2e-06 updt_s:0.157 data_s:0.059 +INFO 2025-05-07 18:28:33 ts/train.py:232 step:768K smpl:12M ep:63K epch:211.64 loss:0.001 grdn:0.133 lr:5.2e-06 updt_s:0.157 data_s:0.059 +INFO 2025-05-07 18:29:16 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.70 loss:0.001 grdn:0.123 lr:5.2e-06 updt_s:0.157 data_s:0.061 +INFO 2025-05-07 18:30:00 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.75 loss:0.001 grdn:0.119 lr:5.2e-06 updt_s:0.157 data_s:0.058 +INFO 2025-05-07 18:30:43 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.81 loss:0.001 grdn:0.138 lr:5.2e-06 updt_s:0.157 data_s:0.059 +INFO 2025-05-07 18:31:27 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.86 loss:0.001 grdn:0.138 lr:5.1e-06 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 18:32:11 ts/train.py:232 step:769K smpl:12M ep:64K epch:211.92 loss:0.001 grdn:0.114 lr:5.1e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 18:32:56 ts/train.py:232 step:770K smpl:12M ep:64K epch:211.97 loss:0.001 grdn:0.135 lr:5.1e-06 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 18:33:39 ts/train.py:232 step:770K smpl:12M ep:64K epch:212.03 loss:0.001 grdn:0.131 lr:5.1e-06 updt_s:0.157 data_s:0.060 +INFO 2025-05-07 18:34:23 ts/train.py:232 step:770K smpl:12M ep:64K epch:212.08 loss:0.001 grdn:0.141 lr:5.1e-06 updt_s:0.157 data_s:0.060 +INFO 2025-05-07 18:35:14 ts/train.py:232 step:770K smpl:12M ep:64K epch:212.14 loss:0.001 grdn:0.132 lr:5.1e-06 updt_s:0.157 data_s:0.098 +INFO 2025-05-07 18:36:02 ts/train.py:232 step:770K smpl:12M ep:64K epch:212.19 loss:0.001 grdn:0.128 lr:5.0e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 18:36:51 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.25 loss:0.001 grdn:0.125 lr:5.0e-06 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 18:37:39 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.31 loss:0.001 grdn:0.127 lr:5.0e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 18:38:26 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.36 loss:0.001 grdn:0.137 lr:5.0e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 18:39:15 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.42 loss:0.001 grdn:0.116 lr:5.0e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 18:40:04 ts/train.py:232 step:771K smpl:12M ep:64K epch:212.47 loss:0.001 grdn:0.125 lr:5.0e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 18:40:52 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.53 loss:0.001 grdn:0.129 lr:5.0e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 18:41:40 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.58 loss:0.001 grdn:0.122 lr:4.9e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 18:42:28 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.64 loss:0.001 grdn:0.124 lr:4.9e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 18:43:16 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.69 loss:0.001 grdn:0.120 lr:4.9e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 18:44:03 ts/train.py:232 step:772K smpl:12M ep:64K epch:212.75 loss:0.001 grdn:0.128 lr:4.9e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 18:44:51 ts/train.py:232 step:773K smpl:12M ep:64K epch:212.80 loss:0.001 grdn:0.113 lr:4.9e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 18:45:39 ts/train.py:232 step:773K smpl:12M ep:64K epch:212.86 loss:0.002 grdn:0.154 lr:4.9e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 18:46:26 ts/train.py:232 step:773K smpl:12M ep:64K epch:212.91 loss:0.001 grdn:0.116 lr:4.8e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 18:47:15 ts/train.py:232 step:773K smpl:12M ep:64K epch:212.97 loss:0.001 grdn:0.130 lr:4.8e-06 updt_s:0.158 data_s:0.088 +INFO 2025-05-07 18:48:05 ts/train.py:232 step:773K smpl:12M ep:64K epch:213.02 loss:0.001 grdn:0.135 lr:4.8e-06 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 18:48:54 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.08 loss:0.001 grdn:0.127 lr:4.8e-06 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 18:49:39 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.13 loss:0.001 grdn:0.132 lr:4.8e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 18:50:25 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.19 loss:0.001 grdn:0.130 lr:4.8e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 18:51:11 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.24 loss:0.001 grdn:0.122 lr:4.8e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 18:51:56 ts/train.py:232 step:774K smpl:12M ep:64K epch:213.30 loss:0.001 grdn:0.120 lr:4.7e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 18:52:42 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.35 loss:0.001 grdn:0.131 lr:4.7e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 18:53:27 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.41 loss:0.001 grdn:0.133 lr:4.7e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 18:54:12 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.46 loss:0.001 grdn:0.138 lr:4.7e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 18:54:57 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.52 loss:0.001 grdn:0.138 lr:4.7e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 18:55:43 ts/train.py:232 step:775K smpl:12M ep:64K epch:213.57 loss:0.001 grdn:0.114 lr:4.7e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 18:56:29 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.63 loss:0.001 grdn:0.131 lr:4.7e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 18:57:16 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.68 loss:0.001 grdn:0.127 lr:4.6e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 18:58:03 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.74 loss:0.001 grdn:0.128 lr:4.6e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 18:58:49 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.79 loss:0.001 grdn:0.124 lr:4.6e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 18:59:35 ts/train.py:232 step:776K smpl:12M ep:64K epch:213.85 loss:0.001 grdn:0.123 lr:4.6e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 19:00:23 ts/train.py:232 step:777K smpl:12M ep:64K epch:213.90 loss:0.001 grdn:0.109 lr:4.6e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 19:01:10 ts/train.py:232 step:777K smpl:12M ep:64K epch:213.96 loss:0.001 grdn:0.129 lr:4.6e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 19:01:56 ts/train.py:232 step:777K smpl:12M ep:64K epch:214.01 loss:0.001 grdn:0.126 lr:4.6e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 19:02:47 ts/train.py:232 step:777K smpl:12M ep:64K epch:214.07 loss:0.001 grdn:0.122 lr:4.5e-06 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 19:03:36 ts/train.py:232 step:777K smpl:12M ep:64K epch:214.12 loss:0.001 grdn:0.128 lr:4.5e-06 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 19:04:25 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.18 loss:0.001 grdn:0.140 lr:4.5e-06 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 19:05:14 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.23 loss:0.001 grdn:0.118 lr:4.5e-06 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 19:06:02 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.29 loss:0.001 grdn:0.124 lr:4.5e-06 updt_s:0.158 data_s:0.085 +INFO 2025-05-07 19:06:50 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.34 loss:0.001 grdn:0.119 lr:4.5e-06 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 19:07:39 ts/train.py:232 step:778K smpl:12M ep:64K epch:214.40 loss:0.001 grdn:0.120 lr:4.4e-06 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 19:08:28 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.45 loss:0.001 grdn:0.140 lr:4.4e-06 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 19:09:18 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.51 loss:0.001 grdn:0.113 lr:4.4e-06 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 19:10:07 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.56 loss:0.001 grdn:0.134 lr:4.4e-06 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 19:10:57 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.62 loss:0.001 grdn:0.137 lr:4.4e-06 updt_s:0.157 data_s:0.094 +INFO 2025-05-07 19:11:47 ts/train.py:232 step:779K smpl:12M ep:64K epch:214.67 loss:0.002 grdn:0.148 lr:4.4e-06 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 19:12:38 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.73 loss:0.001 grdn:0.131 lr:4.4e-06 updt_s:0.157 data_s:0.095 +INFO 2025-05-07 19:13:28 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.78 loss:0.001 grdn:0.137 lr:4.3e-06 updt_s:0.157 data_s:0.093 +INFO 2025-05-07 19:14:18 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.84 loss:0.001 grdn:0.118 lr:4.3e-06 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 19:14:18 ts/train.py:241 Checkpoint policy after step 780000 +INFO 2025-05-07 19:15:10 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.89 loss:0.001 grdn:0.116 lr:4.3e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 19:15:59 ts/train.py:232 step:780K smpl:12M ep:64K epch:214.95 loss:0.001 grdn:0.121 lr:4.3e-06 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 19:16:49 ts/train.py:232 step:781K smpl:12M ep:65K epch:215.00 loss:0.002 grdn:0.137 lr:4.3e-06 updt_s:0.156 data_s:0.094 +INFO 2025-05-07 19:17:36 ts/train.py:232 step:781K smpl:12M ep:65K epch:215.06 loss:0.001 grdn:0.121 lr:4.3e-06 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 19:18:23 ts/train.py:232 step:781K smpl:12M ep:65K epch:215.11 loss:0.001 grdn:0.140 lr:4.3e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 19:19:10 ts/train.py:232 step:781K smpl:12M ep:65K epch:215.17 loss:0.001 grdn:0.126 lr:4.2e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 19:19:56 ts/train.py:232 step:781K smpl:13M ep:65K epch:215.22 loss:0.001 grdn:0.111 lr:4.2e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 19:20:43 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.28 loss:0.001 grdn:0.142 lr:4.2e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 19:21:29 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.33 loss:0.001 grdn:0.124 lr:4.2e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 19:22:15 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.39 loss:0.001 grdn:0.129 lr:4.2e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 19:23:02 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.44 loss:0.001 grdn:0.121 lr:4.2e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 19:23:48 ts/train.py:232 step:782K smpl:13M ep:65K epch:215.50 loss:0.001 grdn:0.125 lr:4.2e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 19:24:34 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.56 loss:0.001 grdn:0.124 lr:4.2e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 19:25:21 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.61 loss:0.001 grdn:0.131 lr:4.1e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 19:26:08 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.67 loss:0.001 grdn:0.124 lr:4.1e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 19:26:56 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.72 loss:0.001 grdn:0.121 lr:4.1e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 19:27:43 ts/train.py:232 step:783K smpl:13M ep:65K epch:215.78 loss:0.001 grdn:0.114 lr:4.1e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 19:28:30 ts/train.py:232 step:784K smpl:13M ep:65K epch:215.83 loss:0.001 grdn:0.138 lr:4.1e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 19:29:16 ts/train.py:232 step:784K smpl:13M ep:65K epch:215.89 loss:0.001 grdn:0.120 lr:4.1e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 19:30:03 ts/train.py:232 step:784K smpl:13M ep:65K epch:215.94 loss:0.001 grdn:0.128 lr:4.1e-06 updt_s:0.156 data_s:0.077 +INFO 2025-05-07 19:30:50 ts/train.py:232 step:784K smpl:13M ep:65K epch:216.00 loss:0.001 grdn:0.119 lr:4.0e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 19:31:34 ts/train.py:232 step:784K smpl:13M ep:65K epch:216.05 loss:0.001 grdn:0.140 lr:4.0e-06 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 19:32:19 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.11 loss:0.001 grdn:0.123 lr:4.0e-06 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 19:33:06 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.16 loss:0.001 grdn:0.138 lr:4.0e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 19:33:50 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.22 loss:0.001 grdn:0.136 lr:4.0e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 19:34:35 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.27 loss:0.001 grdn:0.128 lr:4.0e-06 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 19:35:19 ts/train.py:232 step:785K smpl:13M ep:65K epch:216.33 loss:0.001 grdn:0.144 lr:4.0e-06 updt_s:0.156 data_s:0.065 +INFO 2025-05-07 19:36:03 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.38 loss:0.001 grdn:0.121 lr:3.9e-06 updt_s:0.156 data_s:0.064 +INFO 2025-05-07 19:36:47 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.44 loss:0.001 grdn:0.119 lr:3.9e-06 updt_s:0.156 data_s:0.063 +INFO 2025-05-07 19:37:32 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.49 loss:0.001 grdn:0.140 lr:3.9e-06 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 19:38:16 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.55 loss:0.001 grdn:0.117 lr:3.9e-06 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 19:39:01 ts/train.py:232 step:786K smpl:13M ep:65K epch:216.60 loss:0.001 grdn:0.120 lr:3.9e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 19:39:45 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.66 loss:0.001 grdn:0.127 lr:3.9e-06 updt_s:0.157 data_s:0.061 +INFO 2025-05-07 19:40:30 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.71 loss:0.001 grdn:0.132 lr:3.9e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 19:41:17 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.77 loss:0.001 grdn:0.133 lr:3.9e-06 updt_s:0.156 data_s:0.079 +INFO 2025-05-07 19:42:02 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.82 loss:0.001 grdn:0.136 lr:3.8e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 19:42:48 ts/train.py:232 step:787K smpl:13M ep:65K epch:216.88 loss:0.001 grdn:0.109 lr:3.8e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 19:43:37 ts/train.py:232 step:788K smpl:13M ep:65K epch:216.93 loss:0.001 grdn:0.127 lr:3.8e-06 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 19:44:23 ts/train.py:232 step:788K smpl:13M ep:65K epch:216.99 loss:0.001 grdn:0.124 lr:3.8e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 19:45:09 ts/train.py:232 step:788K smpl:13M ep:65K epch:217.04 loss:0.001 grdn:0.122 lr:3.8e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 19:45:55 ts/train.py:232 step:788K smpl:13M ep:65K epch:217.10 loss:0.001 grdn:0.119 lr:3.8e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 19:46:41 ts/train.py:232 step:788K smpl:13M ep:65K epch:217.15 loss:0.001 grdn:0.136 lr:3.8e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 19:47:27 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.21 loss:0.001 grdn:0.133 lr:3.7e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 19:48:15 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.26 loss:0.001 grdn:0.115 lr:3.7e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 19:49:01 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.32 loss:0.001 grdn:0.114 lr:3.7e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 19:49:47 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.37 loss:0.001 grdn:0.120 lr:3.7e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 19:50:33 ts/train.py:232 step:789K smpl:13M ep:65K epch:217.43 loss:0.001 grdn:0.108 lr:3.7e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 19:51:20 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.48 loss:0.001 grdn:0.115 lr:3.7e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 19:52:07 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.54 loss:0.001 grdn:0.129 lr:3.7e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 19:52:54 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.59 loss:0.001 grdn:0.134 lr:3.7e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 19:53:40 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.65 loss:0.001 grdn:0.112 lr:3.6e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 19:54:26 ts/train.py:232 step:790K smpl:13M ep:65K epch:217.70 loss:0.001 grdn:0.122 lr:3.6e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 19:55:12 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.76 loss:0.001 grdn:0.117 lr:3.6e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 19:55:59 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.81 loss:0.001 grdn:0.103 lr:3.6e-06 updt_s:0.158 data_s:0.073 +INFO 2025-05-07 19:56:45 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.87 loss:0.001 grdn:0.136 lr:3.6e-06 updt_s:0.156 data_s:0.073 +INFO 2025-05-07 19:57:33 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.92 loss:0.001 grdn:0.119 lr:3.6e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 19:58:20 ts/train.py:232 step:791K smpl:13M ep:65K epch:217.98 loss:0.001 grdn:0.129 lr:3.6e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 19:59:07 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.03 loss:0.001 grdn:0.126 lr:3.5e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 19:59:54 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.09 loss:0.001 grdn:0.121 lr:3.5e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 20:00:40 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.14 loss:0.001 grdn:0.120 lr:3.5e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 20:01:27 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.20 loss:0.001 grdn:0.127 lr:3.5e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 20:02:12 ts/train.py:232 step:792K smpl:13M ep:65K epch:218.25 loss:0.001 grdn:0.121 lr:3.5e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 20:02:58 ts/train.py:232 step:793K smpl:13M ep:65K epch:218.31 loss:0.001 grdn:0.121 lr:3.5e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 20:03:46 ts/train.py:232 step:793K smpl:13M ep:66K epch:218.36 loss:0.001 grdn:0.145 lr:3.5e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 20:04:33 ts/train.py:232 step:793K smpl:13M ep:66K epch:218.42 loss:0.001 grdn:0.136 lr:3.5e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 20:05:18 ts/train.py:232 step:793K smpl:13M ep:66K epch:218.47 loss:0.001 grdn:0.125 lr:3.4e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 20:06:04 ts/train.py:232 step:793K smpl:13M ep:66K epch:218.53 loss:0.001 grdn:0.116 lr:3.4e-06 updt_s:0.158 data_s:0.069 +INFO 2025-05-07 20:06:50 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.58 loss:0.001 grdn:0.118 lr:3.4e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 20:07:37 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.64 loss:0.001 grdn:0.112 lr:3.4e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 20:08:24 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.70 loss:0.001 grdn:0.127 lr:3.4e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 20:09:09 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.75 loss:0.001 grdn:0.125 lr:3.4e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 20:09:55 ts/train.py:232 step:794K smpl:13M ep:66K epch:218.81 loss:0.001 grdn:0.129 lr:3.4e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 20:10:43 ts/train.py:232 step:795K smpl:13M ep:66K epch:218.86 loss:0.001 grdn:0.116 lr:3.4e-06 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 20:11:30 ts/train.py:232 step:795K smpl:13M ep:66K epch:218.92 loss:0.001 grdn:0.113 lr:3.3e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 20:12:16 ts/train.py:232 step:795K smpl:13M ep:66K epch:218.97 loss:0.001 grdn:0.118 lr:3.3e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 20:13:02 ts/train.py:232 step:795K smpl:13M ep:66K epch:219.03 loss:0.001 grdn:0.123 lr:3.3e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 20:13:47 ts/train.py:232 step:795K smpl:13M ep:66K epch:219.08 loss:0.001 grdn:0.113 lr:3.3e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 20:14:34 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.14 loss:0.001 grdn:0.122 lr:3.3e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 20:15:21 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.19 loss:0.001 grdn:0.121 lr:3.3e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 20:16:08 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.25 loss:0.001 grdn:0.120 lr:3.3e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 20:16:55 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.30 loss:0.001 grdn:0.116 lr:3.3e-06 updt_s:0.158 data_s:0.076 +INFO 2025-05-07 20:17:42 ts/train.py:232 step:796K smpl:13M ep:66K epch:219.36 loss:0.001 grdn:0.119 lr:3.2e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 20:18:28 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.41 loss:0.001 grdn:0.133 lr:3.2e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 20:19:15 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.47 loss:0.001 grdn:0.111 lr:3.2e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 20:20:00 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.52 loss:0.001 grdn:0.132 lr:3.2e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 20:20:45 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.58 loss:0.001 grdn:0.115 lr:3.2e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 20:21:31 ts/train.py:232 step:797K smpl:13M ep:66K epch:219.63 loss:0.001 grdn:0.127 lr:3.2e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 20:22:16 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.69 loss:0.001 grdn:0.126 lr:3.2e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 20:23:02 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.74 loss:0.001 grdn:0.115 lr:3.2e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 20:23:47 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.80 loss:0.001 grdn:0.112 lr:3.1e-06 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 20:24:34 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.85 loss:0.001 grdn:0.111 lr:3.1e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 20:25:19 ts/train.py:232 step:798K smpl:13M ep:66K epch:219.91 loss:0.001 grdn:0.117 lr:3.1e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 20:26:04 ts/train.py:232 step:799K smpl:13M ep:66K epch:219.96 loss:0.001 grdn:0.113 lr:3.1e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 20:26:49 ts/train.py:232 step:799K smpl:13M ep:66K epch:220.02 loss:0.001 grdn:0.131 lr:3.1e-06 updt_s:0.158 data_s:0.064 +INFO 2025-05-07 20:27:34 ts/train.py:232 step:799K smpl:13M ep:66K epch:220.07 loss:0.001 grdn:0.137 lr:3.1e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 20:28:20 ts/train.py:232 step:799K smpl:13M ep:66K epch:220.13 loss:0.001 grdn:0.123 lr:3.1e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 20:29:06 ts/train.py:232 step:799K smpl:13M ep:66K epch:220.18 loss:0.001 grdn:0.130 lr:3.1e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 20:29:50 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.24 loss:0.001 grdn:0.122 lr:3.0e-06 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 20:30:34 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.29 loss:0.001 grdn:0.123 lr:3.0e-06 updt_s:0.157 data_s:0.062 +INFO 2025-05-07 20:31:18 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.35 loss:0.001 grdn:0.116 lr:3.0e-06 updt_s:0.157 data_s:0.060 +INFO 2025-05-07 20:32:01 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.40 loss:0.001 grdn:0.112 lr:3.0e-06 updt_s:0.157 data_s:0.061 +INFO 2025-05-07 20:32:46 ts/train.py:232 step:800K smpl:13M ep:66K epch:220.46 loss:0.001 grdn:0.128 lr:3.0e-06 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 20:33:30 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.51 loss:0.001 grdn:0.113 lr:3.0e-06 updt_s:0.157 data_s:0.063 +INFO 2025-05-07 20:34:14 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.57 loss:0.001 grdn:0.118 lr:3.0e-06 updt_s:0.157 data_s:0.062 +INFO 2025-05-07 20:34:58 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.62 loss:0.001 grdn:0.122 lr:3.0e-06 updt_s:0.157 data_s:0.061 +INFO 2025-05-07 20:35:42 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.68 loss:0.001 grdn:0.130 lr:3.0e-06 updt_s:0.157 data_s:0.064 +INFO 2025-05-07 20:36:25 ts/train.py:232 step:801K smpl:13M ep:66K epch:220.73 loss:0.001 grdn:0.119 lr:2.9e-06 updt_s:0.157 data_s:0.059 +INFO 2025-05-07 20:37:13 ts/train.py:232 step:802K smpl:13M ep:66K epch:220.79 loss:0.001 grdn:0.126 lr:2.9e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 20:38:01 ts/train.py:232 step:802K smpl:13M ep:66K epch:220.84 loss:0.001 grdn:0.112 lr:2.9e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 20:38:48 ts/train.py:232 step:802K smpl:13M ep:66K epch:220.90 loss:0.001 grdn:0.124 lr:2.9e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 20:39:36 ts/train.py:232 step:802K smpl:13M ep:66K epch:220.95 loss:0.001 grdn:0.124 lr:2.9e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 20:40:23 ts/train.py:232 step:802K smpl:13M ep:66K epch:221.01 loss:0.001 grdn:0.123 lr:2.9e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 20:41:10 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.06 loss:0.001 grdn:0.128 lr:2.9e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 20:41:57 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.12 loss:0.001 grdn:0.110 lr:2.9e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 20:42:44 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.17 loss:0.001 grdn:0.105 lr:2.8e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 20:43:31 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.23 loss:0.001 grdn:0.124 lr:2.8e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 20:44:19 ts/train.py:232 step:803K smpl:13M ep:66K epch:221.28 loss:0.001 grdn:0.129 lr:2.8e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 20:45:06 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.34 loss:0.001 grdn:0.106 lr:2.8e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 20:45:53 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.39 loss:0.001 grdn:0.118 lr:2.8e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 20:46:40 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.45 loss:0.001 grdn:0.112 lr:2.8e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 20:47:27 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.50 loss:0.001 grdn:0.114 lr:2.8e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 20:48:14 ts/train.py:232 step:804K smpl:13M ep:66K epch:221.56 loss:0.001 grdn:0.133 lr:2.8e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 20:49:01 ts/train.py:232 step:805K smpl:13M ep:66K epch:221.61 loss:0.001 grdn:0.118 lr:2.8e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 20:49:48 ts/train.py:232 step:805K smpl:13M ep:67K epch:221.67 loss:0.001 grdn:0.118 lr:2.7e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 20:50:36 ts/train.py:232 step:805K smpl:13M ep:67K epch:221.72 loss:0.001 grdn:0.127 lr:2.7e-06 updt_s:0.156 data_s:0.083 +INFO 2025-05-07 20:51:28 ts/train.py:232 step:805K smpl:13M ep:67K epch:221.78 loss:0.001 grdn:0.115 lr:2.7e-06 updt_s:0.157 data_s:0.102 +INFO 2025-05-07 20:52:17 ts/train.py:232 step:805K smpl:13M ep:67K epch:221.84 loss:0.001 grdn:0.119 lr:2.7e-06 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 20:53:06 ts/train.py:232 step:806K smpl:13M ep:67K epch:221.89 loss:0.001 grdn:0.110 lr:2.7e-06 updt_s:0.157 data_s:0.088 +INFO 2025-05-07 20:53:55 ts/train.py:232 step:806K smpl:13M ep:67K epch:221.95 loss:0.001 grdn:0.121 lr:2.7e-06 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 20:54:44 ts/train.py:232 step:806K smpl:13M ep:67K epch:222.00 loss:0.001 grdn:0.133 lr:2.7e-06 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 20:55:34 ts/train.py:232 step:806K smpl:13M ep:67K epch:222.06 loss:0.001 grdn:0.129 lr:2.7e-06 updt_s:0.157 data_s:0.090 +INFO 2025-05-07 20:56:21 ts/train.py:232 step:806K smpl:13M ep:67K epch:222.11 loss:0.001 grdn:0.123 lr:2.7e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 20:57:08 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.17 loss:0.001 grdn:0.132 lr:2.6e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 20:57:56 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.22 loss:0.001 grdn:0.112 lr:2.6e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 20:58:44 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.28 loss:0.001 grdn:0.133 lr:2.6e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 20:59:31 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.33 loss:0.001 grdn:0.125 lr:2.6e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 21:00:18 ts/train.py:232 step:807K smpl:13M ep:67K epch:222.39 loss:0.001 grdn:0.142 lr:2.6e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 21:01:05 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.44 loss:0.001 grdn:0.129 lr:2.6e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 21:01:53 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.50 loss:0.001 grdn:0.114 lr:2.6e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 21:02:40 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.55 loss:0.001 grdn:0.139 lr:2.6e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 21:03:28 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.61 loss:0.001 grdn:0.128 lr:2.6e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 21:04:17 ts/train.py:232 step:808K smpl:13M ep:67K epch:222.66 loss:0.001 grdn:0.123 lr:2.5e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 21:05:05 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.72 loss:0.001 grdn:0.116 lr:2.5e-06 updt_s:0.156 data_s:0.086 +INFO 2025-05-07 21:05:51 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.77 loss:0.001 grdn:0.126 lr:2.5e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 21:06:40 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.83 loss:0.001 grdn:0.125 lr:2.5e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 21:07:28 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.88 loss:0.001 grdn:0.105 lr:2.5e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 21:08:13 ts/train.py:232 step:809K smpl:13M ep:67K epch:222.94 loss:0.001 grdn:0.118 lr:2.5e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 21:09:00 ts/train.py:232 step:810K smpl:13M ep:67K epch:222.99 loss:0.001 grdn:0.137 lr:2.5e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 21:09:48 ts/train.py:232 step:810K smpl:13M ep:67K epch:223.05 loss:0.001 grdn:0.114 lr:2.5e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 21:10:34 ts/train.py:232 step:810K smpl:13M ep:67K epch:223.10 loss:0.001 grdn:0.120 lr:2.5e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:10:34 ts/train.py:241 Checkpoint policy after step 810000 +INFO 2025-05-07 21:11:23 ts/train.py:232 step:810K smpl:13M ep:67K epch:223.16 loss:0.001 grdn:0.118 lr:2.4e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 21:12:11 ts/train.py:232 step:810K smpl:13M ep:67K epch:223.21 loss:0.001 grdn:0.109 lr:2.4e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 21:12:56 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.27 loss:0.001 grdn:0.138 lr:2.4e-06 updt_s:0.158 data_s:0.071 +INFO 2025-05-07 21:13:43 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.32 loss:0.001 grdn:0.113 lr:2.4e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:14:29 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.38 loss:0.001 grdn:0.124 lr:2.4e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 21:15:15 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.43 loss:0.001 grdn:0.142 lr:2.4e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 21:16:03 ts/train.py:232 step:811K smpl:13M ep:67K epch:223.49 loss:0.001 grdn:0.115 lr:2.4e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 21:16:50 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.54 loss:0.001 grdn:0.129 lr:2.4e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 21:17:37 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.60 loss:0.001 grdn:0.116 lr:2.4e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 21:18:23 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.65 loss:0.001 grdn:0.124 lr:2.3e-06 updt_s:0.156 data_s:0.074 +INFO 2025-05-07 21:19:13 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.71 loss:0.001 grdn:0.121 lr:2.3e-06 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 21:20:01 ts/train.py:232 step:812K smpl:13M ep:67K epch:223.76 loss:0.001 grdn:0.101 lr:2.3e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 21:20:48 ts/train.py:232 step:813K smpl:13M ep:67K epch:223.82 loss:0.001 grdn:0.112 lr:2.3e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 21:21:35 ts/train.py:232 step:813K smpl:13M ep:67K epch:223.87 loss:0.001 grdn:0.141 lr:2.3e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 21:22:22 ts/train.py:232 step:813K smpl:13M ep:67K epch:223.93 loss:0.001 grdn:0.123 lr:2.3e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 21:23:08 ts/train.py:232 step:813K smpl:13M ep:67K epch:223.98 loss:0.001 grdn:0.127 lr:2.3e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:23:55 ts/train.py:232 step:813K smpl:13M ep:67K epch:224.04 loss:0.001 grdn:0.122 lr:2.3e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 21:24:43 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.09 loss:0.001 grdn:0.114 lr:2.3e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 21:25:29 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.15 loss:0.001 grdn:0.110 lr:2.3e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:26:14 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.20 loss:0.001 grdn:0.119 lr:2.2e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 21:27:00 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.26 loss:0.001 grdn:0.133 lr:2.2e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 21:27:46 ts/train.py:232 step:814K smpl:13M ep:67K epch:224.31 loss:0.001 grdn:0.132 lr:2.2e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 21:28:31 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.37 loss:0.001 grdn:0.119 lr:2.2e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 21:29:17 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.42 loss:0.001 grdn:0.126 lr:2.2e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 21:30:02 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.48 loss:0.001 grdn:0.112 lr:2.2e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 21:30:49 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.53 loss:0.001 grdn:0.121 lr:2.2e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 21:31:34 ts/train.py:232 step:815K smpl:13M ep:67K epch:224.59 loss:0.001 grdn:0.113 lr:2.2e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 21:32:22 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.64 loss:0.001 grdn:0.130 lr:2.2e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 21:33:08 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.70 loss:0.001 grdn:0.122 lr:2.2e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 21:33:53 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.75 loss:0.001 grdn:0.123 lr:2.1e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 21:34:41 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.81 loss:0.001 grdn:0.121 lr:2.1e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 21:35:28 ts/train.py:232 step:816K smpl:13M ep:67K epch:224.86 loss:0.001 grdn:0.105 lr:2.1e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 21:36:14 ts/train.py:232 step:817K smpl:13M ep:67K epch:224.92 loss:0.001 grdn:0.120 lr:2.1e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:37:01 ts/train.py:232 step:817K smpl:13M ep:67K epch:224.98 loss:0.001 grdn:0.128 lr:2.1e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 21:37:47 ts/train.py:232 step:817K smpl:13M ep:68K epch:225.03 loss:0.001 grdn:0.123 lr:2.1e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 21:38:34 ts/train.py:232 step:817K smpl:13M ep:68K epch:225.09 loss:0.001 grdn:0.119 lr:2.1e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 21:39:21 ts/train.py:232 step:817K smpl:13M ep:68K epch:225.14 loss:0.001 grdn:0.121 lr:2.1e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 21:40:09 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.20 loss:0.001 grdn:0.127 lr:2.1e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 21:40:58 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.25 loss:0.001 grdn:0.130 lr:2.1e-06 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 21:41:44 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.31 loss:0.001 grdn:0.116 lr:2.0e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 21:42:31 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.36 loss:0.001 grdn:0.131 lr:2.0e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:43:17 ts/train.py:232 step:818K smpl:13M ep:68K epch:225.42 loss:0.001 grdn:0.131 lr:2.0e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 21:44:04 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.47 loss:0.001 grdn:0.141 lr:2.0e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 21:44:50 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.53 loss:0.001 grdn:0.119 lr:2.0e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 21:45:36 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.58 loss:0.001 grdn:0.128 lr:2.0e-06 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 21:46:24 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.64 loss:0.001 grdn:0.125 lr:2.0e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 21:47:11 ts/train.py:232 step:819K smpl:13M ep:68K epch:225.69 loss:0.001 grdn:0.134 lr:2.0e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 21:47:57 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.75 loss:0.001 grdn:0.115 lr:2.0e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 21:48:43 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.80 loss:0.001 grdn:0.115 lr:2.0e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 21:49:30 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.86 loss:0.001 grdn:0.115 lr:1.9e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 21:50:17 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.91 loss:0.001 grdn:0.105 lr:1.9e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:51:03 ts/train.py:232 step:820K smpl:13M ep:68K epch:225.97 loss:0.001 grdn:0.120 lr:1.9e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:51:50 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.02 loss:0.001 grdn:0.119 lr:1.9e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 21:52:37 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.08 loss:0.001 grdn:0.120 lr:1.9e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 21:53:24 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.13 loss:0.001 grdn:0.127 lr:1.9e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 21:54:10 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.19 loss:0.001 grdn:0.118 lr:1.9e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:54:57 ts/train.py:232 step:821K smpl:13M ep:68K epch:226.24 loss:0.001 grdn:0.109 lr:1.9e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:55:43 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.30 loss:0.001 grdn:0.128 lr:1.9e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 21:56:29 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.35 loss:0.001 grdn:0.126 lr:1.9e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 21:57:15 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.41 loss:0.001 grdn:0.107 lr:1.8e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 21:58:01 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.46 loss:0.001 grdn:0.121 lr:1.8e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 21:58:48 ts/train.py:232 step:822K smpl:13M ep:68K epch:226.52 loss:0.001 grdn:0.122 lr:1.8e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 21:59:35 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.57 loss:0.001 grdn:0.120 lr:1.8e-06 updt_s:0.156 data_s:0.078 +INFO 2025-05-07 22:00:20 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.63 loss:0.001 grdn:0.116 lr:1.8e-06 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 22:01:05 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.68 loss:0.001 grdn:0.118 lr:1.8e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 22:01:52 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.74 loss:0.001 grdn:0.124 lr:1.8e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 22:02:39 ts/train.py:232 step:823K smpl:13M ep:68K epch:226.79 loss:0.001 grdn:0.127 lr:1.8e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 22:03:25 ts/train.py:232 step:824K smpl:13M ep:68K epch:226.85 loss:0.001 grdn:0.120 lr:1.8e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 22:04:11 ts/train.py:232 step:824K smpl:13M ep:68K epch:226.90 loss:0.001 grdn:0.114 lr:1.8e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 22:04:58 ts/train.py:232 step:824K smpl:13M ep:68K epch:226.96 loss:0.001 grdn:0.112 lr:1.8e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 22:05:44 ts/train.py:232 step:824K smpl:13M ep:68K epch:227.01 loss:0.001 grdn:0.125 lr:1.7e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 22:06:31 ts/train.py:232 step:824K smpl:13M ep:68K epch:227.07 loss:0.001 grdn:0.115 lr:1.7e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 22:07:18 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.12 loss:0.001 grdn:0.142 lr:1.7e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 22:08:05 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.18 loss:0.001 grdn:0.107 lr:1.7e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 22:08:53 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.23 loss:0.001 grdn:0.121 lr:1.7e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 22:09:39 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.29 loss:0.001 grdn:0.113 lr:1.7e-06 updt_s:0.158 data_s:0.070 +INFO 2025-05-07 22:10:25 ts/train.py:232 step:825K smpl:13M ep:68K epch:227.34 loss:0.001 grdn:0.103 lr:1.7e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 22:11:10 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.40 loss:0.001 grdn:0.126 lr:1.7e-06 updt_s:0.157 data_s:0.071 +INFO 2025-05-07 22:11:57 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.45 loss:0.001 grdn:0.115 lr:1.7e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 22:12:43 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.51 loss:0.001 grdn:0.114 lr:1.7e-06 updt_s:0.156 data_s:0.070 +INFO 2025-05-07 22:13:29 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.56 loss:0.001 grdn:0.119 lr:1.7e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 22:14:15 ts/train.py:232 step:826K smpl:13M ep:68K epch:227.62 loss:0.001 grdn:0.107 lr:1.6e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 22:15:00 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.67 loss:0.001 grdn:0.122 lr:1.6e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 22:15:46 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.73 loss:0.001 grdn:0.108 lr:1.6e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 22:16:31 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.78 loss:0.001 grdn:0.115 lr:1.6e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 22:17:17 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.84 loss:0.001 grdn:0.111 lr:1.6e-06 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 22:18:03 ts/train.py:232 step:827K smpl:13M ep:68K epch:227.89 loss:0.001 grdn:0.111 lr:1.6e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 22:18:48 ts/train.py:232 step:828K smpl:13M ep:68K epch:227.95 loss:0.001 grdn:0.117 lr:1.6e-06 updt_s:0.158 data_s:0.067 +INFO 2025-05-07 22:19:33 ts/train.py:232 step:828K smpl:13M ep:68K epch:228.00 loss:0.001 grdn:0.117 lr:1.6e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 22:20:18 ts/train.py:232 step:828K smpl:13M ep:68K epch:228.06 loss:0.001 grdn:0.140 lr:1.6e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 22:21:04 ts/train.py:232 step:828K smpl:13M ep:68K epch:228.11 loss:0.001 grdn:0.108 lr:1.6e-06 updt_s:0.157 data_s:0.069 +INFO 2025-05-07 22:21:49 ts/train.py:232 step:828K smpl:13M ep:68K epch:228.17 loss:0.001 grdn:0.124 lr:1.6e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 22:22:35 ts/train.py:232 step:829K smpl:13M ep:68K epch:228.23 loss:0.001 grdn:0.132 lr:1.6e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 22:23:19 ts/train.py:232 step:829K smpl:13M ep:68K epch:228.28 loss:0.001 grdn:0.116 lr:1.5e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 22:24:04 ts/train.py:232 step:829K smpl:13M ep:69K epch:228.34 loss:0.001 grdn:0.123 lr:1.5e-06 updt_s:0.157 data_s:0.065 +INFO 2025-05-07 22:24:49 ts/train.py:232 step:829K smpl:13M ep:69K epch:228.39 loss:0.001 grdn:0.116 lr:1.5e-06 updt_s:0.157 data_s:0.068 +INFO 2025-05-07 22:25:34 ts/train.py:232 step:829K smpl:13M ep:69K epch:228.45 loss:0.001 grdn:0.131 lr:1.5e-06 updt_s:0.157 data_s:0.067 +INFO 2025-05-07 22:26:22 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.50 loss:0.001 grdn:0.121 lr:1.5e-06 updt_s:0.156 data_s:0.085 +INFO 2025-05-07 22:27:11 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.56 loss:0.001 grdn:0.133 lr:1.5e-06 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 22:27:59 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.61 loss:0.001 grdn:0.119 lr:1.5e-06 updt_s:0.158 data_s:0.082 +INFO 2025-05-07 22:28:47 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.67 loss:0.001 grdn:0.117 lr:1.5e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 22:29:34 ts/train.py:232 step:830K smpl:13M ep:69K epch:228.72 loss:0.001 grdn:0.111 lr:1.5e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 22:30:22 ts/train.py:232 step:831K smpl:13M ep:69K epch:228.78 loss:0.001 grdn:0.121 lr:1.5e-06 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 22:31:10 ts/train.py:232 step:831K smpl:13M ep:69K epch:228.83 loss:0.001 grdn:0.109 lr:1.5e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 22:31:58 ts/train.py:232 step:831K smpl:13M ep:69K epch:228.89 loss:0.001 grdn:0.120 lr:1.4e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 22:32:48 ts/train.py:232 step:831K smpl:13M ep:69K epch:228.94 loss:0.001 grdn:0.109 lr:1.4e-06 updt_s:0.157 data_s:0.096 +INFO 2025-05-07 22:33:37 ts/train.py:232 step:831K smpl:13M ep:69K epch:229.00 loss:0.001 grdn:0.111 lr:1.4e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 22:34:25 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.05 loss:0.001 grdn:0.090 lr:1.4e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 22:35:13 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.11 loss:0.001 grdn:0.108 lr:1.4e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 22:36:01 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.16 loss:0.001 grdn:0.105 lr:1.4e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 22:36:49 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.22 loss:0.001 grdn:0.116 lr:1.4e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 22:37:37 ts/train.py:232 step:832K smpl:13M ep:69K epch:229.27 loss:0.001 grdn:0.122 lr:1.4e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 22:38:25 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.33 loss:0.001 grdn:0.112 lr:1.4e-06 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 22:39:13 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.38 loss:0.001 grdn:0.099 lr:1.4e-06 updt_s:0.157 data_s:0.084 +INFO 2025-05-07 22:40:02 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.44 loss:0.001 grdn:0.114 lr:1.4e-06 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 22:40:51 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.49 loss:0.001 grdn:0.105 lr:1.4e-06 updt_s:0.157 data_s:0.091 +INFO 2025-05-07 22:41:38 ts/train.py:232 step:833K smpl:13M ep:69K epch:229.55 loss:0.001 grdn:0.102 lr:1.4e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 22:42:24 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.60 loss:0.001 grdn:0.104 lr:1.3e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 22:43:10 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.66 loss:0.001 grdn:0.101 lr:1.3e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 22:43:56 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.71 loss:0.001 grdn:0.125 lr:1.3e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 22:44:43 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.77 loss:0.001 grdn:0.117 lr:1.3e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 22:45:29 ts/train.py:232 step:834K smpl:13M ep:69K epch:229.82 loss:0.001 grdn:0.109 lr:1.3e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 22:46:16 ts/train.py:232 step:835K smpl:13M ep:69K epch:229.88 loss:0.001 grdn:0.115 lr:1.3e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 22:47:03 ts/train.py:232 step:835K smpl:13M ep:69K epch:229.93 loss:0.001 grdn:0.112 lr:1.3e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 22:47:50 ts/train.py:232 step:835K smpl:13M ep:69K epch:229.99 loss:0.001 grdn:0.112 lr:1.3e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 22:48:37 ts/train.py:232 step:835K smpl:13M ep:69K epch:230.04 loss:0.001 grdn:0.106 lr:1.3e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 22:49:23 ts/train.py:232 step:835K smpl:13M ep:69K epch:230.10 loss:0.001 grdn:0.128 lr:1.3e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 22:50:11 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.15 loss:0.001 grdn:0.114 lr:1.3e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 22:50:57 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.21 loss:0.001 grdn:0.121 lr:1.3e-06 updt_s:0.157 data_s:0.073 +INFO 2025-05-07 22:51:44 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.26 loss:0.001 grdn:0.113 lr:1.2e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 22:52:30 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.32 loss:0.001 grdn:0.118 lr:1.2e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 22:53:17 ts/train.py:232 step:836K smpl:13M ep:69K epch:230.37 loss:0.001 grdn:0.109 lr:1.2e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 22:54:06 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.43 loss:0.001 grdn:0.114 lr:1.2e-06 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 22:54:54 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.48 loss:0.001 grdn:0.124 lr:1.2e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 22:55:41 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.54 loss:0.001 grdn:0.107 lr:1.2e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 22:56:28 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.59 loss:0.001 grdn:0.096 lr:1.2e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 22:57:15 ts/train.py:232 step:837K smpl:13M ep:69K epch:230.65 loss:0.001 grdn:0.115 lr:1.2e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 22:58:01 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.70 loss:0.001 grdn:0.127 lr:1.2e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 22:58:47 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.76 loss:0.001 grdn:0.119 lr:1.2e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 22:59:35 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.81 loss:0.001 grdn:0.117 lr:1.2e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 23:00:21 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.87 loss:0.001 grdn:0.113 lr:1.2e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 23:01:08 ts/train.py:232 step:838K smpl:13M ep:69K epch:230.92 loss:0.001 grdn:0.121 lr:1.2e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 23:01:55 ts/train.py:232 step:839K smpl:13M ep:69K epch:230.98 loss:0.001 grdn:0.108 lr:1.1e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 23:02:41 ts/train.py:232 step:839K smpl:13M ep:69K epch:231.03 loss:0.001 grdn:0.112 lr:1.1e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 23:03:28 ts/train.py:232 step:839K smpl:13M ep:69K epch:231.09 loss:0.001 grdn:0.134 lr:1.1e-06 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 23:04:14 ts/train.py:232 step:839K smpl:13M ep:69K epch:231.14 loss:0.001 grdn:0.141 lr:1.1e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 23:05:00 ts/train.py:232 step:839K smpl:13M ep:69K epch:231.20 loss:0.001 grdn:0.110 lr:1.1e-06 updt_s:0.157 data_s:0.072 +INFO 2025-05-07 23:05:46 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.25 loss:0.001 grdn:0.103 lr:1.1e-06 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 23:06:33 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.31 loss:0.001 grdn:0.111 lr:1.1e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 23:07:21 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.37 loss:0.001 grdn:0.128 lr:1.1e-06 updt_s:0.156 data_s:0.082 +INFO 2025-05-07 23:07:21 ts/train.py:241 Checkpoint policy after step 840000 +INFO 2025-05-07 23:08:13 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.42 loss:0.001 grdn:0.129 lr:1.1e-06 updt_s:0.157 data_s:0.089 +INFO 2025-05-07 23:09:01 ts/train.py:232 step:840K smpl:13M ep:69K epch:231.48 loss:0.001 grdn:0.111 lr:1.1e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 23:09:48 ts/train.py:232 step:841K smpl:13M ep:69K epch:231.53 loss:0.001 grdn:0.122 lr:1.1e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 23:10:35 ts/train.py:232 step:841K smpl:13M ep:69K epch:231.59 loss:0.001 grdn:0.115 lr:1.1e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 23:11:23 ts/train.py:232 step:841K smpl:13M ep:69K epch:231.64 loss:0.001 grdn:0.118 lr:1.1e-06 updt_s:0.157 data_s:0.086 +INFO 2025-05-07 23:12:11 ts/train.py:232 step:841K smpl:13M ep:70K epch:231.70 loss:0.001 grdn:0.107 lr:1.1e-06 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 23:12:59 ts/train.py:232 step:841K smpl:13M ep:70K epch:231.75 loss:0.001 grdn:0.102 lr:1.0e-06 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 23:13:45 ts/train.py:232 step:842K smpl:13M ep:70K epch:231.81 loss:0.001 grdn:0.100 lr:1.0e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 23:14:33 ts/train.py:232 step:842K smpl:13M ep:70K epch:231.86 loss:0.001 grdn:0.108 lr:1.0e-06 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 23:15:20 ts/train.py:232 step:842K smpl:13M ep:70K epch:231.92 loss:0.001 grdn:0.120 lr:1.0e-06 updt_s:0.158 data_s:0.078 +INFO 2025-05-07 23:16:07 ts/train.py:232 step:842K smpl:13M ep:70K epch:231.97 loss:0.001 grdn:0.127 lr:1.0e-06 updt_s:0.157 data_s:0.076 +INFO 2025-05-07 23:16:54 ts/train.py:232 step:842K smpl:13M ep:70K epch:232.03 loss:0.001 grdn:0.114 lr:1.0e-06 updt_s:0.157 data_s:0.077 +INFO 2025-05-07 23:17:41 ts/train.py:232 step:843K smpl:13M ep:70K epch:232.08 loss:0.001 grdn:0.114 lr:1.0e-06 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 23:18:29 ts/train.py:232 step:843K smpl:13M ep:70K epch:232.14 loss:0.001 grdn:0.128 lr:1.0e-06 updt_s:0.157 data_s:0.085 +INFO 2025-05-07 23:19:17 ts/train.py:232 step:843K smpl:13M ep:70K epch:232.19 loss:0.001 grdn:0.124 lr:9.9e-07 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 23:20:06 ts/train.py:232 step:843K smpl:13M ep:70K epch:232.25 loss:0.001 grdn:0.118 lr:9.8e-07 updt_s:0.156 data_s:0.087 +INFO 2025-05-07 23:20:53 ts/train.py:232 step:843K smpl:13M ep:70K epch:232.30 loss:0.001 grdn:0.117 lr:9.8e-07 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 23:21:42 ts/train.py:232 step:844K smpl:13M ep:70K epch:232.36 loss:0.001 grdn:0.125 lr:9.7e-07 updt_s:0.156 data_s:0.089 +INFO 2025-05-07 23:22:27 ts/train.py:232 step:844K smpl:14M ep:70K epch:232.41 loss:0.001 grdn:0.110 lr:9.6e-07 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 23:23:13 ts/train.py:232 step:844K smpl:14M ep:70K epch:232.47 loss:0.001 grdn:0.113 lr:9.6e-07 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 23:23:58 ts/train.py:232 step:844K smpl:14M ep:70K epch:232.52 loss:0.001 grdn:0.128 lr:9.5e-07 updt_s:0.156 data_s:0.069 +INFO 2025-05-07 23:24:42 ts/train.py:232 step:844K smpl:14M ep:70K epch:232.58 loss:0.001 grdn:0.102 lr:9.4e-07 updt_s:0.156 data_s:0.065 +INFO 2025-05-07 23:25:27 ts/train.py:232 step:845K smpl:14M ep:70K epch:232.63 loss:0.001 grdn:0.119 lr:9.4e-07 updt_s:0.156 data_s:0.066 +INFO 2025-05-07 23:26:12 ts/train.py:232 step:845K smpl:14M ep:70K epch:232.69 loss:0.001 grdn:0.094 lr:9.3e-07 updt_s:0.157 data_s:0.066 +INFO 2025-05-07 23:26:57 ts/train.py:232 step:845K smpl:14M ep:70K epch:232.74 loss:0.001 grdn:0.111 lr:9.2e-07 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 23:27:42 ts/train.py:232 step:845K smpl:14M ep:70K epch:232.80 loss:0.001 grdn:0.120 lr:9.2e-07 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 23:28:28 ts/train.py:232 step:845K smpl:14M ep:70K epch:232.85 loss:0.001 grdn:0.125 lr:9.1e-07 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 23:29:13 ts/train.py:232 step:846K smpl:14M ep:70K epch:232.91 loss:0.001 grdn:0.121 lr:9.0e-07 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 23:29:57 ts/train.py:232 step:846K smpl:14M ep:70K epch:232.96 loss:0.001 grdn:0.122 lr:9.0e-07 updt_s:0.156 data_s:0.067 +INFO 2025-05-07 23:30:43 ts/train.py:232 step:846K smpl:14M ep:70K epch:233.02 loss:0.001 grdn:0.105 lr:8.9e-07 updt_s:0.156 data_s:0.071 +INFO 2025-05-07 23:31:27 ts/train.py:232 step:846K smpl:14M ep:70K epch:233.07 loss:0.001 grdn:0.121 lr:8.8e-07 updt_s:0.156 data_s:0.065 +INFO 2025-05-07 23:32:12 ts/train.py:232 step:846K smpl:14M ep:70K epch:233.13 loss:0.001 grdn:0.123 lr:8.8e-07 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 23:32:57 ts/train.py:232 step:847K smpl:14M ep:70K epch:233.18 loss:0.001 grdn:0.122 lr:8.7e-07 updt_s:0.156 data_s:0.066 +INFO 2025-05-07 23:33:43 ts/train.py:232 step:847K smpl:14M ep:70K epch:233.24 loss:0.001 grdn:0.107 lr:8.6e-07 updt_s:0.156 data_s:0.072 +INFO 2025-05-07 23:34:28 ts/train.py:232 step:847K smpl:14M ep:70K epch:233.29 loss:0.001 grdn:0.112 lr:8.6e-07 updt_s:0.156 data_s:0.068 +INFO 2025-05-07 23:35:16 ts/train.py:232 step:847K smpl:14M ep:70K epch:233.35 loss:0.001 grdn:0.117 lr:8.5e-07 updt_s:0.156 data_s:0.084 +INFO 2025-05-07 23:36:00 ts/train.py:232 step:847K smpl:14M ep:70K epch:233.40 loss:0.001 grdn:0.108 lr:8.4e-07 updt_s:0.158 data_s:0.064 +INFO 2025-05-07 23:36:46 ts/train.py:232 step:848K smpl:14M ep:70K epch:233.46 loss:0.001 grdn:0.105 lr:8.4e-07 updt_s:0.157 data_s:0.070 +INFO 2025-05-07 23:37:34 ts/train.py:232 step:848K smpl:14M ep:70K epch:233.51 loss:0.001 grdn:0.120 lr:8.3e-07 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 23:38:22 ts/train.py:232 step:848K smpl:14M ep:70K epch:233.57 loss:0.001 grdn:0.126 lr:8.3e-07 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 23:39:09 ts/train.py:232 step:848K smpl:14M ep:70K epch:233.62 loss:0.001 grdn:0.097 lr:8.2e-07 updt_s:0.157 data_s:0.082 +INFO 2025-05-07 23:39:57 ts/train.py:232 step:848K smpl:14M ep:70K epch:233.68 loss:0.001 grdn:0.110 lr:8.1e-07 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 23:40:44 ts/train.py:232 step:849K smpl:14M ep:70K epch:233.73 loss:0.001 grdn:0.098 lr:8.1e-07 updt_s:0.157 data_s:0.079 +INFO 2025-05-07 23:41:31 ts/train.py:232 step:849K smpl:14M ep:70K epch:233.79 loss:0.001 grdn:0.107 lr:8.0e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 23:42:19 ts/train.py:232 step:849K smpl:14M ep:70K epch:233.84 loss:0.001 grdn:0.113 lr:7.9e-07 updt_s:0.157 data_s:0.080 +INFO 2025-05-07 23:43:06 ts/train.py:232 step:849K smpl:14M ep:70K epch:233.90 loss:0.001 grdn:0.108 lr:7.9e-07 updt_s:0.157 data_s:0.081 +INFO 2025-05-07 23:43:55 ts/train.py:232 step:849K smpl:14M ep:70K epch:233.95 loss:0.001 grdn:0.124 lr:7.8e-07 updt_s:0.157 data_s:0.087 +INFO 2025-05-07 23:44:42 ts/train.py:232 step:850K smpl:14M ep:70K epch:234.01 loss:0.001 grdn:0.123 lr:7.8e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-07 23:45:30 ts/train.py:232 step:850K smpl:14M ep:70K epch:234.06 loss:0.001 grdn:0.110 lr:7.7e-07 updt_s:0.157 data_s:0.083 +INFO 2025-05-07 23:46:17 ts/train.py:232 step:850K smpl:14M ep:70K epch:234.12 loss:0.001 grdn:0.115 lr:7.6e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 23:47:03 ts/train.py:232 step:850K smpl:14M ep:70K epch:234.17 loss:0.001 grdn:0.109 lr:7.6e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-07 23:47:50 ts/train.py:232 step:850K smpl:14M ep:70K epch:234.23 loss:0.001 grdn:0.122 lr:7.5e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-07 23:48:40 ts/train.py:232 step:851K smpl:14M ep:70K epch:234.28 loss:0.001 grdn:0.130 lr:7.5e-07 updt_s:0.156 data_s:0.093 +INFO 2025-05-07 23:49:30 ts/train.py:232 step:851K smpl:14M ep:70K epch:234.34 loss:0.001 grdn:0.122 lr:7.4e-07 updt_s:0.156 data_s:0.092 +INFO 2025-05-07 23:50:20 ts/train.py:232 step:851K smpl:14M ep:70K epch:234.39 loss:0.001 grdn:0.129 lr:7.3e-07 updt_s:0.156 data_s:0.093 +INFO 2025-05-07 23:51:09 ts/train.py:232 step:851K smpl:14M ep:70K epch:234.45 loss:0.001 grdn:0.114 lr:7.3e-07 updt_s:0.156 data_s:0.088 +INFO 2025-05-07 23:52:00 ts/train.py:232 step:851K smpl:14M ep:70K epch:234.51 loss:0.001 grdn:0.118 lr:7.2e-07 updt_s:0.156 data_s:0.097 +INFO 2025-05-07 23:52:50 ts/train.py:232 step:852K smpl:14M ep:70K epch:234.56 loss:0.001 grdn:0.114 lr:7.2e-07 updt_s:0.156 data_s:0.096 +INFO 2025-05-07 23:53:42 ts/train.py:232 step:852K smpl:14M ep:70K epch:234.62 loss:0.001 grdn:0.115 lr:7.1e-07 updt_s:0.156 data_s:0.099 +INFO 2025-05-07 23:54:34 ts/train.py:232 step:852K smpl:14M ep:70K epch:234.67 loss:0.001 grdn:0.109 lr:7.0e-07 updt_s:0.156 data_s:0.105 +INFO 2025-05-07 23:55:25 ts/train.py:232 step:852K smpl:14M ep:70K epch:234.73 loss:0.001 grdn:0.106 lr:7.0e-07 updt_s:0.156 data_s:0.099 +INFO 2025-05-07 23:56:16 ts/train.py:232 step:852K smpl:14M ep:70K epch:234.78 loss:0.001 grdn:0.108 lr:6.9e-07 updt_s:0.156 data_s:0.098 +INFO 2025-05-07 23:57:08 ts/train.py:232 step:853K smpl:14M ep:70K epch:234.84 loss:0.001 grdn:0.118 lr:6.9e-07 updt_s:0.156 data_s:0.103 +INFO 2025-05-07 23:57:59 ts/train.py:232 step:853K smpl:14M ep:70K epch:234.89 loss:0.001 grdn:0.112 lr:6.8e-07 updt_s:0.156 data_s:0.099 +INFO 2025-05-07 23:58:51 ts/train.py:232 step:853K smpl:14M ep:70K epch:234.95 loss:0.001 grdn:0.110 lr:6.7e-07 updt_s:0.156 data_s:0.102 +INFO 2025-05-07 23:59:43 ts/train.py:232 step:853K smpl:14M ep:71K epch:235.00 loss:0.001 grdn:0.118 lr:6.7e-07 updt_s:0.156 data_s:0.103 +INFO 2025-05-08 00:00:35 ts/train.py:232 step:853K smpl:14M ep:71K epch:235.06 loss:0.001 grdn:0.100 lr:6.6e-07 updt_s:0.156 data_s:0.101 +INFO 2025-05-08 00:01:27 ts/train.py:232 step:854K smpl:14M ep:71K epch:235.11 loss:0.001 grdn:0.108 lr:6.6e-07 updt_s:0.156 data_s:0.103 +INFO 2025-05-08 00:02:18 ts/train.py:232 step:854K smpl:14M ep:71K epch:235.17 loss:0.001 grdn:0.108 lr:6.5e-07 updt_s:0.156 data_s:0.101 +INFO 2025-05-08 00:03:08 ts/train.py:232 step:854K smpl:14M ep:71K epch:235.22 loss:0.001 grdn:0.126 lr:6.5e-07 updt_s:0.156 data_s:0.093 +INFO 2025-05-08 00:03:57 ts/train.py:232 step:854K smpl:14M ep:71K epch:235.28 loss:0.001 grdn:0.108 lr:6.4e-07 updt_s:0.157 data_s:0.084 +INFO 2025-05-08 00:04:43 ts/train.py:232 step:854K smpl:14M ep:71K epch:235.33 loss:0.001 grdn:0.128 lr:6.4e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 00:05:31 ts/train.py:232 step:855K smpl:14M ep:71K epch:235.39 loss:0.001 grdn:0.112 lr:6.3e-07 updt_s:0.157 data_s:0.081 +INFO 2025-05-08 00:06:19 ts/train.py:232 step:855K smpl:14M ep:71K epch:235.44 loss:0.001 grdn:0.123 lr:6.2e-07 updt_s:0.157 data_s:0.082 +INFO 2025-05-08 00:07:07 ts/train.py:232 step:855K smpl:14M ep:71K epch:235.50 loss:0.001 grdn:0.129 lr:6.2e-07 updt_s:0.157 data_s:0.081 +INFO 2025-05-08 00:07:56 ts/train.py:232 step:855K smpl:14M ep:71K epch:235.55 loss:0.001 grdn:0.116 lr:6.1e-07 updt_s:0.157 data_s:0.088 +INFO 2025-05-08 00:08:46 ts/train.py:232 step:855K smpl:14M ep:71K epch:235.61 loss:0.001 grdn:0.118 lr:6.1e-07 updt_s:0.157 data_s:0.092 +INFO 2025-05-08 00:09:35 ts/train.py:232 step:856K smpl:14M ep:71K epch:235.66 loss:0.001 grdn:0.109 lr:6.0e-07 updt_s:0.157 data_s:0.087 +INFO 2025-05-08 00:10:23 ts/train.py:232 step:856K smpl:14M ep:71K epch:235.72 loss:0.001 grdn:0.117 lr:6.0e-07 updt_s:0.157 data_s:0.083 +INFO 2025-05-08 00:11:11 ts/train.py:232 step:856K smpl:14M ep:71K epch:235.77 loss:0.001 grdn:0.107 lr:5.9e-07 updt_s:0.157 data_s:0.082 +INFO 2025-05-08 00:11:59 ts/train.py:232 step:856K smpl:14M ep:71K epch:235.83 loss:0.001 grdn:0.105 lr:5.9e-07 updt_s:0.157 data_s:0.081 +INFO 2025-05-08 00:12:48 ts/train.py:232 step:856K smpl:14M ep:71K epch:235.88 loss:0.001 grdn:0.112 lr:5.8e-07 updt_s:0.157 data_s:0.088 +INFO 2025-05-08 00:13:36 ts/train.py:232 step:857K smpl:14M ep:71K epch:235.94 loss:0.001 grdn:0.111 lr:5.8e-07 updt_s:0.157 data_s:0.085 +INFO 2025-05-08 00:14:23 ts/train.py:232 step:857K smpl:14M ep:71K epch:235.99 loss:0.001 grdn:0.121 lr:5.7e-07 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 00:15:09 ts/train.py:232 step:857K smpl:14M ep:71K epch:236.05 loss:0.001 grdn:0.106 lr:5.7e-07 updt_s:0.156 data_s:0.073 +INFO 2025-05-08 00:15:55 ts/train.py:232 step:857K smpl:14M ep:71K epch:236.10 loss:0.001 grdn:0.123 lr:5.6e-07 updt_s:0.156 data_s:0.068 +INFO 2025-05-08 00:16:41 ts/train.py:232 step:857K smpl:14M ep:71K epch:236.16 loss:0.001 grdn:0.103 lr:5.5e-07 updt_s:0.156 data_s:0.076 +INFO 2025-05-08 00:17:31 ts/train.py:232 step:858K smpl:14M ep:71K epch:236.21 loss:0.001 grdn:0.118 lr:5.5e-07 updt_s:0.156 data_s:0.089 +INFO 2025-05-08 00:18:19 ts/train.py:232 step:858K smpl:14M ep:71K epch:236.27 loss:0.001 grdn:0.121 lr:5.4e-07 updt_s:0.156 data_s:0.085 +INFO 2025-05-08 00:19:08 ts/train.py:232 step:858K smpl:14M ep:71K epch:236.32 loss:0.001 grdn:0.099 lr:5.4e-07 updt_s:0.156 data_s:0.088 +INFO 2025-05-08 00:19:57 ts/train.py:232 step:858K smpl:14M ep:71K epch:236.38 loss:0.001 grdn:0.115 lr:5.3e-07 updt_s:0.156 data_s:0.088 +INFO 2025-05-08 00:20:47 ts/train.py:232 step:858K smpl:14M ep:71K epch:236.43 loss:0.001 grdn:0.116 lr:5.3e-07 updt_s:0.156 data_s:0.090 +INFO 2025-05-08 00:21:36 ts/train.py:232 step:859K smpl:14M ep:71K epch:236.49 loss:0.001 grdn:0.127 lr:5.2e-07 updt_s:0.157 data_s:0.090 +INFO 2025-05-08 00:22:26 ts/train.py:232 step:859K smpl:14M ep:71K epch:236.54 loss:0.001 grdn:0.105 lr:5.2e-07 updt_s:0.157 data_s:0.093 +INFO 2025-05-08 00:23:15 ts/train.py:232 step:859K smpl:14M ep:71K epch:236.60 loss:0.001 grdn:0.105 lr:5.1e-07 updt_s:0.157 data_s:0.086 +INFO 2025-05-08 00:24:03 ts/train.py:232 step:859K smpl:14M ep:71K epch:236.65 loss:0.001 grdn:0.128 lr:5.1e-07 updt_s:0.156 data_s:0.083 +INFO 2025-05-08 00:24:52 ts/train.py:232 step:859K smpl:14M ep:71K epch:236.71 loss:0.001 grdn:0.116 lr:5.0e-07 updt_s:0.156 data_s:0.089 +INFO 2025-05-08 00:25:42 ts/train.py:232 step:860K smpl:14M ep:71K epch:236.76 loss:0.001 grdn:0.111 lr:5.0e-07 updt_s:0.156 data_s:0.089 +INFO 2025-05-08 00:26:31 ts/train.py:232 step:860K smpl:14M ep:71K epch:236.82 loss:0.001 grdn:0.091 lr:4.9e-07 updt_s:0.157 data_s:0.090 +INFO 2025-05-08 00:27:21 ts/train.py:232 step:860K smpl:14M ep:71K epch:236.87 loss:0.001 grdn:0.122 lr:4.9e-07 updt_s:0.157 data_s:0.090 +INFO 2025-05-08 00:28:11 ts/train.py:232 step:860K smpl:14M ep:71K epch:236.93 loss:0.001 grdn:0.125 lr:4.8e-07 updt_s:0.156 data_s:0.093 +INFO 2025-05-08 00:28:59 ts/train.py:232 step:860K smpl:14M ep:71K epch:236.98 loss:0.001 grdn:0.109 lr:4.8e-07 updt_s:0.156 data_s:0.086 +INFO 2025-05-08 00:29:49 ts/train.py:232 step:861K smpl:14M ep:71K epch:237.04 loss:0.001 grdn:0.121 lr:4.8e-07 updt_s:0.157 data_s:0.090 +INFO 2025-05-08 00:30:39 ts/train.py:232 step:861K smpl:14M ep:71K epch:237.09 loss:0.001 grdn:0.106 lr:4.7e-07 updt_s:0.156 data_s:0.093 +INFO 2025-05-08 00:31:26 ts/train.py:232 step:861K smpl:14M ep:71K epch:237.15 loss:0.001 grdn:0.118 lr:4.7e-07 updt_s:0.156 data_s:0.080 +INFO 2025-05-08 00:32:16 ts/train.py:232 step:861K smpl:14M ep:71K epch:237.20 loss:0.001 grdn:0.124 lr:4.6e-07 updt_s:0.157 data_s:0.092 +INFO 2025-05-08 00:33:02 ts/train.py:232 step:861K smpl:14M ep:71K epch:237.26 loss:0.001 grdn:0.101 lr:4.6e-07 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 00:33:49 ts/train.py:232 step:862K smpl:14M ep:71K epch:237.31 loss:0.001 grdn:0.124 lr:4.5e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 00:34:36 ts/train.py:232 step:862K smpl:14M ep:71K epch:237.37 loss:0.001 grdn:0.107 lr:4.5e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 00:35:22 ts/train.py:232 step:862K smpl:14M ep:71K epch:237.42 loss:0.001 grdn:0.116 lr:4.4e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 00:36:08 ts/train.py:232 step:862K smpl:14M ep:71K epch:237.48 loss:0.001 grdn:0.112 lr:4.4e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 00:36:56 ts/train.py:232 step:862K smpl:14M ep:71K epch:237.53 loss:0.001 grdn:0.106 lr:4.3e-07 updt_s:0.158 data_s:0.080 +INFO 2025-05-08 00:37:42 ts/train.py:232 step:863K smpl:14M ep:71K epch:237.59 loss:0.001 grdn:0.116 lr:4.3e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 00:38:29 ts/train.py:232 step:863K smpl:14M ep:71K epch:237.65 loss:0.001 grdn:0.107 lr:4.2e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 00:39:16 ts/train.py:232 step:863K smpl:14M ep:71K epch:237.70 loss:0.001 grdn:0.102 lr:4.2e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 00:40:02 ts/train.py:232 step:863K smpl:14M ep:71K epch:237.76 loss:0.001 grdn:0.102 lr:4.1e-07 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 00:40:49 ts/train.py:232 step:863K smpl:14M ep:71K epch:237.81 loss:0.001 grdn:0.106 lr:4.1e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 00:41:35 ts/train.py:232 step:864K smpl:14M ep:71K epch:237.87 loss:0.001 grdn:0.113 lr:4.1e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 00:42:21 ts/train.py:232 step:864K smpl:14M ep:71K epch:237.92 loss:0.001 grdn:0.099 lr:4.0e-07 updt_s:0.157 data_s:0.072 +INFO 2025-05-08 00:43:07 ts/train.py:232 step:864K smpl:14M ep:71K epch:237.98 loss:0.001 grdn:0.113 lr:4.0e-07 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 00:43:53 ts/train.py:232 step:864K smpl:14M ep:71K epch:238.03 loss:0.001 grdn:0.118 lr:3.9e-07 updt_s:0.157 data_s:0.072 +INFO 2025-05-08 00:44:40 ts/train.py:232 step:864K smpl:14M ep:71K epch:238.09 loss:0.001 grdn:0.116 lr:3.9e-07 updt_s:0.157 data_s:0.079 +INFO 2025-05-08 00:45:29 ts/train.py:232 step:865K smpl:14M ep:71K epch:238.14 loss:0.001 grdn:0.106 lr:3.8e-07 updt_s:0.156 data_s:0.087 +INFO 2025-05-08 00:46:16 ts/train.py:232 step:865K smpl:14M ep:71K epch:238.20 loss:0.001 grdn:0.107 lr:3.8e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 00:47:02 ts/train.py:232 step:865K smpl:14M ep:71K epch:238.25 loss:0.001 grdn:0.104 lr:3.8e-07 updt_s:0.158 data_s:0.076 +INFO 2025-05-08 00:47:51 ts/train.py:232 step:865K smpl:14M ep:71K epch:238.31 loss:0.001 grdn:0.102 lr:3.7e-07 updt_s:0.157 data_s:0.086 +INFO 2025-05-08 00:48:38 ts/train.py:232 step:865K smpl:14M ep:72K epch:238.36 loss:0.001 grdn:0.109 lr:3.7e-07 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 00:49:25 ts/train.py:232 step:866K smpl:14M ep:72K epch:238.42 loss:0.001 grdn:0.108 lr:3.6e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 00:50:12 ts/train.py:232 step:866K smpl:14M ep:72K epch:238.47 loss:0.001 grdn:0.116 lr:3.6e-07 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 00:50:59 ts/train.py:232 step:866K smpl:14M ep:72K epch:238.53 loss:0.001 grdn:0.116 lr:3.5e-07 updt_s:0.157 data_s:0.076 +INFO 2025-05-08 00:51:45 ts/train.py:232 step:866K smpl:14M ep:72K epch:238.58 loss:0.001 grdn:0.130 lr:3.5e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 00:52:31 ts/train.py:232 step:866K smpl:14M ep:72K epch:238.64 loss:0.001 grdn:0.114 lr:3.5e-07 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 00:53:18 ts/train.py:232 step:867K smpl:14M ep:72K epch:238.69 loss:0.001 grdn:0.129 lr:3.4e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 00:54:05 ts/train.py:232 step:867K smpl:14M ep:72K epch:238.75 loss:0.001 grdn:0.108 lr:3.4e-07 updt_s:0.157 data_s:0.076 +INFO 2025-05-08 00:54:53 ts/train.py:232 step:867K smpl:14M ep:72K epch:238.80 loss:0.001 grdn:0.102 lr:3.3e-07 updt_s:0.157 data_s:0.082 +INFO 2025-05-08 00:55:41 ts/train.py:232 step:867K smpl:14M ep:72K epch:238.86 loss:0.001 grdn:0.121 lr:3.3e-07 updt_s:0.157 data_s:0.082 +INFO 2025-05-08 00:56:28 ts/train.py:232 step:867K smpl:14M ep:72K epch:238.91 loss:0.001 grdn:0.113 lr:3.3e-07 updt_s:0.157 data_s:0.079 +INFO 2025-05-08 00:57:15 ts/train.py:232 step:868K smpl:14M ep:72K epch:238.97 loss:0.001 grdn:0.114 lr:3.2e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 00:58:02 ts/train.py:232 step:868K smpl:14M ep:72K epch:239.02 loss:0.001 grdn:0.105 lr:3.2e-07 updt_s:0.157 data_s:0.076 +INFO 2025-05-08 00:58:48 ts/train.py:232 step:868K smpl:14M ep:72K epch:239.08 loss:0.001 grdn:0.115 lr:3.1e-07 updt_s:0.156 data_s:0.075 +INFO 2025-05-08 00:59:39 ts/train.py:232 step:868K smpl:14M ep:72K epch:239.13 loss:0.001 grdn:0.127 lr:3.1e-07 updt_s:0.157 data_s:0.094 +INFO 2025-05-08 01:00:27 ts/train.py:232 step:868K smpl:14M ep:72K epch:239.19 loss:0.001 grdn:0.110 lr:3.1e-07 updt_s:0.157 data_s:0.087 +INFO 2025-05-08 01:01:16 ts/train.py:232 step:869K smpl:14M ep:72K epch:239.24 loss:0.001 grdn:0.100 lr:3.0e-07 updt_s:0.157 data_s:0.086 +INFO 2025-05-08 01:02:03 ts/train.py:232 step:869K smpl:14M ep:72K epch:239.30 loss:0.001 grdn:0.113 lr:3.0e-07 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 01:02:49 ts/train.py:232 step:869K smpl:14M ep:72K epch:239.35 loss:0.001 grdn:0.124 lr:2.9e-07 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 01:03:34 ts/train.py:232 step:869K smpl:14M ep:72K epch:239.41 loss:0.001 grdn:0.105 lr:2.9e-07 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 01:04:20 ts/train.py:232 step:869K smpl:14M ep:72K epch:239.46 loss:0.001 grdn:0.124 lr:2.9e-07 updt_s:0.157 data_s:0.072 +INFO 2025-05-08 01:05:08 ts/train.py:232 step:870K smpl:14M ep:72K epch:239.52 loss:0.001 grdn:0.101 lr:2.8e-07 updt_s:0.157 data_s:0.081 +INFO 2025-05-08 01:05:56 ts/train.py:232 step:870K smpl:14M ep:72K epch:239.57 loss:0.001 grdn:0.103 lr:2.8e-07 updt_s:0.157 data_s:0.081 +INFO 2025-05-08 01:06:43 ts/train.py:232 step:870K smpl:14M ep:72K epch:239.63 loss:0.001 grdn:0.113 lr:2.8e-07 updt_s:0.157 data_s:0.079 +INFO 2025-05-08 01:06:43 ts/train.py:241 Checkpoint policy after step 870000 +INFO 2025-05-08 01:07:37 ts/train.py:232 step:870K smpl:14M ep:72K epch:239.68 loss:0.001 grdn:0.102 lr:2.7e-07 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 01:08:22 ts/train.py:232 step:870K smpl:14M ep:72K epch:239.74 loss:0.001 grdn:0.128 lr:2.7e-07 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 01:09:08 ts/train.py:232 step:871K smpl:14M ep:72K epch:239.79 loss:0.001 grdn:0.108 lr:2.7e-07 updt_s:0.157 data_s:0.070 +INFO 2025-05-08 01:09:54 ts/train.py:232 step:871K smpl:14M ep:72K epch:239.85 loss:0.001 grdn:0.126 lr:2.6e-07 updt_s:0.157 data_s:0.072 +INFO 2025-05-08 01:10:39 ts/train.py:232 step:871K smpl:14M ep:72K epch:239.90 loss:0.001 grdn:0.092 lr:2.6e-07 updt_s:0.157 data_s:0.068 +INFO 2025-05-08 01:11:25 ts/train.py:232 step:871K smpl:14M ep:72K epch:239.96 loss:0.001 grdn:0.119 lr:2.5e-07 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 01:12:10 ts/train.py:232 step:871K smpl:14M ep:72K epch:240.01 loss:0.001 grdn:0.099 lr:2.5e-07 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 01:13:00 ts/train.py:232 step:872K smpl:14M ep:72K epch:240.07 loss:0.001 grdn:0.100 lr:2.5e-07 updt_s:0.156 data_s:0.089 +INFO 2025-05-08 01:13:48 ts/train.py:232 step:872K smpl:14M ep:72K epch:240.12 loss:0.001 grdn:0.119 lr:2.4e-07 updt_s:0.157 data_s:0.084 +INFO 2025-05-08 01:14:33 ts/train.py:232 step:872K smpl:14M ep:72K epch:240.18 loss:0.001 grdn:0.103 lr:2.4e-07 updt_s:0.157 data_s:0.069 +INFO 2025-05-08 01:15:22 ts/train.py:232 step:872K smpl:14M ep:72K epch:240.23 loss:0.001 grdn:0.098 lr:2.4e-07 updt_s:0.157 data_s:0.086 +INFO 2025-05-08 01:16:08 ts/train.py:232 step:872K smpl:14M ep:72K epch:240.29 loss:0.001 grdn:0.121 lr:2.3e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 01:16:55 ts/train.py:232 step:873K smpl:14M ep:72K epch:240.34 loss:0.001 grdn:0.107 lr:2.3e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 01:17:40 ts/train.py:232 step:873K smpl:14M ep:72K epch:240.40 loss:0.001 grdn:0.115 lr:2.3e-07 updt_s:0.157 data_s:0.069 +INFO 2025-05-08 01:18:25 ts/train.py:232 step:873K smpl:14M ep:72K epch:240.45 loss:0.001 grdn:0.110 lr:2.2e-07 updt_s:0.157 data_s:0.070 +INFO 2025-05-08 01:19:10 ts/train.py:232 step:873K smpl:14M ep:72K epch:240.51 loss:0.001 grdn:0.117 lr:2.2e-07 updt_s:0.157 data_s:0.065 +INFO 2025-05-08 01:19:56 ts/train.py:232 step:873K smpl:14M ep:72K epch:240.56 loss:0.001 grdn:0.104 lr:2.2e-07 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 01:20:41 ts/train.py:232 step:874K smpl:14M ep:72K epch:240.62 loss:0.001 grdn:0.119 lr:2.1e-07 updt_s:0.157 data_s:0.072 +INFO 2025-05-08 01:21:27 ts/train.py:232 step:874K smpl:14M ep:72K epch:240.67 loss:0.001 grdn:0.098 lr:2.1e-07 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 01:22:15 ts/train.py:232 step:874K smpl:14M ep:72K epch:240.73 loss:0.001 grdn:0.117 lr:2.1e-07 updt_s:0.157 data_s:0.083 +INFO 2025-05-08 01:23:02 ts/train.py:232 step:874K smpl:14M ep:72K epch:240.78 loss:0.001 grdn:0.114 lr:2.0e-07 updt_s:0.158 data_s:0.072 +INFO 2025-05-08 01:23:48 ts/train.py:232 step:874K smpl:14M ep:72K epch:240.84 loss:0.001 grdn:0.102 lr:2.0e-07 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 01:24:33 ts/train.py:232 step:875K smpl:14M ep:72K epch:240.90 loss:0.001 grdn:0.116 lr:2.0e-07 updt_s:0.157 data_s:0.067 +INFO 2025-05-08 01:25:19 ts/train.py:232 step:875K smpl:14M ep:72K epch:240.95 loss:0.001 grdn:0.109 lr:2.0e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 01:26:05 ts/train.py:232 step:875K smpl:14M ep:72K epch:241.01 loss:0.001 grdn:0.103 lr:1.9e-07 updt_s:0.156 data_s:0.072 +INFO 2025-05-08 01:26:53 ts/train.py:232 step:875K smpl:14M ep:72K epch:241.06 loss:0.001 grdn:0.116 lr:1.9e-07 updt_s:0.157 data_s:0.086 +INFO 2025-05-08 01:27:40 ts/train.py:232 step:875K smpl:14M ep:72K epch:241.12 loss:0.001 grdn:0.114 lr:1.9e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 01:28:26 ts/train.py:232 step:876K smpl:14M ep:72K epch:241.17 loss:0.001 grdn:0.119 lr:1.8e-07 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 01:29:13 ts/train.py:232 step:876K smpl:14M ep:72K epch:241.23 loss:0.001 grdn:0.114 lr:1.8e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 01:29:59 ts/train.py:232 step:876K smpl:14M ep:72K epch:241.28 loss:0.001 grdn:0.110 lr:1.8e-07 updt_s:0.157 data_s:0.076 +INFO 2025-05-08 01:30:45 ts/train.py:232 step:876K smpl:14M ep:72K epch:241.34 loss:0.001 grdn:0.105 lr:1.7e-07 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 01:31:32 ts/train.py:232 step:876K smpl:14M ep:72K epch:241.39 loss:0.001 grdn:0.102 lr:1.7e-07 updt_s:0.157 data_s:0.076 +INFO 2025-05-08 01:32:18 ts/train.py:232 step:877K smpl:14M ep:72K epch:241.45 loss:0.001 grdn:0.113 lr:1.7e-07 updt_s:0.158 data_s:0.072 +INFO 2025-05-08 01:33:05 ts/train.py:232 step:877K smpl:14M ep:72K epch:241.50 loss:0.001 grdn:0.111 lr:1.7e-07 updt_s:0.157 data_s:0.076 +INFO 2025-05-08 01:33:51 ts/train.py:232 step:877K smpl:14M ep:72K epch:241.56 loss:0.001 grdn:0.120 lr:1.6e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 01:34:38 ts/train.py:232 step:877K smpl:14M ep:72K epch:241.61 loss:0.001 grdn:0.129 lr:1.6e-07 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 01:35:25 ts/train.py:232 step:877K smpl:14M ep:72K epch:241.67 loss:0.001 grdn:0.116 lr:1.6e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 01:36:11 ts/train.py:232 step:878K smpl:14M ep:73K epch:241.72 loss:0.001 grdn:0.107 lr:1.5e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 01:36:58 ts/train.py:232 step:878K smpl:14M ep:73K epch:241.78 loss:0.001 grdn:0.108 lr:1.5e-07 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 01:37:45 ts/train.py:232 step:878K smpl:14M ep:73K epch:241.83 loss:0.001 grdn:0.114 lr:1.5e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 01:38:32 ts/train.py:232 step:878K smpl:14M ep:73K epch:241.89 loss:0.001 grdn:0.112 lr:1.5e-07 updt_s:0.157 data_s:0.080 +INFO 2025-05-08 01:39:19 ts/train.py:232 step:878K smpl:14M ep:73K epch:241.94 loss:0.001 grdn:0.106 lr:1.4e-07 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 01:40:06 ts/train.py:232 step:879K smpl:14M ep:73K epch:242.00 loss:0.001 grdn:0.111 lr:1.4e-07 updt_s:0.156 data_s:0.079 +INFO 2025-05-08 01:40:50 ts/train.py:232 step:879K smpl:14M ep:73K epch:242.05 loss:0.001 grdn:0.121 lr:1.4e-07 updt_s:0.158 data_s:0.061 +INFO 2025-05-08 01:41:34 ts/train.py:232 step:879K smpl:14M ep:73K epch:242.11 loss:0.001 grdn:0.096 lr:1.4e-07 updt_s:0.157 data_s:0.060 +INFO 2025-05-08 01:42:21 ts/train.py:232 step:879K smpl:14M ep:73K epch:242.16 loss:0.001 grdn:0.131 lr:1.3e-07 updt_s:0.158 data_s:0.075 +INFO 2025-05-08 01:43:05 ts/train.py:232 step:879K smpl:14M ep:73K epch:242.22 loss:0.001 grdn:0.119 lr:1.3e-07 updt_s:0.157 data_s:0.064 +INFO 2025-05-08 01:43:49 ts/train.py:232 step:880K smpl:14M ep:73K epch:242.27 loss:0.001 grdn:0.138 lr:1.3e-07 updt_s:0.157 data_s:0.062 +INFO 2025-05-08 01:44:33 ts/train.py:232 step:880K smpl:14M ep:73K epch:242.33 loss:0.001 grdn:0.098 lr:1.3e-07 updt_s:0.157 data_s:0.062 +INFO 2025-05-08 01:45:16 ts/train.py:232 step:880K smpl:14M ep:73K epch:242.38 loss:0.001 grdn:0.104 lr:1.2e-07 updt_s:0.157 data_s:0.061 +INFO 2025-05-08 01:46:01 ts/train.py:232 step:880K smpl:14M ep:73K epch:242.44 loss:0.001 grdn:0.103 lr:1.2e-07 updt_s:0.157 data_s:0.063 +INFO 2025-05-08 01:46:44 ts/train.py:232 step:880K smpl:14M ep:73K epch:242.49 loss:0.001 grdn:0.115 lr:1.2e-07 updt_s:0.157 data_s:0.060 +INFO 2025-05-08 01:47:28 ts/train.py:232 step:881K smpl:14M ep:73K epch:242.55 loss:0.001 grdn:0.105 lr:1.2e-07 updt_s:0.157 data_s:0.060 +INFO 2025-05-08 01:48:12 ts/train.py:232 step:881K smpl:14M ep:73K epch:242.60 loss:0.001 grdn:0.111 lr:1.1e-07 updt_s:0.157 data_s:0.062 +INFO 2025-05-08 01:48:55 ts/train.py:232 step:881K smpl:14M ep:73K epch:242.66 loss:0.001 grdn:0.111 lr:1.1e-07 updt_s:0.157 data_s:0.061 +INFO 2025-05-08 01:49:39 ts/train.py:232 step:881K smpl:14M ep:73K epch:242.71 loss:0.001 grdn:0.110 lr:1.1e-07 updt_s:0.157 data_s:0.061 +INFO 2025-05-08 01:50:23 ts/train.py:232 step:881K smpl:14M ep:73K epch:242.77 loss:0.001 grdn:0.099 lr:1.1e-07 updt_s:0.157 data_s:0.063 +INFO 2025-05-08 01:51:07 ts/train.py:232 step:882K smpl:14M ep:73K epch:242.82 loss:0.001 grdn:0.105 lr:1.0e-07 updt_s:0.157 data_s:0.062 +INFO 2025-05-08 01:51:51 ts/train.py:232 step:882K smpl:14M ep:73K epch:242.88 loss:0.001 grdn:0.113 lr:1.0e-07 updt_s:0.157 data_s:0.061 +INFO 2025-05-08 01:52:35 ts/train.py:232 step:882K smpl:14M ep:73K epch:242.93 loss:0.001 grdn:0.109 lr:1.0e-07 updt_s:0.156 data_s:0.063 +INFO 2025-05-08 01:53:24 ts/train.py:232 step:882K smpl:14M ep:73K epch:242.99 loss:0.001 grdn:0.113 lr:9.8e-08 updt_s:0.157 data_s:0.087 +INFO 2025-05-08 01:54:10 ts/train.py:232 step:882K smpl:14M ep:73K epch:243.04 loss:0.001 grdn:0.101 lr:9.6e-08 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 01:54:57 ts/train.py:232 step:883K smpl:14M ep:73K epch:243.10 loss:0.001 grdn:0.109 lr:9.3e-08 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 01:55:43 ts/train.py:232 step:883K smpl:14M ep:73K epch:243.15 loss:0.001 grdn:0.119 lr:9.1e-08 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 01:56:30 ts/train.py:232 step:883K smpl:14M ep:73K epch:243.21 loss:0.001 grdn:0.111 lr:8.9e-08 updt_s:0.157 data_s:0.076 +INFO 2025-05-08 01:57:16 ts/train.py:232 step:883K smpl:14M ep:73K epch:243.26 loss:0.001 grdn:0.109 lr:8.7e-08 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 01:58:03 ts/train.py:232 step:883K smpl:14M ep:73K epch:243.32 loss:0.001 grdn:0.113 lr:8.5e-08 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 01:58:50 ts/train.py:232 step:884K smpl:14M ep:73K epch:243.37 loss:0.001 grdn:0.110 lr:8.3e-08 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 01:59:36 ts/train.py:232 step:884K smpl:14M ep:73K epch:243.43 loss:0.001 grdn:0.107 lr:8.1e-08 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 02:00:23 ts/train.py:232 step:884K smpl:14M ep:73K epch:243.48 loss:0.001 grdn:0.111 lr:7.9e-08 updt_s:0.157 data_s:0.076 +INFO 2025-05-08 02:01:09 ts/train.py:232 step:884K smpl:14M ep:73K epch:243.54 loss:0.001 grdn:0.108 lr:7.7e-08 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 02:01:56 ts/train.py:232 step:884K smpl:14M ep:73K epch:243.59 loss:0.001 grdn:0.118 lr:7.5e-08 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 02:02:42 ts/train.py:232 step:885K smpl:14M ep:73K epch:243.65 loss:0.001 grdn:0.120 lr:7.3e-08 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 02:03:28 ts/train.py:232 step:885K smpl:14M ep:73K epch:243.70 loss:0.001 grdn:0.104 lr:7.1e-08 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 02:04:14 ts/train.py:232 step:885K smpl:14M ep:73K epch:243.76 loss:0.001 grdn:0.121 lr:7.0e-08 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 02:05:01 ts/train.py:232 step:885K smpl:14M ep:73K epch:243.81 loss:0.001 grdn:0.104 lr:6.8e-08 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 02:05:48 ts/train.py:232 step:885K smpl:14M ep:73K epch:243.87 loss:0.001 grdn:0.096 lr:6.6e-08 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 02:06:38 ts/train.py:232 step:886K smpl:14M ep:73K epch:243.92 loss:0.001 grdn:0.114 lr:6.4e-08 updt_s:0.156 data_s:0.093 +INFO 2025-05-08 02:07:26 ts/train.py:232 step:886K smpl:14M ep:73K epch:243.98 loss:0.001 grdn:0.119 lr:6.2e-08 updt_s:0.157 data_s:0.082 +INFO 2025-05-08 02:08:14 ts/train.py:232 step:886K smpl:14M ep:73K epch:244.04 loss:0.001 grdn:0.108 lr:6.1e-08 updt_s:0.157 data_s:0.084 +INFO 2025-05-08 02:09:01 ts/train.py:232 step:886K smpl:14M ep:73K epch:244.09 loss:0.001 grdn:0.119 lr:5.9e-08 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 02:09:49 ts/train.py:232 step:886K smpl:14M ep:73K epch:244.15 loss:0.001 grdn:0.105 lr:5.7e-08 updt_s:0.157 data_s:0.083 +INFO 2025-05-08 02:10:38 ts/train.py:232 step:887K smpl:14M ep:73K epch:244.20 loss:0.001 grdn:0.098 lr:5.6e-08 updt_s:0.157 data_s:0.088 +INFO 2025-05-08 02:11:27 ts/train.py:232 step:887K smpl:14M ep:73K epch:244.26 loss:0.001 grdn:0.113 lr:5.4e-08 updt_s:0.157 data_s:0.085 +INFO 2025-05-08 02:12:15 ts/train.py:232 step:887K smpl:14M ep:73K epch:244.31 loss:0.001 grdn:0.098 lr:5.2e-08 updt_s:0.157 data_s:0.082 +INFO 2025-05-08 02:13:02 ts/train.py:232 step:887K smpl:14M ep:73K epch:244.37 loss:0.001 grdn:0.098 lr:5.1e-08 updt_s:0.157 data_s:0.082 +INFO 2025-05-08 02:13:51 ts/train.py:232 step:887K smpl:14M ep:73K epch:244.42 loss:0.001 grdn:0.101 lr:4.9e-08 updt_s:0.157 data_s:0.083 +INFO 2025-05-08 02:14:39 ts/train.py:232 step:888K smpl:14M ep:73K epch:244.48 loss:0.001 grdn:0.113 lr:4.8e-08 updt_s:0.157 data_s:0.084 +INFO 2025-05-08 02:15:26 ts/train.py:232 step:888K smpl:14M ep:73K epch:244.53 loss:0.001 grdn:0.104 lr:4.6e-08 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 02:16:13 ts/train.py:232 step:888K smpl:14M ep:73K epch:244.59 loss:0.001 grdn:0.106 lr:4.5e-08 updt_s:0.157 data_s:0.079 +INFO 2025-05-08 02:17:00 ts/train.py:232 step:888K smpl:14M ep:73K epch:244.64 loss:0.001 grdn:0.111 lr:4.3e-08 updt_s:0.157 data_s:0.079 +INFO 2025-05-08 02:17:47 ts/train.py:232 step:888K smpl:14M ep:73K epch:244.70 loss:0.001 grdn:0.103 lr:4.2e-08 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 02:18:33 ts/train.py:232 step:889K smpl:14M ep:73K epch:244.75 loss:0.001 grdn:0.103 lr:4.0e-08 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 02:19:20 ts/train.py:232 step:889K smpl:14M ep:73K epch:244.81 loss:0.001 grdn:0.109 lr:3.9e-08 updt_s:0.158 data_s:0.075 +INFO 2025-05-08 02:20:06 ts/train.py:232 step:889K smpl:14M ep:73K epch:244.86 loss:0.001 grdn:0.112 lr:3.8e-08 updt_s:0.157 data_s:0.072 +INFO 2025-05-08 02:20:54 ts/train.py:232 step:889K smpl:14M ep:73K epch:244.92 loss:0.001 grdn:0.092 lr:3.6e-08 updt_s:0.157 data_s:0.083 +INFO 2025-05-08 02:21:40 ts/train.py:232 step:889K smpl:14M ep:73K epch:244.97 loss:0.001 grdn:0.117 lr:3.5e-08 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 02:22:26 ts/train.py:232 step:890K smpl:14M ep:74K epch:245.03 loss:0.001 grdn:0.109 lr:3.4e-08 updt_s:0.157 data_s:0.072 +INFO 2025-05-08 02:23:13 ts/train.py:232 step:890K smpl:14M ep:74K epch:245.08 loss:0.001 grdn:0.108 lr:3.2e-08 updt_s:0.157 data_s:0.079 +INFO 2025-05-08 02:23:58 ts/train.py:232 step:890K smpl:14M ep:74K epch:245.14 loss:0.001 grdn:0.109 lr:3.1e-08 updt_s:0.157 data_s:0.068 +INFO 2025-05-08 02:24:44 ts/train.py:232 step:890K smpl:14M ep:74K epch:245.19 loss:0.001 grdn:0.122 lr:3.0e-08 updt_s:0.157 data_s:0.072 +INFO 2025-05-08 02:25:30 ts/train.py:232 step:890K smpl:14M ep:74K epch:245.25 loss:0.001 grdn:0.119 lr:2.9e-08 updt_s:0.158 data_s:0.072 +INFO 2025-05-08 02:26:16 ts/train.py:232 step:891K smpl:14M ep:74K epch:245.30 loss:0.001 grdn:0.111 lr:2.8e-08 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 02:27:02 ts/train.py:232 step:891K smpl:14M ep:74K epch:245.36 loss:0.001 grdn:0.099 lr:2.6e-08 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 02:27:48 ts/train.py:232 step:891K smpl:14M ep:74K epch:245.41 loss:0.001 grdn:0.116 lr:2.5e-08 updt_s:0.159 data_s:0.067 +INFO 2025-05-08 02:28:34 ts/train.py:232 step:891K smpl:14M ep:74K epch:245.47 loss:0.001 grdn:0.131 lr:2.4e-08 updt_s:0.157 data_s:0.074 +INFO 2025-05-08 02:29:20 ts/train.py:232 step:891K smpl:14M ep:74K epch:245.52 loss:0.001 grdn:0.118 lr:2.3e-08 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 02:30:06 ts/train.py:232 step:892K smpl:14M ep:74K epch:245.58 loss:0.001 grdn:0.114 lr:2.2e-08 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 02:30:53 ts/train.py:232 step:892K smpl:14M ep:74K epch:245.63 loss:0.001 grdn:0.097 lr:2.1e-08 updt_s:0.157 data_s:0.078 +INFO 2025-05-08 02:31:39 ts/train.py:232 step:892K smpl:14M ep:74K epch:245.69 loss:0.001 grdn:0.103 lr:2.0e-08 updt_s:0.157 data_s:0.072 +INFO 2025-05-08 02:32:24 ts/train.py:232 step:892K smpl:14M ep:74K epch:245.74 loss:0.001 grdn:0.112 lr:1.9e-08 updt_s:0.157 data_s:0.064 +INFO 2025-05-08 02:33:09 ts/train.py:232 step:892K smpl:14M ep:74K epch:245.80 loss:0.001 grdn:0.120 lr:1.8e-08 updt_s:0.157 data_s:0.070 +INFO 2025-05-08 02:33:55 ts/train.py:232 step:893K smpl:14M ep:74K epch:245.85 loss:0.001 grdn:0.120 lr:1.7e-08 updt_s:0.156 data_s:0.073 +INFO 2025-05-08 02:34:38 ts/train.py:232 step:893K smpl:14M ep:74K epch:245.91 loss:0.001 grdn:0.111 lr:1.6e-08 updt_s:0.157 data_s:0.057 +INFO 2025-05-08 02:35:20 ts/train.py:232 step:893K smpl:14M ep:74K epch:245.96 loss:0.001 grdn:0.114 lr:1.5e-08 updt_s:0.157 data_s:0.053 +INFO 2025-05-08 02:36:03 ts/train.py:232 step:893K smpl:14M ep:74K epch:246.02 loss:0.001 grdn:0.111 lr:1.5e-08 updt_s:0.157 data_s:0.055 +INFO 2025-05-08 02:36:45 ts/train.py:232 step:893K smpl:14M ep:74K epch:246.07 loss:0.001 grdn:0.107 lr:1.4e-08 updt_s:0.157 data_s:0.053 +INFO 2025-05-08 02:37:29 ts/train.py:232 step:894K smpl:14M ep:74K epch:246.13 loss:0.001 grdn:0.119 lr:1.3e-08 updt_s:0.158 data_s:0.061 +INFO 2025-05-08 02:38:15 ts/train.py:232 step:894K smpl:14M ep:74K epch:246.18 loss:0.001 grdn:0.115 lr:1.2e-08 updt_s:0.157 data_s:0.075 +INFO 2025-05-08 02:39:01 ts/train.py:232 step:894K smpl:14M ep:74K epch:246.24 loss:0.001 grdn:0.104 lr:1.1e-08 updt_s:0.157 data_s:0.068 +INFO 2025-05-08 02:39:43 ts/train.py:232 step:894K smpl:14M ep:74K epch:246.29 loss:0.001 grdn:0.119 lr:1.1e-08 updt_s:0.157 data_s:0.056 +INFO 2025-05-08 02:40:26 ts/train.py:232 step:894K smpl:14M ep:74K epch:246.35 loss:0.001 grdn:0.116 lr:9.9e-09 updt_s:0.158 data_s:0.056 +INFO 2025-05-08 02:41:09 ts/train.py:232 step:895K smpl:14M ep:74K epch:246.40 loss:0.001 grdn:0.101 lr:9.2e-09 updt_s:0.157 data_s:0.054 +INFO 2025-05-08 02:41:52 ts/train.py:232 step:895K smpl:14M ep:74K epch:246.46 loss:0.001 grdn:0.105 lr:8.6e-09 updt_s:0.157 data_s:0.057 +INFO 2025-05-08 02:42:34 ts/train.py:232 step:895K smpl:14M ep:74K epch:246.51 loss:0.001 grdn:0.102 lr:7.9e-09 updt_s:0.157 data_s:0.054 +INFO 2025-05-08 02:43:17 ts/train.py:232 step:895K smpl:14M ep:74K epch:246.57 loss:0.001 grdn:0.106 lr:7.3e-09 updt_s:0.157 data_s:0.057 +INFO 2025-05-08 02:44:00 ts/train.py:232 step:895K smpl:14M ep:74K epch:246.62 loss:0.001 grdn:0.105 lr:6.7e-09 updt_s:0.157 data_s:0.055 +INFO 2025-05-08 02:44:42 ts/train.py:232 step:896K smpl:14M ep:74K epch:246.68 loss:0.001 grdn:0.123 lr:6.2e-09 updt_s:0.157 data_s:0.054 +INFO 2025-05-08 02:45:25 ts/train.py:232 step:896K smpl:14M ep:74K epch:246.73 loss:0.001 grdn:0.104 lr:5.6e-09 updt_s:0.157 data_s:0.055 +INFO 2025-05-08 02:46:08 ts/train.py:232 step:896K smpl:14M ep:74K epch:246.79 loss:0.001 grdn:0.112 lr:5.1e-09 updt_s:0.157 data_s:0.062 +INFO 2025-05-08 02:46:57 ts/train.py:232 step:896K smpl:14M ep:74K epch:246.84 loss:0.001 grdn:0.128 lr:4.6e-09 updt_s:0.158 data_s:0.084 +INFO 2025-05-08 02:47:42 ts/train.py:232 step:896K smpl:14M ep:74K epch:246.90 loss:0.001 grdn:0.109 lr:4.2e-09 updt_s:0.157 data_s:0.067 +INFO 2025-05-08 02:48:27 ts/train.py:232 step:897K smpl:14M ep:74K epch:246.95 loss:0.001 grdn:0.100 lr:3.7e-09 updt_s:0.157 data_s:0.068 +INFO 2025-05-08 02:49:12 ts/train.py:232 step:897K smpl:14M ep:74K epch:247.01 loss:0.001 grdn:0.105 lr:3.3e-09 updt_s:0.157 data_s:0.070 +INFO 2025-05-08 02:49:57 ts/train.py:232 step:897K smpl:14M ep:74K epch:247.06 loss:0.001 grdn:0.114 lr:2.9e-09 updt_s:0.157 data_s:0.066 +INFO 2025-05-08 02:50:42 ts/train.py:232 step:897K smpl:14M ep:74K epch:247.12 loss:0.001 grdn:0.097 lr:2.6e-09 updt_s:0.157 data_s:0.067 +INFO 2025-05-08 02:51:28 ts/train.py:232 step:897K smpl:14M ep:74K epch:247.18 loss:0.001 grdn:0.093 lr:2.2e-09 updt_s:0.157 data_s:0.069 +INFO 2025-05-08 02:52:13 ts/train.py:232 step:898K smpl:14M ep:74K epch:247.23 loss:0.001 grdn:0.111 lr:1.9e-09 updt_s:0.156 data_s:0.070 +INFO 2025-05-08 02:52:58 ts/train.py:232 step:898K smpl:14M ep:74K epch:247.29 loss:0.001 grdn:0.113 lr:1.6e-09 updt_s:0.157 data_s:0.066 +INFO 2025-05-08 02:53:43 ts/train.py:232 step:898K smpl:14M ep:74K epch:247.34 loss:0.001 grdn:0.111 lr:1.3e-09 updt_s:0.157 data_s:0.070 +INFO 2025-05-08 02:54:28 ts/train.py:232 step:898K smpl:14M ep:74K epch:247.40 loss:0.001 grdn:0.118 lr:1.1e-09 updt_s:0.157 data_s:0.068 +INFO 2025-05-08 02:55:15 ts/train.py:232 step:898K smpl:14M ep:74K epch:247.45 loss:0.001 grdn:0.115 lr:8.8e-10 updt_s:0.157 data_s:0.073 +INFO 2025-05-08 02:56:00 ts/train.py:232 step:899K smpl:14M ep:74K epch:247.51 loss:0.001 grdn:0.093 lr:6.9e-10 updt_s:0.157 data_s:0.067 +INFO 2025-05-08 02:56:45 ts/train.py:232 step:899K smpl:14M ep:74K epch:247.56 loss:0.001 grdn:0.097 lr:5.2e-10 updt_s:0.157 data_s:0.066 +INFO 2025-05-08 02:57:30 ts/train.py:232 step:899K smpl:14M ep:74K epch:247.62 loss:0.001 grdn:0.101 lr:3.7e-10 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 02:58:15 ts/train.py:232 step:899K smpl:14M ep:74K epch:247.67 loss:0.001 grdn:0.108 lr:2.5e-10 updt_s:0.157 data_s:0.068 +INFO 2025-05-08 02:59:01 ts/train.py:232 step:899K smpl:14M ep:74K epch:247.73 loss:0.001 grdn:0.097 lr:1.5e-10 updt_s:0.157 data_s:0.071 +INFO 2025-05-08 02:59:48 ts/train.py:232 step:900K smpl:14M ep:74K epch:247.78 loss:0.001 grdn:0.118 lr:7.7e-11 updt_s:0.156 data_s:0.077 +INFO 2025-05-08 03:00:35 ts/train.py:232 step:900K smpl:14M ep:74K epch:247.84 loss:0.001 grdn:0.105 lr:2.8e-11 updt_s:0.157 data_s:0.077 +INFO 2025-05-08 03:01:18 ts/train.py:232 step:900K smpl:14M ep:74K epch:247.89 loss:0.001 grdn:0.115 lr:4.0e-12 updt_s:0.157 data_s:0.059 +INFO 2025-05-08 03:01:18 ts/train.py:241 Checkpoint policy after step 900000 +INFO 2025-05-08 03:01:21 ts/train.py:283 End of training diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/wandb-summary.json b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..97f869cc0842c4a33adc4d511186e3cae355e85a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/samples":14400000,"train/steps":900000,"train/episodes":74367.36099156481,"_wandb":{"runtime":220724},"train/epochs":247.89120330521604,"train/grad_norm":0.1150171234481968,"train/update_s":0.15668027934618295,"train/dataloading_s":0.05906350366421975,"_timestamp":1.7466732785116746e+09,"train/lr":4.0356373227079544e-12,"train/loss":0.001142242236783204,"_step":900000,"_runtime":220724.889251413} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-core.log b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-core.log index c2331edbd30b477eb02d34f66b879bcab476ffc6..a7722971978d11f4fe8200384ceac3d171cb1f8b 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-core.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-core.log @@ -4,3 +4,11 @@ {"time":"2025-05-05T13:42:37.652236998Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:33318"} {"time":"2025-05-05T13:42:37.659541939Z","level":"INFO","msg":"handleInformInit: received","streamId":"92tvlwhj","id":"127.0.0.1:33318"} {"time":"2025-05-05T13:42:37.999648702Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"92tvlwhj","id":"127.0.0.1:33318"} +{"time":"2025-05-08T03:01:22.548085513Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:33318"} +{"time":"2025-05-08T03:01:22.548228293Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:33318"} +{"time":"2025-05-08T03:01:22.548279353Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-05-08T03:01:22.548356593Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:33318"} +{"time":"2025-05-08T03:01:41.739678319Z","level":"ERROR","msg":"processOutgoingData: flush error","error":"write tcp 127.0.0.1:41215->127.0.0.1:33318: use of closed network connection","id":"127.0.0.1:33318"} +{"time":"2025-05-08T03:01:42.071078615Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:33318"} +{"time":"2025-05-08T03:01:42.071109235Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:33318"} +{"time":"2025-05-08T03:01:42.071121055Z","level":"INFO","msg":"server is closed"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-internal.log b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-internal.log index 6dd46a936b68234dcfbedfd645937ba693897139..d93f4520937a44899889b6c25a81fecccb483edc 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug-internal.log @@ -13,3 +13,13 @@ {"time":"2025-05-06T10:36:28.34160936Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:59320->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T11:17:58.342345052Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49666->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T23:06:26.097134902Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} +{"time":"2025-05-07T16:56:18.625880571Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/92tvlwhj/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:48120->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-07T23:27:42.933986298Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 172.17.0.4:38206->35.186.228.49:443: read: connection reset by peer"} +{"time":"2025-05-08T03:01:22.548356473Z","level":"INFO","msg":"stream: closing","id":"92tvlwhj"} +{"time":"2025-05-08T03:01:22.548431522Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-05-08T03:01:22.548580182Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-05-08T03:01:41.789898019Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-05-08T03:01:42.069164311Z","level":"INFO","msg":"handler: closed","stream_id":"92tvlwhj"} +{"time":"2025-05-08T03:01:42.06924091Z","level":"INFO","msg":"sender: closed","stream_id":"92tvlwhj"} +{"time":"2025-05-08T03:01:42.06923903Z","level":"INFO","msg":"writer: Close: closed","stream_id":"92tvlwhj"} +{"time":"2025-05-08T03:01:42.06937473Z","level":"INFO","msg":"stream: closed","id":"92tvlwhj"} diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug.log b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug.log index 6ea2688767a75521b6667fd6ce61f64355ddcb0b..5daad7b257415bdef856df5de33094bbaec2a06e 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug.log +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/logs/debug.log @@ -20,3 +20,4 @@ config: {'dataset': {'repo_id': 'shylee/so100_pengripA', 'root': '/SSD/LSY/lerob 2025-05-05 13:42:38,419 INFO MainThread:99163 [wandb_run.py:_redirect():2371] Wrapping output streams. 2025-05-05 13:42:38,419 INFO MainThread:99163 [wandb_run.py:_redirect():2394] Redirects installed. 2025-05-05 13:42:38,420 INFO MainThread:99163 [wandb_init.py:init():1056] run started, returning control to user process +2025-05-08 03:01:22,547 INFO MsgRouterThr:99163 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles. diff --git a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb index 33948994927451d3bb289eb58a369ea8df858fd6..d4a55a87969314b3f7ded466e39af46908bd0089 100644 --- a/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb +++ b/DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:123be03fff83191a6b7d8a24991f304a0b3a57b672160aa6bd01491d133dd680 -size 39059456 +oid sha256:82b65ee440c4c9e92e66f636b68695fa7cf757afab3549713de6b71d68f775a3 +size 52190565