diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..529e7123f8d8bf48b99d1b773ef33f1c57f7efd1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bdbf9bf26c09160f6998bb8c30e39a708403a19982b4d2120334e67e3cfec60 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..a2ebd55b052e1e980a503765e694b0de8af480ec --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 8.720200055053977e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..60660f72a7b300d2c75a319803645dc3d8d77485 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d309778b2475b1af373d4b733027d9d74cf578b2d3ca96dfec02ef21cde8f162 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9e3968815529d98948da3c9d18ae94fd26a14bb5 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25054dbb5044f555604cb471ae674206455ee244dd05c8f7350c21c52aa64f6d +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..97ab942fad8a77c663b0037063c5951ba5dbc781 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 210000, + "verbose": false, + "_step_count": 210001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 8.720200055053977e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..6147d40ee6d82bb05c893a1b35d8f59f302da656 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/210000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 210000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cc856f71d614323e2b05965aa8fcce155fe2ee27 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63a879004bd9456a241cade2b6196071b2a8c02f3d0230187c821976001384c0 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..b973d80db1e60787fa3565daf1f4712fdb7b364e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 8.350408716128216e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e4f39194ab875391555c51f6de07699120f3fed4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9427247320e45739fbe8cd2c7529afef03b190b6a59e3fe26145d81598556ab +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d9cadb48c4523819f6d08bedc47a99237fa03f48 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fe8d6a2a3f36c3decee752a91800282faeffbdab6d1a00981634c6da1ce7097 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..20c00e1b8846ac18ae698337037774699743306a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 240000, + "verbose": false, + "_step_count": 240001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 8.350408716128216e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..cd776dcc291fdcc40079da5c594255ca04238bce --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/240000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 240000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b8560e38f8b3f5021258923215f4e3b35bc7c6c2 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:422244400050dc0f5ef1557ae60528525e3751558aafb61b729aafe6484818e6 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1085c71f91668de124c2f5269441008dc91a6665 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.943868815107594e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..241aebdfeb1970b6652a7dc18ebfddae070b0492 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9127467b7d89673d445a0f9302d694cb3fb2c23e8b52623c1e956c30c79b191e +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f1045fda35ae3e8b8b8021a41b8af855d40fa2f3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bec12d7e5fe4aa7967aafcf9e0bde003eb34912f1fa7ee2e8951034fa44a765 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e2fc5c0727d511fc139fe9e28a78e368a31b04b7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 270000, + "verbose": false, + "_step_count": 270001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.943868815107594e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..537f4e9f0bf2092976dffaa70a673dde4c3fd66d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/270000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 270000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f39ce260b9ab1b4a2db592baf8bb58a0825aa49a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9fb15217f7d8fd2fd8eaad727f83e22506b9b0750a4e5b99fa1eb585a7597f1 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..441b7eada7faf4a78ccaa1e14c1d8083b6e9035c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.505039436856827e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2089891235f30adeda9e23231a46b2778694e3d2 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7e0bf2cdef3d31de6cd0d28984491951874593c659f7e61e3e5f9d3d62892f6 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..afef1085ac2382824aa588a648ccf121aeff9f50 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7014000a6ab6c57ea9b868f3958996e23692519eb7903184e1affdcf0c9f8bd8 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..46f0924834037be23d642823ee826bc729c90d27 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 300000, + "verbose": false, + "_step_count": 300001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.505039436856827e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..db7e948ddab8257b846ad5ef28f8c555a9524598 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/300000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 300000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ef79762745a7f7ecbb0826a7303585a98a325f1d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:514220a15443cf6f952f5763189655160697f5040a0affc56b3d0e042b053678 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..cb2ac96baeaf8f3382dc8f0f62087f73edc7f970 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.038733829556077e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3bb8906ecb83e0ff3b3d4611d05f4b23dad8c663 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e31eb3d6bb2fc6e0c169c997955ff0e734f524a89d1c8c9b7ef57a4bae014dc6 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..76b5567074e9bae6e613299d85583cfb05bf1c09 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78cbe464e6e610ece64881b538d057b203339611dc48270e21ba01b289aead17 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..594d01a0a1b45b388016e6e6c58a1074f424337e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 330000, + "verbose": false, + "_step_count": 330001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.038733829556077e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..f5ea2cc3e4569f3f5076d82101c911d3ca4dadb3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/330000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 330000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..208c1b1c907e8ffedd0f0839e1bc8eb836001ef0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7e2acf35eb7c6b431775a72c90075d648e5cab24990c129e6f90a02af9d8b3b +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..c95ffe05c43624624f92810f8806852a0be08830 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.550066611157049e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b4d1e2afa612731db10ce0d8d1a0f30c78c4e8c9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4451ca85524f4a291edc1d0bd1b887e56722212ec2026fd1a149bd0a6c11c689 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7d23ceb6fc8848b24ba3178f641719f5e15ee72f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6bca1a2cb1a63f65963296b05cfc7369f494dd2898cf4490d6f97d5a32913ed +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a39d6e5bb2a4966dfcb991d5bc882f8c4dadb29b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 360000, + "verbose": false, + "_step_count": 360001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.550066611157049e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..bd79b4d9c6aaae91547528200cf9573568cb1ede --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/360000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 360000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log index 12cf918367d22cc11e03be8cc2518db67592db79..eb106f8bffbb1de9a45650186cca3cf6bd2790c4 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log @@ -11,3 +11,5 @@ {"time":"2025-05-06T10:30:03.039602464Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:41710->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T12:25:33.040402545Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:56185->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T12:44:48.040371676Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:43741->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T16:50:48.039369614Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:38891->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T18:02:28.241678384Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log index f00c484d7033d17647899613f1997fabf4132cb0..76782edb640ecc6b31d76fc22e0eacd4ac2ecd30 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log @@ -1049,3 +1049,904 @@ INFO 2025-05-06 14:22:24 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.34 loss INFO 2025-05-06 14:23:49 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.37 loss:0.009 grdn:0.187 lr:8.8e-05 updt_s:0.424 data_s:0.001 INFO 2025-05-06 14:25:14 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.40 loss:0.008 grdn:0.185 lr:8.8e-05 updt_s:0.423 data_s:0.001 INFO 2025-05-06 14:26:39 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.42 loss:0.009 grdn:0.190 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:28:04 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.45 loss:0.008 grdn:0.174 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:29:29 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.48 loss:0.008 grdn:0.190 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:30:54 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.51 loss:0.009 grdn:0.188 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:32:19 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.54 loss:0.009 grdn:0.186 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:33:44 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.56 loss:0.009 grdn:0.180 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:35:09 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.59 loss:0.009 grdn:0.189 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:36:34 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.62 loss:0.009 grdn:0.187 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:37:59 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.65 loss:0.008 grdn:0.192 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:39:24 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.67 loss:0.008 grdn:0.183 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:40:50 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.70 loss:0.008 grdn:0.184 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:42:15 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.73 loss:0.009 grdn:0.191 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:43:40 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.76 loss:0.009 grdn:0.189 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:45:05 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.78 loss:0.008 grdn:0.173 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:46:30 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.81 loss:0.008 grdn:0.179 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:47:55 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.84 loss:0.010 grdn:0.211 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:49:20 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.87 loss:0.009 grdn:0.191 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:50:45 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.89 loss:0.009 grdn:0.187 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:52:11 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.92 loss:0.009 grdn:0.197 lr:8.7e-05 updt_s:0.423 data_s:0.006 +INFO 2025-05-06 14:52:11 ts/train.py:241 Checkpoint policy after step 210000 +INFO 2025-05-06 14:53:39 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.95 loss:0.008 grdn:0.187 lr:8.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 14:55:04 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.98 loss:0.008 grdn:0.189 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:56:29 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.00 loss:0.009 grdn:0.192 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:57:54 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.03 loss:0.009 grdn:0.192 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:59:18 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.06 loss:0.008 grdn:0.179 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:00:44 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.09 loss:0.008 grdn:0.179 lr:8.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 15:02:08 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.11 loss:0.010 grdn:0.207 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:03:33 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.14 loss:0.008 grdn:0.172 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:04:58 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.17 loss:0.008 grdn:0.183 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:06:23 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.20 loss:0.009 grdn:0.196 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:07:48 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.22 loss:0.009 grdn:0.197 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:09:13 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.25 loss:0.009 grdn:0.187 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:10:38 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.28 loss:0.009 grdn:0.189 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:12:03 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.31 loss:0.009 grdn:0.192 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:13:28 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.33 loss:0.008 grdn:0.188 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:14:53 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.36 loss:0.009 grdn:0.194 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:16:18 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.39 loss:0.009 grdn:0.193 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:17:43 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.42 loss:0.009 grdn:0.187 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:19:08 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.44 loss:0.008 grdn:0.186 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:20:33 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.47 loss:0.009 grdn:0.193 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:21:58 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.50 loss:0.008 grdn:0.189 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:23:23 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.53 loss:0.009 grdn:0.191 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:24:48 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.55 loss:0.009 grdn:0.199 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:26:12 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.58 loss:0.009 grdn:0.182 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:27:37 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.61 loss:0.007 grdn:0.176 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:29:02 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.64 loss:0.009 grdn:0.196 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:30:27 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.66 loss:0.008 grdn:0.187 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:31:52 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.69 loss:0.007 grdn:0.169 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:33:17 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.72 loss:0.009 grdn:0.186 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:34:42 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.75 loss:0.008 grdn:0.176 lr:8.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:36:07 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.77 loss:0.009 grdn:0.190 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:37:31 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.80 loss:0.009 grdn:0.194 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:38:56 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.83 loss:0.008 grdn:0.180 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:40:21 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.86 loss:0.009 grdn:0.198 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:41:47 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.88 loss:0.008 grdn:0.176 lr:8.6e-05 updt_s:0.423 data_s:0.007 +INFO 2025-05-06 15:43:12 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.91 loss:0.008 grdn:0.181 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:44:37 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.94 loss:0.009 grdn:0.186 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:46:02 ts/train.py:232 step:218K smpl:2M ep:9K epch:29.97 loss:0.007 grdn:0.175 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:47:27 ts/train.py:232 step:218K smpl:2M ep:9K epch:29.99 loss:0.008 grdn:0.193 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:48:52 ts/train.py:232 step:218K smpl:2M ep:9K epch:30.02 loss:0.008 grdn:0.187 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:50:17 ts/train.py:232 step:218K smpl:2M ep:9K epch:30.05 loss:0.007 grdn:0.171 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:51:42 ts/train.py:232 step:218K smpl:2M ep:9K epch:30.08 loss:0.007 grdn:0.172 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:53:07 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.11 loss:0.008 grdn:0.181 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:54:32 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.13 loss:0.008 grdn:0.189 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:55:57 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.16 loss:0.008 grdn:0.180 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 15:57:22 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.19 loss:0.008 grdn:0.182 lr:8.6e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 15:58:47 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.22 loss:0.008 grdn:0.182 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:00:12 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.24 loss:0.010 grdn:0.190 lr:8.6e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 16:01:37 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.27 loss:0.009 grdn:0.184 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:03:02 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.30 loss:0.008 grdn:0.177 lr:8.6e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 16:04:27 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.33 loss:0.008 grdn:0.182 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:05:52 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.35 loss:0.008 grdn:0.176 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:07:17 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.38 loss:0.008 grdn:0.191 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:08:42 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.41 loss:0.009 grdn:0.190 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:10:07 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.44 loss:0.008 grdn:0.175 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:11:32 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.46 loss:0.009 grdn:0.187 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:12:57 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.49 loss:0.008 grdn:0.179 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:14:22 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.52 loss:0.008 grdn:0.181 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:15:47 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.55 loss:0.009 grdn:0.195 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:17:12 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.57 loss:0.008 grdn:0.183 lr:8.6e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 16:18:36 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.60 loss:0.008 grdn:0.182 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:20:01 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.63 loss:0.009 grdn:0.197 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:21:26 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.66 loss:0.008 grdn:0.177 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:22:51 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.68 loss:0.007 grdn:0.165 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:24:16 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.71 loss:0.008 grdn:0.188 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:25:41 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.74 loss:0.009 grdn:0.191 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:27:06 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.77 loss:0.008 grdn:0.189 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:28:31 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.79 loss:0.008 grdn:0.180 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:29:56 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.82 loss:0.008 grdn:0.185 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:31:22 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.85 loss:0.008 grdn:0.182 lr:8.6e-05 updt_s:0.422 data_s:0.008 +INFO 2025-05-06 16:32:47 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.88 loss:0.009 grdn:0.188 lr:8.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:34:12 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.90 loss:0.008 grdn:0.185 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:35:37 ts/train.py:232 step:225K smpl:2M ep:9K epch:30.93 loss:0.008 grdn:0.178 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:37:02 ts/train.py:232 step:225K smpl:2M ep:9K epch:30.96 loss:0.008 grdn:0.182 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:38:26 ts/train.py:232 step:225K smpl:2M ep:9K epch:30.99 loss:0.008 grdn:0.186 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:39:51 ts/train.py:232 step:225K smpl:2M ep:9K epch:31.01 loss:0.009 grdn:0.198 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:41:16 ts/train.py:232 step:225K smpl:2M ep:9K epch:31.04 loss:0.008 grdn:0.177 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:42:41 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.07 loss:0.008 grdn:0.184 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:44:06 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.10 loss:0.008 grdn:0.183 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:45:31 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.12 loss:0.009 grdn:0.194 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:46:56 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.15 loss:0.007 grdn:0.179 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:48:21 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.18 loss:0.008 grdn:0.177 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:49:46 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.21 loss:0.009 grdn:0.194 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:51:10 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.23 loss:0.008 grdn:0.180 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:52:35 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.26 loss:0.009 grdn:0.197 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:54:00 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.29 loss:0.008 grdn:0.182 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:55:25 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.32 loss:0.009 grdn:0.189 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:56:50 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.34 loss:0.009 grdn:0.184 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:58:15 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.37 loss:0.008 grdn:0.193 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 16:59:40 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.40 loss:0.008 grdn:0.184 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:01:05 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.43 loss:0.009 grdn:0.198 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:02:30 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.45 loss:0.008 grdn:0.194 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:03:55 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.48 loss:0.009 grdn:0.188 lr:8.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:05:20 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.51 loss:0.009 grdn:0.190 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:06:44 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.54 loss:0.008 grdn:0.179 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:08:09 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.56 loss:0.009 grdn:0.192 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:09:34 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.59 loss:0.008 grdn:0.193 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:10:59 ts/train.py:232 step:230K smpl:2M ep:9K epch:31.62 loss:0.008 grdn:0.191 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:12:24 ts/train.py:232 step:230K smpl:2M ep:9K epch:31.65 loss:0.009 grdn:0.191 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:13:49 ts/train.py:232 step:230K smpl:2M ep:10K epch:31.67 loss:0.007 grdn:0.177 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:15:14 ts/train.py:232 step:230K smpl:2M ep:10K epch:31.70 loss:0.008 grdn:0.183 lr:8.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:16:39 ts/train.py:232 step:230K smpl:2M ep:10K epch:31.73 loss:0.009 grdn:0.195 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:18:04 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.76 loss:0.008 grdn:0.189 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:19:29 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.79 loss:0.009 grdn:0.189 lr:8.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:20:55 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.81 loss:0.008 grdn:0.180 lr:8.5e-05 updt_s:0.423 data_s:0.007 +INFO 2025-05-06 17:22:20 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.84 loss:0.009 grdn:0.197 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:23:45 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.87 loss:0.007 grdn:0.175 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:25:10 ts/train.py:232 step:232K smpl:2M ep:10K epch:31.90 loss:0.008 grdn:0.185 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:26:35 ts/train.py:232 step:232K smpl:2M ep:10K epch:31.92 loss:0.008 grdn:0.179 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:28:00 ts/train.py:232 step:232K smpl:2M ep:10K epch:31.95 loss:0.009 grdn:0.200 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:29:25 ts/train.py:232 step:232K smpl:2M ep:10K epch:31.98 loss:0.008 grdn:0.195 lr:8.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:30:50 ts/train.py:232 step:232K smpl:2M ep:10K epch:32.01 loss:0.007 grdn:0.176 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:32:15 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.03 loss:0.008 grdn:0.189 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:33:40 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.06 loss:0.008 grdn:0.189 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:35:05 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.09 loss:0.008 grdn:0.187 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:36:30 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.12 loss:0.008 grdn:0.186 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:37:55 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.14 loss:0.008 grdn:0.181 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:39:20 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.17 loss:0.008 grdn:0.179 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:40:45 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.20 loss:0.008 grdn:0.184 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:42:10 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.23 loss:0.008 grdn:0.177 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:43:35 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.25 loss:0.009 grdn:0.196 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:45:00 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.28 loss:0.008 grdn:0.175 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:46:25 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.31 loss:0.008 grdn:0.183 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 17:47:50 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.34 loss:0.008 grdn:0.189 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:49:15 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.36 loss:0.009 grdn:0.190 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:50:40 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.39 loss:0.009 grdn:0.200 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:52:05 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.42 loss:0.008 grdn:0.193 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:53:30 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.45 loss:0.007 grdn:0.169 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:54:55 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.47 loss:0.008 grdn:0.179 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:56:20 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.50 loss:0.007 grdn:0.174 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:57:45 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.53 loss:0.008 grdn:0.182 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 17:59:10 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.56 loss:0.008 grdn:0.187 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:00:35 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.58 loss:0.008 grdn:0.188 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:02:00 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.61 loss:0.007 grdn:0.174 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:03:25 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.64 loss:0.008 grdn:0.183 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 18:04:50 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.67 loss:0.008 grdn:0.188 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:06:15 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.69 loss:0.008 grdn:0.193 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:07:40 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.72 loss:0.008 grdn:0.193 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:09:05 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.75 loss:0.009 grdn:0.193 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 18:10:32 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.78 loss:0.008 grdn:0.180 lr:8.4e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-06 18:11:57 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.80 loss:0.008 grdn:0.180 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 18:13:22 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.83 loss:0.008 grdn:0.193 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 18:14:47 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.86 loss:0.008 grdn:0.177 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:16:12 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.89 loss:0.008 grdn:0.189 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 18:17:37 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.91 loss:0.008 grdn:0.190 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 18:19:02 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.94 loss:0.007 grdn:0.184 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:20:27 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.97 loss:0.008 grdn:0.186 lr:8.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 18:21:52 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.00 loss:0.007 grdn:0.178 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:23:17 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.02 loss:0.007 grdn:0.181 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:24:42 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.05 loss:0.008 grdn:0.186 lr:8.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:24:42 ts/train.py:241 Checkpoint policy after step 240000 +INFO 2025-05-06 18:26:10 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.08 loss:0.009 grdn:0.196 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 18:27:35 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.11 loss:0.008 grdn:0.191 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:29:00 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.13 loss:0.008 grdn:0.178 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:30:25 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.16 loss:0.008 grdn:0.188 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:31:50 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.19 loss:0.008 grdn:0.181 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:33:15 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.22 loss:0.008 grdn:0.176 lr:8.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 18:34:41 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.24 loss:0.008 grdn:0.187 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:36:06 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.27 loss:0.008 grdn:0.193 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:37:31 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.30 loss:0.007 grdn:0.166 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:38:56 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.33 loss:0.008 grdn:0.193 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:40:21 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.36 loss:0.008 grdn:0.193 lr:8.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 18:41:46 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.38 loss:0.008 grdn:0.188 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:43:12 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.41 loss:0.009 grdn:0.197 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:44:37 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.44 loss:0.008 grdn:0.188 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:46:02 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.47 loss:0.008 grdn:0.185 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:47:27 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.49 loss:0.009 grdn:0.196 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:48:52 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.52 loss:0.008 grdn:0.177 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 18:50:17 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.55 loss:0.008 grdn:0.182 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:51:42 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.58 loss:0.007 grdn:0.179 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:53:07 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.60 loss:0.007 grdn:0.184 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:54:32 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.63 loss:0.007 grdn:0.180 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:55:57 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.66 loss:0.008 grdn:0.186 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:57:22 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.69 loss:0.008 grdn:0.183 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 18:58:47 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.71 loss:0.009 grdn:0.199 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 19:00:13 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.74 loss:0.008 grdn:0.182 lr:8.3e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-06 19:01:38 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.77 loss:0.008 grdn:0.185 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:03:03 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.80 loss:0.008 grdn:0.173 lr:8.3e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-06 19:04:28 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.82 loss:0.008 grdn:0.184 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:05:53 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.85 loss:0.008 grdn:0.189 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 19:07:18 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.88 loss:0.007 grdn:0.178 lr:8.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 19:08:43 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.91 loss:0.009 grdn:0.187 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:10:08 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.93 loss:0.008 grdn:0.186 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:11:32 ts/train.py:232 step:247K smpl:2M ep:10K epch:33.96 loss:0.008 grdn:0.189 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:12:57 ts/train.py:232 step:247K smpl:2M ep:10K epch:33.99 loss:0.008 grdn:0.182 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:14:22 ts/train.py:232 step:247K smpl:2M ep:10K epch:34.02 loss:0.007 grdn:0.178 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:15:47 ts/train.py:232 step:247K smpl:2M ep:10K epch:34.04 loss:0.007 grdn:0.180 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:17:12 ts/train.py:232 step:247K smpl:2M ep:10K epch:34.07 loss:0.008 grdn:0.183 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:18:37 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.10 loss:0.008 grdn:0.180 lr:8.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:20:02 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.13 loss:0.009 grdn:0.198 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:21:27 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.15 loss:0.008 grdn:0.180 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:22:52 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.18 loss:0.008 grdn:0.190 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:24:17 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.21 loss:0.007 grdn:0.176 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:25:42 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.24 loss:0.008 grdn:0.185 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:27:07 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.26 loss:0.008 grdn:0.185 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:28:32 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.29 loss:0.008 grdn:0.180 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:29:56 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.32 loss:0.008 grdn:0.183 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:31:21 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.35 loss:0.008 grdn:0.184 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:32:46 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.37 loss:0.008 grdn:0.185 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:34:11 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.40 loss:0.008 grdn:0.185 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:35:36 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.43 loss:0.009 grdn:0.195 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:37:01 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.46 loss:0.008 grdn:0.197 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:38:26 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.48 loss:0.007 grdn:0.169 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:39:51 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.51 loss:0.008 grdn:0.181 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:41:15 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.54 loss:0.007 grdn:0.169 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:42:40 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.57 loss:0.008 grdn:0.187 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:44:05 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.59 loss:0.007 grdn:0.175 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:45:30 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.62 loss:0.008 grdn:0.185 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:46:55 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.65 loss:0.008 grdn:0.188 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 19:48:20 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.68 loss:0.008 grdn:0.188 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 19:49:46 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.70 loss:0.007 grdn:0.181 lr:8.2e-05 updt_s:0.422 data_s:0.007 +INFO 2025-05-06 19:51:10 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.73 loss:0.008 grdn:0.183 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 19:52:35 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.76 loss:0.007 grdn:0.170 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 19:54:00 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.79 loss:0.008 grdn:0.185 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 19:55:25 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.81 loss:0.008 grdn:0.191 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 19:56:49 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.84 loss:0.008 grdn:0.186 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 19:58:14 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.87 loss:0.009 grdn:0.193 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 19:59:39 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.90 loss:0.008 grdn:0.187 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:01:04 ts/train.py:232 step:254K smpl:2M ep:10K epch:34.93 loss:0.007 grdn:0.177 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:02:28 ts/train.py:232 step:254K smpl:2M ep:10K epch:34.95 loss:0.007 grdn:0.172 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:03:53 ts/train.py:232 step:254K smpl:2M ep:10K epch:34.98 loss:0.007 grdn:0.181 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:05:18 ts/train.py:232 step:254K smpl:2M ep:11K epch:35.01 loss:0.007 grdn:0.179 lr:8.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:06:43 ts/train.py:232 step:254K smpl:2M ep:11K epch:35.04 loss:0.008 grdn:0.184 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:08:08 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.06 loss:0.008 grdn:0.187 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:09:32 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.09 loss:0.008 grdn:0.184 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:10:57 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.12 loss:0.008 grdn:0.181 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:12:22 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.15 loss:0.008 grdn:0.187 lr:8.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:13:46 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.17 loss:0.008 grdn:0.183 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:15:11 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.20 loss:0.007 grdn:0.180 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:16:36 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.23 loss:0.008 grdn:0.188 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:18:01 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.26 loss:0.007 grdn:0.178 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:19:25 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.28 loss:0.008 grdn:0.192 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:20:50 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.31 loss:0.008 grdn:0.193 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:22:15 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.34 loss:0.008 grdn:0.193 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:23:39 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.37 loss:0.008 grdn:0.186 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:25:04 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.39 loss:0.007 grdn:0.181 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:26:29 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.42 loss:0.007 grdn:0.178 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:27:54 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.45 loss:0.008 grdn:0.182 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:29:18 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.48 loss:0.007 grdn:0.181 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:30:43 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.50 loss:0.007 grdn:0.183 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:32:08 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.53 loss:0.007 grdn:0.173 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:33:32 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.56 loss:0.008 grdn:0.186 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:34:57 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.59 loss:0.008 grdn:0.191 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:36:22 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.61 loss:0.008 grdn:0.178 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:37:46 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.64 loss:0.008 grdn:0.179 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:39:12 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.67 loss:0.007 grdn:0.174 lr:8.1e-05 updt_s:0.421 data_s:0.007 +INFO 2025-05-06 20:40:37 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.70 loss:0.008 grdn:0.186 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:42:02 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.72 loss:0.008 grdn:0.187 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:43:27 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.75 loss:0.008 grdn:0.182 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:44:51 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.78 loss:0.008 grdn:0.193 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:46:16 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.81 loss:0.008 grdn:0.185 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:47:41 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.83 loss:0.008 grdn:0.182 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:49:06 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.86 loss:0.008 grdn:0.185 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:50:31 ts/train.py:232 step:261K smpl:2M ep:11K epch:35.89 loss:0.007 grdn:0.187 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:51:55 ts/train.py:232 step:261K smpl:2M ep:11K epch:35.92 loss:0.009 grdn:0.188 lr:8.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 20:53:20 ts/train.py:232 step:261K smpl:2M ep:11K epch:35.94 loss:0.008 grdn:0.190 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:54:45 ts/train.py:232 step:261K smpl:2M ep:11K epch:35.97 loss:0.008 grdn:0.183 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:56:10 ts/train.py:232 step:261K smpl:2M ep:11K epch:36.00 loss:0.009 grdn:0.197 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:57:35 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.03 loss:0.007 grdn:0.176 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 20:59:00 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.05 loss:0.008 grdn:0.188 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:00:24 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.08 loss:0.008 grdn:0.181 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:01:49 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.11 loss:0.007 grdn:0.176 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:03:14 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.14 loss:0.008 grdn:0.181 lr:8.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:04:39 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.16 loss:0.008 grdn:0.183 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:06:04 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.19 loss:0.008 grdn:0.187 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:07:29 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.22 loss:0.008 grdn:0.177 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:08:54 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.25 loss:0.008 grdn:0.189 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:10:19 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.27 loss:0.007 grdn:0.178 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:11:43 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.30 loss:0.008 grdn:0.193 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:13:08 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.33 loss:0.007 grdn:0.179 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:14:33 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.36 loss:0.008 grdn:0.185 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:15:58 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.38 loss:0.007 grdn:0.176 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:17:23 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.41 loss:0.007 grdn:0.181 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:18:48 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.44 loss:0.007 grdn:0.172 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:20:12 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.47 loss:0.008 grdn:0.184 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:21:37 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.50 loss:0.008 grdn:0.189 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:23:02 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.52 loss:0.008 grdn:0.199 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:24:27 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.55 loss:0.007 grdn:0.181 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:25:52 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.58 loss:0.008 grdn:0.189 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:27:17 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.61 loss:0.007 grdn:0.186 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:28:43 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.63 loss:0.008 grdn:0.178 lr:8.0e-05 updt_s:0.422 data_s:0.010 +INFO 2025-05-06 21:30:08 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.66 loss:0.007 grdn:0.185 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:31:33 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.69 loss:0.007 grdn:0.179 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:32:58 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.72 loss:0.008 grdn:0.192 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:34:23 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.74 loss:0.007 grdn:0.188 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:35:48 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.77 loss:0.008 grdn:0.185 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:37:13 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.80 loss:0.008 grdn:0.191 lr:8.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 21:38:38 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.83 loss:0.007 grdn:0.170 lr:8.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 21:40:03 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.85 loss:0.008 grdn:0.190 lr:8.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 21:41:28 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.88 loss:0.007 grdn:0.181 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:42:53 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.91 loss:0.008 grdn:0.186 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:44:18 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.94 loss:0.008 grdn:0.188 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:45:43 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.96 loss:0.008 grdn:0.194 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:47:08 ts/train.py:232 step:269K smpl:2M ep:11K epch:36.99 loss:0.007 grdn:0.166 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:48:33 ts/train.py:232 step:269K smpl:2M ep:11K epch:37.02 loss:0.008 grdn:0.189 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:49:58 ts/train.py:232 step:269K smpl:2M ep:11K epch:37.05 loss:0.008 grdn:0.186 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:51:23 ts/train.py:232 step:269K smpl:2M ep:11K epch:37.07 loss:0.008 grdn:0.176 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:52:48 ts/train.py:232 step:269K smpl:2M ep:11K epch:37.10 loss:0.007 grdn:0.175 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:54:13 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.13 loss:0.008 grdn:0.185 lr:8.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:55:38 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.16 loss:0.007 grdn:0.178 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:57:02 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.18 loss:0.008 grdn:0.192 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 21:57:02 ts/train.py:241 Checkpoint policy after step 270000 +INFO 2025-05-06 21:58:30 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.21 loss:0.008 grdn:0.193 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 21:59:55 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.24 loss:0.009 grdn:0.197 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:01:20 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.27 loss:0.007 grdn:0.179 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:02:45 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.29 loss:0.008 grdn:0.191 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:04:09 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.32 loss:0.009 grdn:0.200 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:05:34 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.35 loss:0.008 grdn:0.180 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:06:59 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.38 loss:0.007 grdn:0.172 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:08:24 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.40 loss:0.007 grdn:0.178 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:09:49 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.43 loss:0.007 grdn:0.170 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:11:14 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.46 loss:0.008 grdn:0.190 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:12:38 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.49 loss:0.008 grdn:0.184 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:14:03 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.51 loss:0.007 grdn:0.168 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:15:28 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.54 loss:0.008 grdn:0.193 lr:7.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 22:16:54 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.57 loss:0.008 grdn:0.178 lr:7.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 22:18:20 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.60 loss:0.008 grdn:0.185 lr:7.9e-05 updt_s:0.423 data_s:0.010 +INFO 2025-05-06 22:19:45 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.62 loss:0.008 grdn:0.184 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:21:10 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.65 loss:0.007 grdn:0.175 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:22:35 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.68 loss:0.007 grdn:0.182 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:24:00 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.71 loss:0.008 grdn:0.190 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:25:25 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.73 loss:0.008 grdn:0.174 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:26:49 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.76 loss:0.007 grdn:0.183 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:28:14 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.79 loss:0.008 grdn:0.180 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:29:39 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.82 loss:0.008 grdn:0.194 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:31:04 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.84 loss:0.009 grdn:0.196 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:32:29 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.87 loss:0.009 grdn:0.188 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:33:53 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.90 loss:0.007 grdn:0.182 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:35:18 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.93 loss:0.008 grdn:0.186 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:36:43 ts/train.py:232 step:276K smpl:2M ep:11K epch:37.95 loss:0.008 grdn:0.181 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:38:07 ts/train.py:232 step:276K smpl:2M ep:11K epch:37.98 loss:0.008 grdn:0.186 lr:7.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 22:39:32 ts/train.py:232 step:276K smpl:2M ep:11K epch:38.01 loss:0.008 grdn:0.189 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:40:57 ts/train.py:232 step:276K smpl:2M ep:11K epch:38.04 loss:0.008 grdn:0.185 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:42:22 ts/train.py:232 step:276K smpl:2M ep:11K epch:38.07 loss:0.007 grdn:0.187 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:43:47 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.09 loss:0.008 grdn:0.187 lr:7.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:45:12 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.12 loss:0.008 grdn:0.189 lr:7.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 22:46:37 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.15 loss:0.008 grdn:0.185 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:48:02 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.18 loss:0.008 grdn:0.186 lr:7.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 22:49:27 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.20 loss:0.007 grdn:0.178 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:50:52 ts/train.py:232 step:278K smpl:2M ep:11K epch:38.23 loss:0.007 grdn:0.184 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:52:17 ts/train.py:232 step:278K smpl:2M ep:11K epch:38.26 loss:0.008 grdn:0.193 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:53:42 ts/train.py:232 step:278K smpl:2M ep:11K epch:38.29 loss:0.009 grdn:0.191 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:55:07 ts/train.py:232 step:278K smpl:2M ep:11K epch:38.31 loss:0.007 grdn:0.179 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:56:32 ts/train.py:232 step:278K smpl:2M ep:12K epch:38.34 loss:0.007 grdn:0.185 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:57:57 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.37 loss:0.008 grdn:0.189 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 22:59:22 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.40 loss:0.007 grdn:0.186 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:00:47 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.42 loss:0.007 grdn:0.182 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:02:12 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.45 loss:0.008 grdn:0.189 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:03:37 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.48 loss:0.008 grdn:0.183 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:05:02 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.51 loss:0.008 grdn:0.202 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:06:26 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.53 loss:0.007 grdn:0.171 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:07:52 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.56 loss:0.007 grdn:0.176 lr:7.8e-05 updt_s:0.422 data_s:0.007 +INFO 2025-05-06 23:09:17 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.59 loss:0.008 grdn:0.189 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:10:42 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.62 loss:0.007 grdn:0.181 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:12:07 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.64 loss:0.007 grdn:0.170 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:13:32 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.67 loss:0.007 grdn:0.184 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:14:57 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.70 loss:0.008 grdn:0.183 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:16:22 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.73 loss:0.008 grdn:0.183 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:17:47 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.75 loss:0.007 grdn:0.172 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:19:12 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.78 loss:0.006 grdn:0.158 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:20:37 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.81 loss:0.007 grdn:0.166 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:22:02 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.84 loss:0.008 grdn:0.185 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:23:26 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.86 loss:0.008 grdn:0.191 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:24:51 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.89 loss:0.008 grdn:0.194 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:26:16 ts/train.py:232 step:283K smpl:2M ep:12K epch:38.92 loss:0.008 grdn:0.187 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:27:41 ts/train.py:232 step:283K smpl:2M ep:12K epch:38.95 loss:0.007 grdn:0.179 lr:7.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:29:06 ts/train.py:232 step:283K smpl:2M ep:12K epch:38.97 loss:0.006 grdn:0.162 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:30:31 ts/train.py:232 step:283K smpl:2M ep:12K epch:39.00 loss:0.007 grdn:0.178 lr:7.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:31:55 ts/train.py:232 step:283K smpl:2M ep:12K epch:39.03 loss:0.007 grdn:0.164 lr:7.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:33:20 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.06 loss:0.007 grdn:0.180 lr:7.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:34:45 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.08 loss:0.007 grdn:0.180 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:36:10 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.11 loss:0.008 grdn:0.189 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:37:35 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.14 loss:0.007 grdn:0.181 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:38:59 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.17 loss:0.008 grdn:0.191 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:40:24 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.19 loss:0.008 grdn:0.192 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:41:49 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.22 loss:0.007 grdn:0.167 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:43:14 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.25 loss:0.007 grdn:0.185 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:44:39 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.28 loss:0.008 grdn:0.181 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:46:04 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.30 loss:0.008 grdn:0.180 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:47:28 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.33 loss:0.007 grdn:0.186 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:48:53 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.36 loss:0.007 grdn:0.178 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:50:18 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.39 loss:0.007 grdn:0.182 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:51:43 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.41 loss:0.007 grdn:0.177 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:53:08 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.44 loss:0.007 grdn:0.176 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:54:32 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.47 loss:0.007 grdn:0.171 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 23:55:57 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.50 loss:0.008 grdn:0.183 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-06 23:57:23 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.52 loss:0.008 grdn:0.190 lr:7.7e-05 updt_s:0.422 data_s:0.007 +INFO 2025-05-06 23:58:48 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.55 loss:0.007 grdn:0.182 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 00:00:13 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.58 loss:0.007 grdn:0.176 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:01:38 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.61 loss:0.008 grdn:0.192 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:03:02 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.64 loss:0.008 grdn:0.183 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:04:27 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.66 loss:0.007 grdn:0.177 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:05:52 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.69 loss:0.007 grdn:0.188 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:07:17 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.72 loss:0.007 grdn:0.177 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:08:42 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.75 loss:0.007 grdn:0.180 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:10:06 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.77 loss:0.008 grdn:0.180 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:11:31 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.80 loss:0.008 grdn:0.195 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:12:56 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.83 loss:0.008 grdn:0.188 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:14:21 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.86 loss:0.007 grdn:0.183 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:15:45 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.88 loss:0.007 grdn:0.175 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:17:10 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.91 loss:0.008 grdn:0.188 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:18:35 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.94 loss:0.007 grdn:0.179 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:19:59 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.97 loss:0.008 grdn:0.184 lr:7.7e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:21:24 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.99 loss:0.007 grdn:0.170 lr:7.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 00:22:49 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.02 loss:0.007 grdn:0.177 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:24:14 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.05 loss:0.008 grdn:0.189 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:25:38 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.08 loss:0.007 grdn:0.182 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 00:27:03 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.10 loss:0.007 grdn:0.180 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:28:28 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.13 loss:0.007 grdn:0.181 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:29:53 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.16 loss:0.007 grdn:0.176 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:31:17 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.19 loss:0.007 grdn:0.183 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:32:42 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.21 loss:0.008 grdn:0.191 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:34:07 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.24 loss:0.007 grdn:0.186 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:35:31 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.27 loss:0.008 grdn:0.196 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:36:56 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.30 loss:0.007 grdn:0.176 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 00:38:21 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.32 loss:0.008 grdn:0.185 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:39:46 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.35 loss:0.007 grdn:0.171 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:41:10 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.38 loss:0.007 grdn:0.179 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:42:35 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.41 loss:0.008 grdn:0.189 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:44:00 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.43 loss:0.007 grdn:0.184 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:45:25 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.46 loss:0.007 grdn:0.178 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:46:51 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.49 loss:0.007 grdn:0.190 lr:7.6e-05 updt_s:0.422 data_s:0.008 +INFO 2025-05-07 00:48:15 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.52 loss:0.007 grdn:0.180 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 00:49:40 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.54 loss:0.007 grdn:0.172 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 00:51:05 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.57 loss:0.007 grdn:0.171 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:52:30 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.60 loss:0.006 grdn:0.176 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:53:54 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.63 loss:0.008 grdn:0.185 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:55:19 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.65 loss:0.006 grdn:0.166 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:56:44 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.68 loss:0.007 grdn:0.184 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 00:58:09 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.71 loss:0.007 grdn:0.183 lr:7.6e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 00:59:34 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.74 loss:0.006 grdn:0.167 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:00:58 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.76 loss:0.007 grdn:0.180 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:02:23 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.79 loss:0.007 grdn:0.178 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:03:48 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.82 loss:0.008 grdn:0.193 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:05:13 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.85 loss:0.007 grdn:0.182 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:06:38 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.87 loss:0.007 grdn:0.182 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:08:03 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.90 loss:0.008 grdn:0.181 lr:7.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:09:27 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.93 loss:0.007 grdn:0.178 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:10:52 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.96 loss:0.007 grdn:0.178 lr:7.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:12:17 ts/train.py:232 step:298K smpl:2M ep:12K epch:40.98 loss:0.007 grdn:0.182 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:13:42 ts/train.py:232 step:298K smpl:2M ep:12K epch:41.01 loss:0.007 grdn:0.182 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:15:06 ts/train.py:232 step:298K smpl:2M ep:12K epch:41.04 loss:0.006 grdn:0.176 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:16:31 ts/train.py:232 step:298K smpl:2M ep:12K epch:41.07 loss:0.007 grdn:0.185 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:17:56 ts/train.py:232 step:298K smpl:2M ep:12K epch:41.09 loss:0.007 grdn:0.179 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:19:21 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.12 loss:0.007 grdn:0.182 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:20:45 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.15 loss:0.007 grdn:0.176 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:22:10 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.18 loss:0.007 grdn:0.177 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:23:35 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.21 loss:0.008 grdn:0.193 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:25:00 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.23 loss:0.006 grdn:0.167 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:26:24 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.26 loss:0.007 grdn:0.178 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:27:49 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.29 loss:0.007 grdn:0.175 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:29:14 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.32 loss:0.007 grdn:0.179 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:29:14 ts/train.py:241 Checkpoint policy after step 300000 +INFO 2025-05-07 01:30:42 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.34 loss:0.007 grdn:0.175 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:32:06 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.37 loss:0.007 grdn:0.180 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:33:31 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.40 loss:0.008 grdn:0.181 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:34:56 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.43 loss:0.007 grdn:0.181 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:36:22 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.45 loss:0.007 grdn:0.185 lr:7.5e-05 updt_s:0.421 data_s:0.007 +INFO 2025-05-07 01:37:46 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.48 loss:0.007 grdn:0.176 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:39:11 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.51 loss:0.007 grdn:0.184 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:40:36 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.54 loss:0.007 grdn:0.187 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:42:01 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.56 loss:0.008 grdn:0.189 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:43:25 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.59 loss:0.007 grdn:0.184 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:44:50 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.62 loss:0.007 grdn:0.181 lr:7.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:46:15 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.65 loss:0.007 grdn:0.176 lr:7.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:47:40 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.67 loss:0.007 grdn:0.181 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:49:04 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.70 loss:0.007 grdn:0.184 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:50:29 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.73 loss:0.007 grdn:0.182 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:51:54 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.76 loss:0.007 grdn:0.178 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:53:19 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.78 loss:0.007 grdn:0.181 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:54:43 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.81 loss:0.007 grdn:0.179 lr:7.5e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 01:56:08 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.84 loss:0.008 grdn:0.194 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:57:33 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.87 loss:0.007 grdn:0.185 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 01:58:58 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.89 loss:0.006 grdn:0.174 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:00:22 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.92 loss:0.007 grdn:0.184 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:01:47 ts/train.py:232 step:305K smpl:2M ep:13K epch:41.95 loss:0.006 grdn:0.172 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:03:12 ts/train.py:232 step:305K smpl:2M ep:13K epch:41.98 loss:0.006 grdn:0.165 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:04:37 ts/train.py:232 step:305K smpl:2M ep:13K epch:42.00 loss:0.007 grdn:0.172 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:06:02 ts/train.py:232 step:305K smpl:2M ep:13K epch:42.03 loss:0.007 grdn:0.181 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:07:27 ts/train.py:232 step:305K smpl:2M ep:13K epch:42.06 loss:0.008 grdn:0.187 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:08:51 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.09 loss:0.007 grdn:0.185 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:10:16 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.11 loss:0.007 grdn:0.172 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:11:41 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.14 loss:0.007 grdn:0.180 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:13:05 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.17 loss:0.006 grdn:0.172 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:14:30 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.20 loss:0.007 grdn:0.190 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:15:55 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.22 loss:0.007 grdn:0.178 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:17:19 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.25 loss:0.007 grdn:0.185 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:18:44 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.28 loss:0.007 grdn:0.176 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:20:08 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.31 loss:0.007 grdn:0.184 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:21:33 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.33 loss:0.008 grdn:0.176 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:22:58 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.36 loss:0.007 grdn:0.179 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:24:23 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.39 loss:0.007 grdn:0.182 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:25:49 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.42 loss:0.007 grdn:0.182 lr:7.4e-05 updt_s:0.422 data_s:0.007 +INFO 2025-05-07 02:27:13 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.44 loss:0.007 grdn:0.179 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:28:38 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.47 loss:0.007 grdn:0.182 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:30:03 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.50 loss:0.007 grdn:0.178 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:31:28 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.53 loss:0.007 grdn:0.188 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:32:53 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.55 loss:0.007 grdn:0.182 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:34:18 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.58 loss:0.007 grdn:0.191 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:35:42 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.61 loss:0.007 grdn:0.179 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:37:07 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.64 loss:0.007 grdn:0.189 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:38:32 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.66 loss:0.007 grdn:0.166 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:39:57 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.69 loss:0.007 grdn:0.167 lr:7.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:41:22 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.72 loss:0.007 grdn:0.187 lr:7.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:42:46 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.75 loss:0.007 grdn:0.186 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:44:11 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.78 loss:0.007 grdn:0.185 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:45:36 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.80 loss:0.008 grdn:0.184 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:47:01 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.83 loss:0.007 grdn:0.173 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 02:48:25 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.86 loss:0.007 grdn:0.181 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:49:50 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.89 loss:0.007 grdn:0.177 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:51:15 ts/train.py:232 step:312K smpl:2M ep:13K epch:42.91 loss:0.007 grdn:0.172 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:52:40 ts/train.py:232 step:312K smpl:2M ep:13K epch:42.94 loss:0.007 grdn:0.185 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:54:04 ts/train.py:232 step:312K smpl:2M ep:13K epch:42.97 loss:0.006 grdn:0.169 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:55:29 ts/train.py:232 step:312K smpl:2M ep:13K epch:43.00 loss:0.006 grdn:0.175 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:56:54 ts/train.py:232 step:312K smpl:2M ep:13K epch:43.02 loss:0.007 grdn:0.187 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:58:19 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.05 loss:0.007 grdn:0.187 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 02:59:43 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.08 loss:0.008 grdn:0.189 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:01:08 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.11 loss:0.006 grdn:0.166 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:02:33 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.13 loss:0.007 grdn:0.182 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:03:58 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.16 loss:0.007 grdn:0.184 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:05:22 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.19 loss:0.007 grdn:0.189 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:06:47 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.22 loss:0.007 grdn:0.176 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:08:12 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.24 loss:0.007 grdn:0.178 lr:7.3e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-07 03:09:37 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.27 loss:0.007 grdn:0.180 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:11:02 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.30 loss:0.007 grdn:0.176 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:12:27 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.33 loss:0.007 grdn:0.180 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:13:51 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.35 loss:0.007 grdn:0.194 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:15:18 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.38 loss:0.008 grdn:0.190 lr:7.3e-05 updt_s:0.422 data_s:0.008 +INFO 2025-05-07 03:16:42 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.41 loss:0.007 grdn:0.178 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:18:07 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.44 loss:0.007 grdn:0.175 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:19:32 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.46 loss:0.008 grdn:0.187 lr:7.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:20:57 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.49 loss:0.007 grdn:0.176 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:22:22 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.52 loss:0.007 grdn:0.177 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:23:46 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.55 loss:0.006 grdn:0.176 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:25:11 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.57 loss:0.007 grdn:0.174 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:26:35 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.60 loss:0.007 grdn:0.182 lr:7.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:28:00 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.63 loss:0.007 grdn:0.185 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:29:25 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.66 loss:0.006 grdn:0.171 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:30:49 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.68 loss:0.006 grdn:0.168 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:32:14 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.71 loss:0.007 grdn:0.174 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:33:39 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.74 loss:0.008 grdn:0.187 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:35:03 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.77 loss:0.008 grdn:0.188 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:36:28 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.79 loss:0.006 grdn:0.165 lr:7.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:37:53 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.82 loss:0.007 grdn:0.176 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:39:18 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.85 loss:0.007 grdn:0.180 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:40:42 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.88 loss:0.006 grdn:0.168 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:42:07 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.90 loss:0.007 grdn:0.179 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:43:32 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.93 loss:0.007 grdn:0.184 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:44:57 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.96 loss:0.007 grdn:0.184 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:46:21 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.99 loss:0.007 grdn:0.184 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:47:46 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.01 loss:0.007 grdn:0.182 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:49:11 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.04 loss:0.008 grdn:0.189 lr:7.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:50:36 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.07 loss:0.007 grdn:0.189 lr:7.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:52:01 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.10 loss:0.006 grdn:0.166 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:53:25 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.12 loss:0.007 grdn:0.171 lr:7.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:54:50 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.15 loss:0.007 grdn:0.188 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:56:15 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.18 loss:0.007 grdn:0.176 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 03:57:40 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.21 loss:0.007 grdn:0.176 lr:7.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 03:59:04 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.23 loss:0.007 grdn:0.178 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:00:29 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.26 loss:0.007 grdn:0.188 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:01:54 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.29 loss:0.006 grdn:0.188 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:03:18 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.32 loss:0.007 grdn:0.177 lr:7.2e-05 updt_s:0.421 data_s:0.001 +INFO 2025-05-07 04:04:44 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.34 loss:0.007 grdn:0.191 lr:7.2e-05 updt_s:0.421 data_s:0.008 +INFO 2025-05-07 04:06:09 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.37 loss:0.006 grdn:0.175 lr:7.2e-05 updt_s:0.421 data_s:0.001 +INFO 2025-05-07 04:07:33 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.40 loss:0.007 grdn:0.178 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:08:58 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.43 loss:0.007 grdn:0.182 lr:7.2e-05 updt_s:0.421 data_s:0.001 +INFO 2025-05-07 04:10:23 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.46 loss:0.006 grdn:0.170 lr:7.2e-05 updt_s:0.421 data_s:0.001 +INFO 2025-05-07 04:11:47 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.48 loss:0.007 grdn:0.184 lr:7.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:13:12 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.51 loss:0.007 grdn:0.174 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:14:36 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.54 loss:0.007 grdn:0.191 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:16:01 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.57 loss:0.007 grdn:0.173 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:17:26 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.59 loss:0.007 grdn:0.180 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:18:50 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.62 loss:0.007 grdn:0.183 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:20:15 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.65 loss:0.007 grdn:0.182 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:21:40 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.68 loss:0.007 grdn:0.180 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:23:04 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.70 loss:0.007 grdn:0.187 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:24:29 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.73 loss:0.007 grdn:0.183 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:25:54 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.76 loss:0.006 grdn:0.173 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:27:19 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.79 loss:0.008 grdn:0.198 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:28:44 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.81 loss:0.006 grdn:0.182 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:30:08 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.84 loss:0.008 grdn:0.195 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:31:33 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.87 loss:0.006 grdn:0.169 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:32:58 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.90 loss:0.007 grdn:0.178 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:34:23 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.92 loss:0.007 grdn:0.184 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:35:48 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.95 loss:0.007 grdn:0.176 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:37:12 ts/train.py:232 step:327K smpl:3M ep:13K epch:44.98 loss:0.008 grdn:0.197 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:38:37 ts/train.py:232 step:327K smpl:3M ep:14K epch:45.01 loss:0.007 grdn:0.184 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:40:02 ts/train.py:232 step:327K smpl:3M ep:14K epch:45.03 loss:0.007 grdn:0.185 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:41:27 ts/train.py:232 step:327K smpl:3M ep:14K epch:45.06 loss:0.006 grdn:0.173 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:42:52 ts/train.py:232 step:327K smpl:3M ep:14K epch:45.09 loss:0.006 grdn:0.162 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:44:17 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.12 loss:0.006 grdn:0.176 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:45:41 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.14 loss:0.006 grdn:0.177 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:47:06 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.17 loss:0.006 grdn:0.166 lr:7.1e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:48:31 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.20 loss:0.007 grdn:0.171 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:49:56 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.23 loss:0.007 grdn:0.183 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:51:21 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.25 loss:0.008 grdn:0.188 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:52:46 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.28 loss:0.007 grdn:0.180 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:54:12 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.31 loss:0.007 grdn:0.179 lr:7.1e-05 updt_s:0.422 data_s:0.008 +INFO 2025-05-07 04:55:36 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.34 loss:0.007 grdn:0.185 lr:7.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:57:01 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.36 loss:0.007 grdn:0.176 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 04:58:26 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.39 loss:0.006 grdn:0.174 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 04:59:51 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.42 loss:0.007 grdn:0.179 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:01:15 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.45 loss:0.007 grdn:0.179 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:01:15 ts/train.py:241 Checkpoint policy after step 330000 +INFO 2025-05-07 05:02:43 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.47 loss:0.007 grdn:0.186 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:04:08 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.50 loss:0.007 grdn:0.187 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:05:33 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.53 loss:0.007 grdn:0.177 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:06:58 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.56 loss:0.007 grdn:0.172 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:08:22 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.58 loss:0.007 grdn:0.183 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:09:47 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.61 loss:0.007 grdn:0.182 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:11:12 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.64 loss:0.007 grdn:0.180 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:12:37 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.67 loss:0.006 grdn:0.178 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:14:01 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.69 loss:0.007 grdn:0.193 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:15:26 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.72 loss:0.007 grdn:0.185 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:16:51 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.75 loss:0.007 grdn:0.181 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:18:16 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.78 loss:0.006 grdn:0.169 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:19:41 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.80 loss:0.007 grdn:0.179 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:21:06 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.83 loss:0.007 grdn:0.189 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:22:30 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.86 loss:0.007 grdn:0.180 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:23:55 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.89 loss:0.006 grdn:0.172 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:25:20 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.91 loss:0.007 grdn:0.181 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:26:45 ts/train.py:232 step:334K smpl:3M ep:14K epch:45.94 loss:0.006 grdn:0.166 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:28:10 ts/train.py:232 step:334K smpl:3M ep:14K epch:45.97 loss:0.006 grdn:0.180 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:29:35 ts/train.py:232 step:334K smpl:3M ep:14K epch:46.00 loss:0.007 grdn:0.194 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:30:59 ts/train.py:232 step:334K smpl:3M ep:14K epch:46.03 loss:0.007 grdn:0.176 lr:7.0e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 05:32:24 ts/train.py:232 step:334K smpl:3M ep:14K epch:46.05 loss:0.006 grdn:0.169 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:33:49 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.08 loss:0.007 grdn:0.185 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:35:14 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.11 loss:0.006 grdn:0.180 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:36:39 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.14 loss:0.007 grdn:0.185 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:38:04 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.16 loss:0.006 grdn:0.167 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:39:29 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.19 loss:0.007 grdn:0.186 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:40:54 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.22 loss:0.008 grdn:0.201 lr:7.0e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:42:19 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.25 loss:0.007 grdn:0.188 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:43:44 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.27 loss:0.007 grdn:0.177 lr:6.9e-05 updt_s:0.422 data_s:0.006 +INFO 2025-05-07 05:45:09 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.30 loss:0.007 grdn:0.187 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:46:34 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.33 loss:0.007 grdn:0.180 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:47:59 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.36 loss:0.007 grdn:0.189 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:49:24 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.38 loss:0.007 grdn:0.176 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:50:49 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.41 loss:0.008 grdn:0.196 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:52:14 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.44 loss:0.006 grdn:0.178 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:53:39 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.47 loss:0.006 grdn:0.173 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:55:04 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.49 loss:0.007 grdn:0.189 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:56:29 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.52 loss:0.007 grdn:0.185 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 05:57:54 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.55 loss:0.006 grdn:0.171 lr:6.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 05:59:19 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.58 loss:0.006 grdn:0.168 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:00:44 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.60 loss:0.006 grdn:0.167 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:02:09 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.63 loss:0.007 grdn:0.181 lr:6.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 06:03:34 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.66 loss:0.007 grdn:0.191 lr:6.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 06:04:59 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.69 loss:0.006 grdn:0.177 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:06:24 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.71 loss:0.006 grdn:0.167 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:07:49 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.74 loss:0.006 grdn:0.181 lr:6.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 06:09:14 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.77 loss:0.007 grdn:0.189 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:10:39 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.80 loss:0.007 grdn:0.177 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:12:04 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.82 loss:0.007 grdn:0.181 lr:6.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 06:13:29 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.85 loss:0.006 grdn:0.183 lr:6.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 06:14:54 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.88 loss:0.007 grdn:0.187 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:16:19 ts/train.py:232 step:341K smpl:3M ep:14K epch:46.91 loss:0.007 grdn:0.181 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:17:44 ts/train.py:232 step:341K smpl:3M ep:14K epch:46.93 loss:0.006 grdn:0.167 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:19:08 ts/train.py:232 step:341K smpl:3M ep:14K epch:46.96 loss:0.007 grdn:0.184 lr:6.9e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 06:20:33 ts/train.py:232 step:341K smpl:3M ep:14K epch:46.99 loss:0.007 grdn:0.180 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:21:58 ts/train.py:232 step:341K smpl:3M ep:14K epch:47.02 loss:0.006 grdn:0.172 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:23:23 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.04 loss:0.007 grdn:0.185 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:24:48 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.07 loss:0.006 grdn:0.173 lr:6.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:26:13 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.10 loss:0.007 grdn:0.186 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:27:37 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.13 loss:0.007 grdn:0.182 lr:6.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 06:29:02 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.15 loss:0.006 grdn:0.166 lr:6.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 06:30:27 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.18 loss:0.006 grdn:0.180 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:31:52 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.21 loss:0.007 grdn:0.181 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:33:18 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.24 loss:0.006 grdn:0.169 lr:6.8e-05 updt_s:0.422 data_s:0.008 +INFO 2025-05-07 06:34:43 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.26 loss:0.007 grdn:0.179 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:36:08 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.29 loss:0.007 grdn:0.189 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:37:33 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.32 loss:0.007 grdn:0.191 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:38:57 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.35 loss:0.006 grdn:0.178 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:40:22 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.37 loss:0.007 grdn:0.180 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:41:47 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.40 loss:0.007 grdn:0.188 lr:6.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 06:43:12 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.43 loss:0.006 grdn:0.170 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:44:37 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.46 loss:0.006 grdn:0.171 lr:6.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 06:46:02 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.48 loss:0.006 grdn:0.179 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:47:27 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.51 loss:0.008 grdn:0.196 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:48:51 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.54 loss:0.006 grdn:0.178 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:50:16 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.57 loss:0.007 grdn:0.181 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:51:41 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.60 loss:0.007 grdn:0.185 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:53:06 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.62 loss:0.008 grdn:0.210 lr:6.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 06:54:31 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.65 loss:0.007 grdn:0.176 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:55:56 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.68 loss:0.007 grdn:0.177 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:57:20 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.71 loss:0.007 grdn:0.183 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 06:58:45 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.73 loss:0.006 grdn:0.176 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:00:10 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.76 loss:0.006 grdn:0.176 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:01:35 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.79 loss:0.006 grdn:0.183 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:03:00 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.82 loss:0.007 grdn:0.180 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:04:24 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.84 loss:0.006 grdn:0.163 lr:6.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:05:49 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.87 loss:0.007 grdn:0.182 lr:6.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 07:07:14 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.90 loss:0.007 grdn:0.184 lr:6.8e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 07:08:39 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.93 loss:0.006 grdn:0.183 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:10:04 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.95 loss:0.007 grdn:0.182 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:11:28 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.98 loss:0.007 grdn:0.193 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:12:53 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.01 loss:0.007 grdn:0.194 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:14:18 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.04 loss:0.007 grdn:0.191 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:15:43 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.06 loss:0.007 grdn:0.181 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:17:08 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.09 loss:0.007 grdn:0.186 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:18:33 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.12 loss:0.007 grdn:0.175 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:19:58 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.15 loss:0.007 grdn:0.196 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:21:23 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.17 loss:0.007 grdn:0.189 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:22:50 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.20 loss:0.007 grdn:0.187 lr:6.7e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-07 07:24:15 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.23 loss:0.006 grdn:0.171 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:25:40 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.26 loss:0.007 grdn:0.179 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:27:05 ts/train.py:232 step:351K smpl:3M ep:14K epch:48.28 loss:0.007 grdn:0.182 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:28:30 ts/train.py:232 step:351K smpl:3M ep:14K epch:48.31 loss:0.007 grdn:0.183 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:29:55 ts/train.py:232 step:351K smpl:3M ep:15K epch:48.34 loss:0.006 grdn:0.179 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:31:20 ts/train.py:232 step:351K smpl:3M ep:15K epch:48.37 loss:0.007 grdn:0.190 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:32:45 ts/train.py:232 step:351K smpl:3M ep:15K epch:48.39 loss:0.006 grdn:0.180 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:34:10 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.42 loss:0.006 grdn:0.167 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:35:35 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.45 loss:0.007 grdn:0.184 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:37:00 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.48 loss:0.006 grdn:0.177 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:38:25 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.50 loss:0.007 grdn:0.180 lr:6.7e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 07:39:50 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.53 loss:0.006 grdn:0.176 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:41:15 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.56 loss:0.007 grdn:0.182 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:42:40 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.59 loss:0.007 grdn:0.185 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:44:04 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.61 loss:0.007 grdn:0.184 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:45:29 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.64 loss:0.007 grdn:0.188 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:46:54 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.67 loss:0.006 grdn:0.180 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:48:19 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.70 loss:0.007 grdn:0.178 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:49:44 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.72 loss:0.006 grdn:0.170 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:51:09 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.75 loss:0.006 grdn:0.170 lr:6.7e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:52:34 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.78 loss:0.006 grdn:0.177 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:53:59 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.81 loss:0.007 grdn:0.183 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:55:24 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.83 loss:0.005 grdn:0.160 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:56:49 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.86 loss:0.006 grdn:0.174 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:58:13 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.89 loss:0.006 grdn:0.176 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 07:59:38 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.92 loss:0.007 grdn:0.182 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:01:03 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.94 loss:0.006 grdn:0.175 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:02:28 ts/train.py:232 step:356K smpl:3M ep:15K epch:48.97 loss:0.008 grdn:0.191 lr:6.6e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 08:03:53 ts/train.py:232 step:356K smpl:3M ep:15K epch:49.00 loss:0.007 grdn:0.195 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:05:18 ts/train.py:232 step:356K smpl:3M ep:15K epch:49.03 loss:0.006 grdn:0.181 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:06:43 ts/train.py:232 step:356K smpl:3M ep:15K epch:49.05 loss:0.006 grdn:0.178 lr:6.6e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 08:08:08 ts/train.py:232 step:356K smpl:3M ep:15K epch:49.08 loss:0.006 grdn:0.167 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:09:33 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.11 loss:0.006 grdn:0.170 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:10:58 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.14 loss:0.006 grdn:0.170 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:12:24 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.17 loss:0.007 grdn:0.198 lr:6.6e-05 updt_s:0.423 data_s:0.007 +INFO 2025-05-07 08:13:49 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.19 loss:0.006 grdn:0.170 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:15:14 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.22 loss:0.006 grdn:0.175 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:16:39 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.25 loss:0.006 grdn:0.179 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:18:04 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.28 loss:0.006 grdn:0.178 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:19:29 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.30 loss:0.007 grdn:0.183 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:20:54 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.33 loss:0.007 grdn:0.177 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:22:19 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.36 loss:0.006 grdn:0.173 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:23:44 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.39 loss:0.006 grdn:0.179 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:25:09 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.41 loss:0.007 grdn:0.182 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:26:34 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.44 loss:0.007 grdn:0.181 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:27:59 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.47 loss:0.006 grdn:0.176 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:29:23 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.50 loss:0.006 grdn:0.175 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:30:48 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.52 loss:0.007 grdn:0.186 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:32:13 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.55 loss:0.006 grdn:0.179 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:33:38 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.58 loss:0.006 grdn:0.169 lr:6.6e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:33:38 ts/train.py:241 Checkpoint policy after step 360000 +INFO 2025-05-07 08:35:07 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.61 loss:0.007 grdn:0.187 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:36:32 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.63 loss:0.006 grdn:0.175 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:37:57 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.66 loss:0.007 grdn:0.188 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:39:22 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.69 loss:0.006 grdn:0.179 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:40:47 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.72 loss:0.006 grdn:0.171 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 08:42:12 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.74 loss:0.006 grdn:0.184 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:43:37 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.77 loss:0.007 grdn:0.180 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:45:02 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.80 loss:0.007 grdn:0.186 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:46:27 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.83 loss:0.006 grdn:0.177 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 08:47:52 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.85 loss:0.006 grdn:0.177 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 08:49:17 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.88 loss:0.007 grdn:0.187 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 08:50:42 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.91 loss:0.006 grdn:0.174 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:52:07 ts/train.py:232 step:363K smpl:3M ep:15K epch:49.94 loss:0.006 grdn:0.172 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:53:32 ts/train.py:232 step:363K smpl:3M ep:15K epch:49.96 loss:0.006 grdn:0.178 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 08:54:57 ts/train.py:232 step:363K smpl:3M ep:15K epch:49.99 loss:0.007 grdn:0.192 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 08:56:22 ts/train.py:232 step:363K smpl:3M ep:15K epch:50.02 loss:0.006 grdn:0.175 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 08:57:47 ts/train.py:232 step:363K smpl:3M ep:15K epch:50.05 loss:0.006 grdn:0.170 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 08:59:12 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.07 loss:0.006 grdn:0.179 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:00:37 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.10 loss:0.006 grdn:0.172 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:02:04 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.13 loss:0.006 grdn:0.179 lr:6.5e-05 updt_s:0.423 data_s:0.010 +INFO 2025-05-07 09:03:29 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.16 loss:0.006 grdn:0.180 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 09:04:54 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.18 loss:0.006 grdn:0.174 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:06:19 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.21 loss:0.006 grdn:0.178 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 09:07:44 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.24 loss:0.006 grdn:0.178 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:09:09 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.27 loss:0.006 grdn:0.178 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:10:34 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.29 loss:0.005 grdn:0.168 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:11:59 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.32 loss:0.007 grdn:0.182 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:13:24 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.35 loss:0.007 grdn:0.190 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:14:49 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.38 loss:0.006 grdn:0.163 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:16:13 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.40 loss:0.005 grdn:0.170 lr:6.5e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:17:38 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.43 loss:0.006 grdn:0.170 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:19:03 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.46 loss:0.006 grdn:0.177 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:20:28 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.49 loss:0.006 grdn:0.178 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:21:53 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.51 loss:0.006 grdn:0.183 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:23:17 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.54 loss:0.006 grdn:0.176 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:24:42 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.57 loss:0.006 grdn:0.185 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:26:07 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.60 loss:0.006 grdn:0.173 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:27:32 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.62 loss:0.006 grdn:0.179 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:28:56 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.65 loss:0.006 grdn:0.174 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:30:21 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.68 loss:0.006 grdn:0.182 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:31:46 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.71 loss:0.006 grdn:0.173 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:33:10 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.74 loss:0.007 grdn:0.193 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:34:35 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.76 loss:0.007 grdn:0.185 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:36:00 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.79 loss:0.006 grdn:0.175 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:37:25 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.82 loss:0.007 grdn:0.192 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:38:49 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.85 loss:0.006 grdn:0.177 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:40:14 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.87 loss:0.006 grdn:0.177 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:41:39 ts/train.py:232 step:370K smpl:3M ep:15K epch:50.90 loss:0.006 grdn:0.177 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:43:04 ts/train.py:232 step:370K smpl:3M ep:15K epch:50.93 loss:0.006 grdn:0.179 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:44:29 ts/train.py:232 step:370K smpl:3M ep:15K epch:50.96 loss:0.007 grdn:0.177 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:45:54 ts/train.py:232 step:370K smpl:3M ep:15K epch:50.98 loss:0.007 grdn:0.177 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:47:18 ts/train.py:232 step:370K smpl:3M ep:15K epch:51.01 loss:0.006 grdn:0.184 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:48:43 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.04 loss:0.007 grdn:0.196 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:50:08 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.07 loss:0.006 grdn:0.176 lr:6.4e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 09:51:35 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.09 loss:0.007 grdn:0.186 lr:6.4e-05 updt_s:0.422 data_s:0.010 +INFO 2025-05-07 09:53:00 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.12 loss:0.006 grdn:0.178 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:54:25 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.15 loss:0.006 grdn:0.176 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:55:49 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.18 loss:0.007 grdn:0.193 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:57:14 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.20 loss:0.006 grdn:0.177 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 09:58:39 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.23 loss:0.006 grdn:0.180 lr:6.4e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:00:04 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.26 loss:0.007 grdn:0.183 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:01:29 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.29 loss:0.006 grdn:0.182 lr:6.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 10:02:54 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.31 loss:0.007 grdn:0.180 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:04:18 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.34 loss:0.007 grdn:0.191 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:05:43 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.37 loss:0.006 grdn:0.173 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:07:08 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.40 loss:0.006 grdn:0.177 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:08:33 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.42 loss:0.006 grdn:0.178 lr:6.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 10:09:57 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.45 loss:0.006 grdn:0.176 lr:6.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 10:11:22 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.48 loss:0.006 grdn:0.177 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:12:47 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.51 loss:0.006 grdn:0.185 lr:6.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 10:14:12 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.53 loss:0.006 grdn:0.177 lr:6.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 10:15:37 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.56 loss:0.007 grdn:0.182 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:17:01 ts/train.py:232 step:375K smpl:3M ep:15K epch:51.59 loss:0.006 grdn:0.185 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:18:26 ts/train.py:232 step:375K smpl:3M ep:15K epch:51.62 loss:0.007 grdn:0.189 lr:6.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 10:19:51 ts/train.py:232 step:375K smpl:3M ep:15K epch:51.64 loss:0.005 grdn:0.169 lr:6.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 10:21:16 ts/train.py:232 step:375K smpl:3M ep:16K epch:51.67 loss:0.006 grdn:0.184 lr:6.3e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 10:22:41 ts/train.py:232 step:375K smpl:3M ep:16K epch:51.70 loss:0.006 grdn:0.178 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:24:05 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.73 loss:0.007 grdn:0.186 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:25:30 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.75 loss:0.006 grdn:0.178 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:26:55 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.78 loss:0.007 grdn:0.188 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:28:20 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.81 loss:0.007 grdn:0.183 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:29:45 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.84 loss:0.007 grdn:0.181 lr:6.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 10:31:10 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.86 loss:0.007 grdn:0.184 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:32:35 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.89 loss:0.006 grdn:0.187 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:34:00 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.92 loss:0.006 grdn:0.183 lr:6.3e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:35:25 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.95 loss:0.006 grdn:0.175 lr:6.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 10:36:50 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.97 loss:0.006 grdn:0.177 lr:6.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 10:38:15 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.00 loss:0.006 grdn:0.176 lr:6.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 10:39:40 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.03 loss:0.006 grdn:0.181 lr:6.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 10:41:06 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.06 loss:0.006 grdn:0.183 lr:6.3e-05 updt_s:0.423 data_s:0.007 +INFO 2025-05-07 10:42:31 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.08 loss:0.006 grdn:0.176 lr:6.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 10:43:56 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.11 loss:0.006 grdn:0.173 lr:6.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 10:45:21 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.14 loss:0.005 grdn:0.166 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:46:46 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.17 loss:0.006 grdn:0.184 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:48:11 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.19 loss:0.006 grdn:0.178 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:49:36 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.22 loss:0.006 grdn:0.177 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:51:01 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.25 loss:0.006 grdn:0.177 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:52:26 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.28 loss:0.006 grdn:0.171 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:53:51 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.31 loss:0.007 grdn:0.176 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:55:16 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.33 loss:0.006 grdn:0.184 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:56:40 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.36 loss:0.006 grdn:0.175 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:58:05 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.39 loss:0.006 grdn:0.177 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 10:59:30 ts/train.py:232 step:381K smpl:3M ep:16K epch:52.42 loss:0.006 grdn:0.174 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:00:55 ts/train.py:232 step:381K smpl:3M ep:16K epch:52.44 loss:0.007 grdn:0.188 lr:6.2e-05 updt_s:0.422 data_s:0.001 +INFO 2025-05-07 11:02:20 ts/train.py:232 step:381K smpl:3M ep:16K epch:52.47 loss:0.006 grdn:0.178 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:03:45 ts/train.py:232 step:381K smpl:3M ep:16K epch:52.50 loss:0.005 grdn:0.168 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:05:10 ts/train.py:232 step:381K smpl:3M ep:16K epch:52.53 loss:0.006 grdn:0.176 lr:6.2e-05 updt_s:0.423 data_s:0.000 +INFO 2025-05-07 11:06:34 ts/train.py:232 step:382K smpl:3M ep:16K epch:52.55 loss:0.006 grdn:0.184 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:07:59 ts/train.py:232 step:382K smpl:3M ep:16K epch:52.58 loss:0.006 grdn:0.188 lr:6.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:09:24 ts/train.py:232 step:382K smpl:3M ep:16K epch:52.61 loss:0.006 grdn:0.179 lr:6.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:10:49 ts/train.py:232 step:382K smpl:3M ep:16K epch:52.64 loss:0.006 grdn:0.180 lr:6.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:12:14 ts/train.py:232 step:382K smpl:3M ep:16K epch:52.66 loss:0.006 grdn:0.178 lr:6.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:13:39 ts/train.py:232 step:383K smpl:3M ep:16K epch:52.69 loss:0.006 grdn:0.184 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:15:04 ts/train.py:232 step:383K smpl:3M ep:16K epch:52.72 loss:0.006 grdn:0.186 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:16:29 ts/train.py:232 step:383K smpl:3M ep:16K epch:52.75 loss:0.006 grdn:0.172 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:17:54 ts/train.py:232 step:383K smpl:3M ep:16K epch:52.77 loss:0.006 grdn:0.175 lr:6.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:19:19 ts/train.py:232 step:383K smpl:3M ep:16K epch:52.80 loss:0.006 grdn:0.183 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:20:44 ts/train.py:232 step:384K smpl:3M ep:16K epch:52.83 loss:0.006 grdn:0.172 lr:6.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:22:09 ts/train.py:232 step:384K smpl:3M ep:16K epch:52.86 loss:0.006 grdn:0.183 lr:6.2e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:23:34 ts/train.py:232 step:384K smpl:3M ep:16K epch:52.88 loss:0.006 grdn:0.178 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:24:59 ts/train.py:232 step:384K smpl:3M ep:16K epch:52.91 loss:0.005 grdn:0.168 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:26:24 ts/train.py:232 step:384K smpl:3M ep:16K epch:52.94 loss:0.006 grdn:0.179 lr:6.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 11:27:49 ts/train.py:232 step:385K smpl:3M ep:16K epch:52.97 loss:0.007 grdn:0.191 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:29:14 ts/train.py:232 step:385K smpl:3M ep:16K epch:52.99 loss:0.006 grdn:0.174 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:30:40 ts/train.py:232 step:385K smpl:3M ep:16K epch:53.02 loss:0.006 grdn:0.180 lr:6.1e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-07 11:32:05 ts/train.py:232 step:385K smpl:3M ep:16K epch:53.05 loss:0.006 grdn:0.179 lr:6.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-07 11:33:30 ts/train.py:232 step:385K smpl:3M ep:16K epch:53.08 loss:0.007 grdn:0.183 lr:6.1e-05 updt_s:0.423 data_s:0.001 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log index 12cf918367d22cc11e03be8cc2518db67592db79..eb106f8bffbb1de9a45650186cca3cf6bd2790c4 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log @@ -11,3 +11,5 @@ {"time":"2025-05-06T10:30:03.039602464Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:41710->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T12:25:33.040402545Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:56185->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T12:44:48.040371676Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:43741->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T16:50:48.039369614Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:38891->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T18:02:28.241678384Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb index f948fc4a32a2436624707a9cf4e63ff78424d972..fdd48abb34d1c18998692657f13cfd3e37850457 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d888a831344268b1be8c52b9993e710067272ba5ae0436d3b10c613c95bcf1ff -size 19431424 +oid sha256:f42dd539beda5413d25e35410a3c4bdb0d43c667fd6a26ed875149d6e1db6b13 +size 36175872