diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78fe4ca386029187083599910e8174c90dc3bd6b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dca94a59fd08bd60b2e1e59d07a7dd382a26885f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2738d07c64d0368f729ca5d6ff8c045593a03cd8831f33133b5e1a2944c2d05d +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..440cbe09b9cb94b2336540652baab494d823f71e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..a2ebd55b052e1e980a503765e694b0de8af480ec --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 8.720200055053977e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..12f9e579d1edb3e5cc933d00ce896244faf79545 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:806d155d395a1a44883b4815873d9105d962b2abdb4393e6061316e2969c83b3 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9e3968815529d98948da3c9d18ae94fd26a14bb5 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25054dbb5044f555604cb471ae674206455ee244dd05c8f7350c21c52aa64f6d +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..97ab942fad8a77c663b0037063c5951ba5dbc781 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 210000, + "verbose": false, + "_step_count": 210001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 8.720200055053977e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..6147d40ee6d82bb05c893a1b35d8f59f302da656 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/210000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 210000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78fe4ca386029187083599910e8174c90dc3bd6b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..06be46032dd4214a9170b5588c191013297c897f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b10e8039b4e67bb172e0d441996c664534510d840d10ae9215ffa6782bb8de93 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..440cbe09b9cb94b2336540652baab494d823f71e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..b973d80db1e60787fa3565daf1f4712fdb7b364e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 8.350408716128216e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..621f1b9bbcbd9119b310a40663b072e63857d789 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d46199c0717200c2096f44f43b3d7c686a576a2b9b00edae493e02aa505e3590 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d9cadb48c4523819f6d08bedc47a99237fa03f48 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fe8d6a2a3f36c3decee752a91800282faeffbdab6d1a00981634c6da1ce7097 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..20c00e1b8846ac18ae698337037774699743306a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 240000, + "verbose": false, + "_step_count": 240001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 8.350408716128216e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..cd776dcc291fdcc40079da5c594255ca04238bce --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/240000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 240000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78fe4ca386029187083599910e8174c90dc3bd6b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0be9b4b4defc951c49e0310e80b4f5b4e5bbf481 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d412314059a338e248058788b901c8e74e08d7ad91f15cd49b5bbae1c162bf91 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..440cbe09b9cb94b2336540652baab494d823f71e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1085c71f91668de124c2f5269441008dc91a6665 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.943868815107594e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..25485bf2fa3554657108fe99a16c6e302de6f54c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4e63511f776418fbdf6b595270d6110b332528fc1a3bd8b47873e5ef2b3a427 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f1045fda35ae3e8b8b8021a41b8af855d40fa2f3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bec12d7e5fe4aa7967aafcf9e0bde003eb34912f1fa7ee2e8951034fa44a765 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e2fc5c0727d511fc139fe9e28a78e368a31b04b7 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 270000, + "verbose": false, + "_step_count": 270001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.943868815107594e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..537f4e9f0bf2092976dffaa70a673dde4c3fd66d --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/270000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 270000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78fe4ca386029187083599910e8174c90dc3bd6b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..39c3eb575aaceefc662b5f1d31019745514a4a37 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3af5d5c95f6ace62b7a687dd3c80ddd5c2d91c2159f01d028494f26cc7ba142b +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..440cbe09b9cb94b2336540652baab494d823f71e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..441b7eada7faf4a78ccaa1e14c1d8083b6e9035c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.505039436856827e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c7b5850a91110e59c7b6b52f9ebcf6492890e297 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab126b4cefe5888b3fde9a5776f2069d590867b30c00fa2c92ecc1334bf41a4b +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..afef1085ac2382824aa588a648ccf121aeff9f50 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7014000a6ab6c57ea9b868f3958996e23692519eb7903184e1affdcf0c9f8bd8 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..46f0924834037be23d642823ee826bc729c90d27 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 300000, + "verbose": false, + "_step_count": 300001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.505039436856827e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..db7e948ddab8257b846ad5ef28f8c555a9524598 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/300000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 300000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78fe4ca386029187083599910e8174c90dc3bd6b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bc0c74a835e1916dc16472d77fe49d0b1d66d345 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26d6674523899c87345208013d020896aa3441d17723f11de8f1932648af2e99 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..440cbe09b9cb94b2336540652baab494d823f71e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..cb2ac96baeaf8f3382dc8f0f62087f73edc7f970 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.038733829556077e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ec956508321dbe934d53ca6916ec308ce8b21749 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8691d6e38cff4acf4a4eef0adee01c0f89a9741a45e1966bd060f3450cd96c5f +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..76b5567074e9bae6e613299d85583cfb05bf1c09 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78cbe464e6e610ece64881b538d057b203339611dc48270e21ba01b289aead17 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..594d01a0a1b45b388016e6e6c58a1074f424337e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 330000, + "verbose": false, + "_step_count": 330001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.038733829556077e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..f5ea2cc3e4569f3f5076d82101c911d3ca4dadb3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/330000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 330000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..78fe4ca386029187083599910e8174c90dc3bd6b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..044e7bd72135df9b1666c1963808613d5131a3bd --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b5ff5b6f44c3275d2639214379a768bd295ae9cf80ad7305e91c794e9a220b5 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..440cbe09b9cb94b2336540652baab494d823f71e --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 16, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..c95ffe05c43624624f92810f8806852a0be08830 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.550066611157049e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d5e5f25f4af7078cd7d06e2fc11a0d172d8e2bad --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe3fc4edca3bcbe6b1479677c0642ef27bd7b8dd72c6800d8a1414a4926bfdbb +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7d23ceb6fc8848b24ba3178f641719f5e15ee72f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6bca1a2cb1a63f65963296b05cfc7369f494dd2898cf4490d6f97d5a32913ed +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a39d6e5bb2a4966dfcb991d5bc882f8c4dadb29b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 360000, + "verbose": false, + "_step_count": 360001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.550066611157049e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..bd79b4d9c6aaae91547528200cf9573568cb1ede --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/checkpoints/360000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 360000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log index 92598c742a153b961b64179845bcc4ecb500cc40..6450f58d0252a6a4885872d37011b5eeb9df7503 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/debug-internal.log @@ -8,3 +8,6 @@ {"time":"2025-05-05T17:38:08.29480052Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/ygbytty1/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} {"time":"2025-05-06T09:12:57.512959236Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/ygbytty1/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49420->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T11:39:57.516626433Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/ygbytty1/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:59675->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T18:01:54.895827141Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2025-05-06T18:02:07.71693416Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/ygbytty1/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-05-06T22:47:19.821719943Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/files/output.log b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/files/output.log index 1d48f601bad59faa3fef49eedf54774e0f040517..dcfb59857c624b4f48ecdd6fb7095a6406b2de5e 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/files/output.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/files/output.log @@ -1036,3 +1036,894 @@ INFO 2025-05-06 14:13:09 ts/train.py:232 step:203K smpl:2M ep:8K epch:27.98 loss INFO 2025-05-06 14:14:34 ts/train.py:232 step:203K smpl:2M ep:8K epch:28.01 loss:0.011 grdn:0.243 lr:8.8e-05 updt_s:0.427 data_s:0.001 INFO 2025-05-06 14:16:00 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.04 loss:0.010 grdn:0.231 lr:8.8e-05 updt_s:0.428 data_s:0.001 INFO 2025-05-06 14:17:26 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.07 loss:0.011 grdn:0.239 lr:8.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-06 14:18:52 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.09 loss:0.009 grdn:0.216 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:20:18 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.12 loss:0.010 grdn:0.238 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:21:43 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.15 loss:0.010 grdn:0.233 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:23:09 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.18 loss:0.010 grdn:0.233 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:24:35 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.20 loss:0.010 grdn:0.225 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:26:00 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.23 loss:0.011 grdn:0.234 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:27:26 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.26 loss:0.010 grdn:0.229 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:28:52 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.29 loss:0.010 grdn:0.231 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:30:18 ts/train.py:232 step:206K smpl:2M ep:8K epch:28.31 loss:0.011 grdn:0.245 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:31:43 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.34 loss:0.011 grdn:0.244 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:33:09 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.37 loss:0.010 grdn:0.230 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:34:35 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.40 loss:0.009 grdn:0.220 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:36:00 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.42 loss:0.010 grdn:0.230 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:37:26 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.45 loss:0.009 grdn:0.224 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:38:52 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.48 loss:0.010 grdn:0.222 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:40:18 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.51 loss:0.010 grdn:0.242 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:41:43 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.54 loss:0.011 grdn:0.228 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:43:09 ts/train.py:232 step:207K smpl:2M ep:9K epch:28.56 loss:0.010 grdn:0.221 lr:8.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:44:35 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.59 loss:0.010 grdn:0.230 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:46:00 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.62 loss:0.011 grdn:0.247 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:47:26 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.65 loss:0.009 grdn:0.218 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:48:52 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.67 loss:0.009 grdn:0.221 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:50:17 ts/train.py:232 step:208K smpl:2M ep:9K epch:28.70 loss:0.008 grdn:0.195 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:51:43 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.73 loss:0.009 grdn:0.230 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:53:09 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.76 loss:0.010 grdn:0.236 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 14:54:34 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.78 loss:0.009 grdn:0.207 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:56:00 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.81 loss:0.009 grdn:0.219 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:57:26 ts/train.py:232 step:209K smpl:2M ep:9K epch:28.84 loss:0.011 grdn:0.245 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 14:58:51 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.87 loss:0.010 grdn:0.236 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:00:17 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.89 loss:0.009 grdn:0.218 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:01:44 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.92 loss:0.011 grdn:0.244 lr:8.7e-05 updt_s:0.427 data_s:0.009 +INFO 2025-05-06 15:01:44 ts/train.py:241 Checkpoint policy after step 210000 +INFO 2025-05-06 15:03:13 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.95 loss:0.009 grdn:0.221 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:04:39 ts/train.py:232 step:210K smpl:2M ep:9K epch:28.98 loss:0.009 grdn:0.227 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:06:04 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.00 loss:0.010 grdn:0.224 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:07:30 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.03 loss:0.010 grdn:0.234 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:08:56 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.06 loss:0.010 grdn:0.233 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:10:21 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.09 loss:0.008 grdn:0.206 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:11:47 ts/train.py:232 step:211K smpl:2M ep:9K epch:29.11 loss:0.012 grdn:0.261 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:13:13 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.14 loss:0.009 grdn:0.218 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:14:38 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.17 loss:0.009 grdn:0.223 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:16:04 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.20 loss:0.011 grdn:0.245 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:17:30 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.22 loss:0.011 grdn:0.249 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:18:55 ts/train.py:232 step:212K smpl:2M ep:9K epch:29.25 loss:0.010 grdn:0.232 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:20:21 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.28 loss:0.010 grdn:0.228 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:21:46 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.31 loss:0.011 grdn:0.239 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:23:12 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.33 loss:0.010 grdn:0.231 lr:8.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:24:38 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.36 loss:0.010 grdn:0.238 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:26:03 ts/train.py:232 step:213K smpl:2M ep:9K epch:29.39 loss:0.010 grdn:0.233 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:27:29 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.42 loss:0.010 grdn:0.219 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:28:54 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.44 loss:0.009 grdn:0.220 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:30:20 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.47 loss:0.010 grdn:0.243 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:31:45 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.50 loss:0.010 grdn:0.232 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:33:11 ts/train.py:232 step:214K smpl:2M ep:9K epch:29.53 loss:0.011 grdn:0.244 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:34:37 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.55 loss:0.011 grdn:0.249 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:36:02 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.58 loss:0.010 grdn:0.230 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:37:28 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.61 loss:0.009 grdn:0.223 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:38:53 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.64 loss:0.010 grdn:0.229 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:40:19 ts/train.py:232 step:215K smpl:2M ep:9K epch:29.66 loss:0.009 grdn:0.214 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:41:44 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.69 loss:0.008 grdn:0.217 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:43:10 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.72 loss:0.010 grdn:0.234 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:44:36 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.75 loss:0.009 grdn:0.223 lr:8.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:46:01 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.77 loss:0.010 grdn:0.227 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:47:27 ts/train.py:232 step:216K smpl:2M ep:9K epch:29.80 loss:0.011 grdn:0.251 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:48:52 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.83 loss:0.010 grdn:0.227 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:50:18 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.86 loss:0.010 grdn:0.223 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:51:45 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.88 loss:0.010 grdn:0.229 lr:8.6e-05 updt_s:0.426 data_s:0.009 +INFO 2025-05-06 15:53:10 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.91 loss:0.009 grdn:0.220 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:54:36 ts/train.py:232 step:217K smpl:2M ep:9K epch:29.94 loss:0.011 grdn:0.232 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:56:02 ts/train.py:232 step:218K smpl:2M ep:9K epch:29.97 loss:0.008 grdn:0.215 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 15:57:27 ts/train.py:232 step:218K smpl:2M ep:9K epch:29.99 loss:0.009 grdn:0.225 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 15:58:53 ts/train.py:232 step:218K smpl:2M ep:9K epch:30.02 loss:0.009 grdn:0.213 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:00:19 ts/train.py:232 step:218K smpl:2M ep:9K epch:30.05 loss:0.009 grdn:0.219 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:01:44 ts/train.py:232 step:218K smpl:2M ep:9K epch:30.08 loss:0.008 grdn:0.206 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:03:10 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.11 loss:0.010 grdn:0.237 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:04:36 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.13 loss:0.010 grdn:0.234 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:06:01 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.16 loss:0.010 grdn:0.224 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:07:27 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.19 loss:0.009 grdn:0.213 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:08:53 ts/train.py:232 step:219K smpl:2M ep:9K epch:30.22 loss:0.009 grdn:0.230 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:10:18 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.24 loss:0.011 grdn:0.247 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:11:44 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.27 loss:0.010 grdn:0.224 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:13:09 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.30 loss:0.009 grdn:0.226 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:14:35 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.33 loss:0.009 grdn:0.219 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:16:01 ts/train.py:232 step:220K smpl:2M ep:9K epch:30.35 loss:0.009 grdn:0.222 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:17:26 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.38 loss:0.010 grdn:0.236 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:18:52 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.41 loss:0.010 grdn:0.232 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:20:18 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.44 loss:0.009 grdn:0.223 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:21:43 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.46 loss:0.011 grdn:0.230 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:23:09 ts/train.py:232 step:221K smpl:2M ep:9K epch:30.49 loss:0.009 grdn:0.218 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:24:35 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.52 loss:0.010 grdn:0.233 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:26:00 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.55 loss:0.010 grdn:0.234 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:27:26 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.57 loss:0.009 grdn:0.218 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:28:52 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.60 loss:0.010 grdn:0.236 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:30:17 ts/train.py:232 step:222K smpl:2M ep:9K epch:30.63 loss:0.010 grdn:0.233 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:31:43 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.66 loss:0.009 grdn:0.233 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:33:09 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.68 loss:0.009 grdn:0.220 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:34:34 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.71 loss:0.010 grdn:0.243 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:36:00 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.74 loss:0.010 grdn:0.222 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:37:26 ts/train.py:232 step:223K smpl:2M ep:9K epch:30.77 loss:0.010 grdn:0.240 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:38:51 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.79 loss:0.009 grdn:0.218 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:40:17 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.82 loss:0.009 grdn:0.236 lr:8.6e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:41:44 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.85 loss:0.010 grdn:0.236 lr:8.6e-05 updt_s:0.426 data_s:0.008 +INFO 2025-05-06 16:43:09 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.88 loss:0.011 grdn:0.245 lr:8.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:44:35 ts/train.py:232 step:224K smpl:2M ep:9K epch:30.90 loss:0.009 grdn:0.225 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 16:46:01 ts/train.py:232 step:225K smpl:2M ep:9K epch:30.93 loss:0.008 grdn:0.206 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:47:26 ts/train.py:232 step:225K smpl:2M ep:9K epch:30.96 loss:0.009 grdn:0.220 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:48:52 ts/train.py:232 step:225K smpl:2M ep:9K epch:30.99 loss:0.009 grdn:0.220 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:50:17 ts/train.py:232 step:225K smpl:2M ep:9K epch:31.01 loss:0.010 grdn:0.236 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:51:43 ts/train.py:232 step:225K smpl:2M ep:9K epch:31.04 loss:0.009 grdn:0.216 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:53:08 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.07 loss:0.010 grdn:0.237 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:54:34 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.10 loss:0.009 grdn:0.213 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:55:59 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.12 loss:0.009 grdn:0.239 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:57:25 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.15 loss:0.009 grdn:0.215 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 16:58:50 ts/train.py:232 step:226K smpl:2M ep:9K epch:31.18 loss:0.009 grdn:0.219 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:00:16 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.21 loss:0.010 grdn:0.230 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:01:41 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.23 loss:0.010 grdn:0.230 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:03:07 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.26 loss:0.010 grdn:0.238 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:04:32 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.29 loss:0.009 grdn:0.230 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:05:58 ts/train.py:232 step:227K smpl:2M ep:9K epch:31.32 loss:0.010 grdn:0.241 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:07:23 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.34 loss:0.010 grdn:0.225 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:08:49 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.37 loss:0.010 grdn:0.229 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:10:14 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.40 loss:0.010 grdn:0.238 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:11:40 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.43 loss:0.011 grdn:0.247 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:13:05 ts/train.py:232 step:228K smpl:2M ep:9K epch:31.45 loss:0.010 grdn:0.231 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:14:31 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.48 loss:0.011 grdn:0.242 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:15:57 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.51 loss:0.010 grdn:0.231 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:17:22 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.54 loss:0.009 grdn:0.221 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:18:48 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.56 loss:0.010 grdn:0.234 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:20:13 ts/train.py:232 step:229K smpl:2M ep:9K epch:31.59 loss:0.010 grdn:0.229 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:21:39 ts/train.py:232 step:230K smpl:2M ep:9K epch:31.62 loss:0.010 grdn:0.236 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:23:05 ts/train.py:232 step:230K smpl:2M ep:9K epch:31.65 loss:0.010 grdn:0.230 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:24:30 ts/train.py:232 step:230K smpl:2M ep:10K epch:31.67 loss:0.008 grdn:0.213 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:25:56 ts/train.py:232 step:230K smpl:2M ep:10K epch:31.70 loss:0.009 grdn:0.212 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:27:22 ts/train.py:232 step:230K smpl:2M ep:10K epch:31.73 loss:0.010 grdn:0.224 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:28:48 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.76 loss:0.009 grdn:0.222 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:30:13 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.79 loss:0.010 grdn:0.231 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:31:40 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.81 loss:0.009 grdn:0.223 lr:8.5e-05 updt_s:0.426 data_s:0.009 +INFO 2025-05-06 17:33:06 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.84 loss:0.010 grdn:0.232 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:34:32 ts/train.py:232 step:231K smpl:2M ep:10K epch:31.87 loss:0.008 grdn:0.219 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:35:57 ts/train.py:232 step:232K smpl:2M ep:10K epch:31.90 loss:0.009 grdn:0.211 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:37:23 ts/train.py:232 step:232K smpl:2M ep:10K epch:31.92 loss:0.009 grdn:0.224 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:38:48 ts/train.py:232 step:232K smpl:2M ep:10K epch:31.95 loss:0.011 grdn:0.246 lr:8.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:40:14 ts/train.py:232 step:232K smpl:2M ep:10K epch:31.98 loss:0.010 grdn:0.244 lr:8.5e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:41:40 ts/train.py:232 step:232K smpl:2M ep:10K epch:32.01 loss:0.008 grdn:0.213 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 17:43:05 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.03 loss:0.009 grdn:0.224 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:44:31 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.06 loss:0.010 grdn:0.231 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:45:56 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.09 loss:0.009 grdn:0.205 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:47:22 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.12 loss:0.009 grdn:0.223 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:48:47 ts/train.py:232 step:233K smpl:2M ep:10K epch:32.14 loss:0.010 grdn:0.226 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:50:13 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.17 loss:0.009 grdn:0.226 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:51:38 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.20 loss:0.009 grdn:0.217 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:53:04 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.23 loss:0.009 grdn:0.216 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:54:30 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.25 loss:0.010 grdn:0.225 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:55:55 ts/train.py:232 step:234K smpl:2M ep:10K epch:32.28 loss:0.009 grdn:0.214 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:57:21 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.31 loss:0.009 grdn:0.223 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 17:58:46 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.34 loss:0.010 grdn:0.241 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:00:12 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.36 loss:0.010 grdn:0.240 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:01:37 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.39 loss:0.010 grdn:0.226 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:03:03 ts/train.py:232 step:235K smpl:2M ep:10K epch:32.42 loss:0.009 grdn:0.219 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:04:29 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.45 loss:0.009 grdn:0.223 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:05:54 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.47 loss:0.009 grdn:0.225 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:07:20 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.50 loss:0.009 grdn:0.216 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:08:45 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.53 loss:0.009 grdn:0.232 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:10:11 ts/train.py:232 step:236K smpl:2M ep:10K epch:32.56 loss:0.009 grdn:0.226 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:11:37 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.58 loss:0.010 grdn:0.240 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:13:02 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.61 loss:0.009 grdn:0.232 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:14:28 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.64 loss:0.009 grdn:0.224 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:15:53 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.67 loss:0.010 grdn:0.227 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:17:19 ts/train.py:232 step:237K smpl:2M ep:10K epch:32.69 loss:0.009 grdn:0.226 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:18:44 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.72 loss:0.010 grdn:0.236 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:20:10 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.75 loss:0.010 grdn:0.236 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:21:37 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.78 loss:0.010 grdn:0.231 lr:8.4e-05 updt_s:0.426 data_s:0.008 +INFO 2025-05-06 18:23:02 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.80 loss:0.009 grdn:0.223 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:24:28 ts/train.py:232 step:238K smpl:2M ep:10K epch:32.83 loss:0.010 grdn:0.231 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:25:54 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.86 loss:0.009 grdn:0.224 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:27:19 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.89 loss:0.010 grdn:0.240 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:28:45 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.91 loss:0.009 grdn:0.234 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:30:11 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.94 loss:0.009 grdn:0.218 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:31:36 ts/train.py:232 step:239K smpl:2M ep:10K epch:32.97 loss:0.009 grdn:0.221 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:33:02 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.00 loss:0.009 grdn:0.219 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:34:27 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.02 loss:0.009 grdn:0.228 lr:8.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:35:53 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.05 loss:0.010 grdn:0.231 lr:8.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 18:35:53 ts/train.py:241 Checkpoint policy after step 240000 +INFO 2025-05-06 18:37:22 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.08 loss:0.010 grdn:0.243 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:38:47 ts/train.py:232 step:240K smpl:2M ep:10K epch:33.11 loss:0.010 grdn:0.233 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:40:13 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.13 loss:0.008 grdn:0.216 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:41:38 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.16 loss:0.010 grdn:0.231 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:43:04 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.19 loss:0.009 grdn:0.215 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:44:29 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.22 loss:0.009 grdn:0.222 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:45:55 ts/train.py:232 step:241K smpl:2M ep:10K epch:33.24 loss:0.009 grdn:0.227 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:47:20 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.27 loss:0.009 grdn:0.233 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:48:46 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.30 loss:0.008 grdn:0.201 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:50:11 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.33 loss:0.009 grdn:0.237 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:51:37 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.36 loss:0.010 grdn:0.230 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:53:02 ts/train.py:232 step:242K smpl:2M ep:10K epch:33.38 loss:0.009 grdn:0.226 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:54:28 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.41 loss:0.010 grdn:0.229 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:55:54 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.44 loss:0.009 grdn:0.228 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:57:19 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.47 loss:0.009 grdn:0.229 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 18:58:45 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.49 loss:0.010 grdn:0.237 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:00:10 ts/train.py:232 step:243K smpl:2M ep:10K epch:33.52 loss:0.009 grdn:0.215 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:01:36 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.55 loss:0.009 grdn:0.235 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:03:01 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.58 loss:0.009 grdn:0.231 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:04:27 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.60 loss:0.009 grdn:0.223 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:05:52 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.63 loss:0.008 grdn:0.221 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:07:18 ts/train.py:232 step:244K smpl:2M ep:10K epch:33.66 loss:0.009 grdn:0.230 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:08:43 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.69 loss:0.009 grdn:0.232 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:10:09 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.71 loss:0.011 grdn:0.252 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:11:35 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.74 loss:0.009 grdn:0.230 lr:8.3e-05 updt_s:0.425 data_s:0.008 +INFO 2025-05-06 19:13:01 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.77 loss:0.010 grdn:0.230 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:14:26 ts/train.py:232 step:245K smpl:2M ep:10K epch:33.80 loss:0.009 grdn:0.215 lr:8.3e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-06 19:15:52 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.82 loss:0.009 grdn:0.234 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:17:17 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.85 loss:0.010 grdn:0.238 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:18:43 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.88 loss:0.008 grdn:0.207 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:20:08 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.91 loss:0.010 grdn:0.233 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:21:34 ts/train.py:232 step:246K smpl:2M ep:10K epch:33.93 loss:0.009 grdn:0.214 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:22:59 ts/train.py:232 step:247K smpl:2M ep:10K epch:33.96 loss:0.009 grdn:0.221 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:24:25 ts/train.py:232 step:247K smpl:2M ep:10K epch:33.99 loss:0.010 grdn:0.224 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:25:50 ts/train.py:232 step:247K smpl:2M ep:10K epch:34.02 loss:0.009 grdn:0.220 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:27:16 ts/train.py:232 step:247K smpl:2M ep:10K epch:34.04 loss:0.008 grdn:0.214 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:28:41 ts/train.py:232 step:247K smpl:2M ep:10K epch:34.07 loss:0.009 grdn:0.231 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:30:07 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.10 loss:0.010 grdn:0.237 lr:8.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:31:32 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.13 loss:0.011 grdn:0.249 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 19:32:58 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.15 loss:0.009 grdn:0.230 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:34:23 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.18 loss:0.009 grdn:0.227 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:35:49 ts/train.py:232 step:248K smpl:2M ep:10K epch:34.21 loss:0.008 grdn:0.210 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:37:14 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.24 loss:0.009 grdn:0.230 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:38:40 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.26 loss:0.009 grdn:0.216 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 19:40:05 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.29 loss:0.010 grdn:0.232 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:41:30 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.32 loss:0.009 grdn:0.232 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:42:56 ts/train.py:232 step:249K smpl:2M ep:10K epch:34.35 loss:0.009 grdn:0.225 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:44:21 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.37 loss:0.009 grdn:0.225 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:45:47 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.40 loss:0.009 grdn:0.245 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:47:12 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.43 loss:0.009 grdn:0.221 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:48:38 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.46 loss:0.009 grdn:0.235 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 19:50:03 ts/train.py:232 step:250K smpl:2M ep:10K epch:34.48 loss:0.009 grdn:0.212 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 19:51:29 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.51 loss:0.009 grdn:0.216 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 19:52:54 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.54 loss:0.009 grdn:0.223 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 19:54:19 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.57 loss:0.009 grdn:0.245 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 19:55:45 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.59 loss:0.009 grdn:0.223 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 19:57:10 ts/train.py:232 step:251K smpl:2M ep:10K epch:34.62 loss:0.009 grdn:0.230 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 19:58:36 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.65 loss:0.009 grdn:0.220 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:00:01 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.68 loss:0.009 grdn:0.231 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:01:28 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.70 loss:0.009 grdn:0.220 lr:8.2e-05 updt_s:0.425 data_s:0.007 +INFO 2025-05-06 20:02:53 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.73 loss:0.009 grdn:0.228 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:04:18 ts/train.py:232 step:252K smpl:2M ep:10K epch:34.76 loss:0.008 grdn:0.211 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:05:44 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.79 loss:0.010 grdn:0.235 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:07:09 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.81 loss:0.010 grdn:0.234 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:08:34 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.84 loss:0.009 grdn:0.229 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:10:00 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.87 loss:0.009 grdn:0.225 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:11:25 ts/train.py:232 step:253K smpl:2M ep:10K epch:34.90 loss:0.009 grdn:0.241 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:12:51 ts/train.py:232 step:254K smpl:2M ep:10K epch:34.93 loss:0.008 grdn:0.215 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:14:16 ts/train.py:232 step:254K smpl:2M ep:10K epch:34.95 loss:0.008 grdn:0.218 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:15:42 ts/train.py:232 step:254K smpl:2M ep:10K epch:34.98 loss:0.008 grdn:0.212 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:17:07 ts/train.py:232 step:254K smpl:2M ep:11K epch:35.01 loss:0.008 grdn:0.216 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:18:32 ts/train.py:232 step:254K smpl:2M ep:11K epch:35.04 loss:0.009 grdn:0.223 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:19:58 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.06 loss:0.009 grdn:0.221 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:21:23 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.09 loss:0.008 grdn:0.214 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:22:49 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.12 loss:0.009 grdn:0.217 lr:8.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:24:14 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.15 loss:0.009 grdn:0.220 lr:8.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:25:39 ts/train.py:232 step:255K smpl:2M ep:11K epch:35.17 loss:0.009 grdn:0.226 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:27:05 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.20 loss:0.009 grdn:0.218 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:28:30 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.23 loss:0.009 grdn:0.229 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:29:56 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.26 loss:0.008 grdn:0.208 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:31:21 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.28 loss:0.009 grdn:0.230 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:32:47 ts/train.py:232 step:256K smpl:2M ep:11K epch:35.31 loss:0.009 grdn:0.224 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:34:12 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.34 loss:0.009 grdn:0.233 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:35:37 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.37 loss:0.010 grdn:0.225 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:37:03 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.39 loss:0.009 grdn:0.216 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:38:28 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.42 loss:0.009 grdn:0.207 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:39:53 ts/train.py:232 step:257K smpl:2M ep:11K epch:35.45 loss:0.009 grdn:0.239 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:41:19 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.48 loss:0.009 grdn:0.218 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:42:44 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.50 loss:0.008 grdn:0.211 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:44:10 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.53 loss:0.009 grdn:0.229 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:45:35 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.56 loss:0.009 grdn:0.228 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:47:00 ts/train.py:232 step:258K smpl:2M ep:11K epch:35.59 loss:0.010 grdn:0.244 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:48:26 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.61 loss:0.009 grdn:0.225 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:49:51 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.64 loss:0.010 grdn:0.230 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:51:18 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.67 loss:0.008 grdn:0.208 lr:8.1e-05 updt_s:0.426 data_s:0.008 +INFO 2025-05-06 20:52:44 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.70 loss:0.008 grdn:0.218 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:54:09 ts/train.py:232 step:259K smpl:2M ep:11K epch:35.72 loss:0.009 grdn:0.231 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:55:34 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.75 loss:0.009 grdn:0.225 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 20:57:00 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.78 loss:0.009 grdn:0.233 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:58:25 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.81 loss:0.009 grdn:0.230 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 20:59:50 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.83 loss:0.010 grdn:0.232 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:01:16 ts/train.py:232 step:260K smpl:2M ep:11K epch:35.86 loss:0.009 grdn:0.229 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:02:41 ts/train.py:232 step:261K smpl:2M ep:11K epch:35.89 loss:0.008 grdn:0.214 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:04:06 ts/train.py:232 step:261K smpl:2M ep:11K epch:35.92 loss:0.009 grdn:0.222 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:05:32 ts/train.py:232 step:261K smpl:2M ep:11K epch:35.94 loss:0.009 grdn:0.231 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:06:57 ts/train.py:232 step:261K smpl:2M ep:11K epch:35.97 loss:0.009 grdn:0.214 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:08:22 ts/train.py:232 step:261K smpl:2M ep:11K epch:36.00 loss:0.010 grdn:0.244 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:09:48 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.03 loss:0.008 grdn:0.222 lr:8.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:11:13 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.05 loss:0.010 grdn:0.239 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:12:38 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.08 loss:0.009 grdn:0.237 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:14:04 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.11 loss:0.008 grdn:0.214 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:15:29 ts/train.py:232 step:262K smpl:2M ep:11K epch:36.14 loss:0.009 grdn:0.229 lr:8.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:16:54 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.16 loss:0.009 grdn:0.224 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:18:20 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.19 loss:0.009 grdn:0.225 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:19:45 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.22 loss:0.009 grdn:0.203 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:21:11 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.25 loss:0.010 grdn:0.239 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:22:36 ts/train.py:232 step:263K smpl:2M ep:11K epch:36.27 loss:0.008 grdn:0.216 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:24:02 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.30 loss:0.010 grdn:0.247 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:25:27 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.33 loss:0.008 grdn:0.221 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:26:53 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.36 loss:0.009 grdn:0.227 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:28:18 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.38 loss:0.008 grdn:0.208 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:29:43 ts/train.py:232 step:264K smpl:2M ep:11K epch:36.41 loss:0.009 grdn:0.216 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:31:09 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.44 loss:0.008 grdn:0.203 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:32:34 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.47 loss:0.009 grdn:0.226 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:34:00 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.50 loss:0.010 grdn:0.237 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:35:25 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.52 loss:0.010 grdn:0.232 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:36:50 ts/train.py:232 step:265K smpl:2M ep:11K epch:36.55 loss:0.009 grdn:0.225 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:38:16 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.58 loss:0.009 grdn:0.243 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:39:41 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.61 loss:0.008 grdn:0.220 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:41:08 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.63 loss:0.009 grdn:0.221 lr:8.0e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-06 21:42:33 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.66 loss:0.009 grdn:0.220 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:43:59 ts/train.py:232 step:266K smpl:2M ep:11K epch:36.69 loss:0.008 grdn:0.225 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:45:24 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.72 loss:0.008 grdn:0.225 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:46:50 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.74 loss:0.009 grdn:0.222 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:48:15 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.77 loss:0.009 grdn:0.219 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:49:40 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.80 loss:0.011 grdn:0.256 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:51:06 ts/train.py:232 step:267K smpl:2M ep:11K epch:36.83 loss:0.008 grdn:0.216 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:52:31 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.85 loss:0.009 grdn:0.232 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:53:56 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.88 loss:0.008 grdn:0.221 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:55:22 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.91 loss:0.009 grdn:0.224 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:56:47 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.94 loss:0.009 grdn:0.224 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 21:58:13 ts/train.py:232 step:268K smpl:2M ep:11K epch:36.96 loss:0.009 grdn:0.233 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 21:59:38 ts/train.py:232 step:269K smpl:2M ep:11K epch:36.99 loss:0.008 grdn:0.206 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:01:03 ts/train.py:232 step:269K smpl:2M ep:11K epch:37.02 loss:0.009 grdn:0.227 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:02:29 ts/train.py:232 step:269K smpl:2M ep:11K epch:37.05 loss:0.009 grdn:0.224 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:03:54 ts/train.py:232 step:269K smpl:2M ep:11K epch:37.07 loss:0.009 grdn:0.214 lr:8.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:05:19 ts/train.py:232 step:269K smpl:2M ep:11K epch:37.10 loss:0.008 grdn:0.214 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:06:45 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.13 loss:0.009 grdn:0.220 lr:8.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:08:10 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.16 loss:0.009 grdn:0.222 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:09:36 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.18 loss:0.010 grdn:0.248 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:09:36 ts/train.py:241 Checkpoint policy after step 270000 +INFO 2025-05-06 22:11:04 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.21 loss:0.010 grdn:0.241 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:12:29 ts/train.py:232 step:270K smpl:2M ep:11K epch:37.24 loss:0.010 grdn:0.235 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:13:55 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.27 loss:0.008 grdn:0.208 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:15:20 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.29 loss:0.010 grdn:0.237 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:16:46 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.32 loss:0.010 grdn:0.244 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:18:11 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.35 loss:0.008 grdn:0.215 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:19:36 ts/train.py:232 step:271K smpl:2M ep:11K epch:37.38 loss:0.008 grdn:0.217 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:21:02 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.40 loss:0.009 grdn:0.222 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:22:27 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.43 loss:0.007 grdn:0.211 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:23:52 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.46 loss:0.009 grdn:0.236 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:25:18 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.49 loss:0.009 grdn:0.228 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:26:43 ts/train.py:232 step:272K smpl:2M ep:11K epch:37.51 loss:0.008 grdn:0.212 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:28:08 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.54 loss:0.009 grdn:0.226 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:29:34 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.57 loss:0.009 grdn:0.220 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:31:00 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.60 loss:0.009 grdn:0.233 lr:7.9e-05 updt_s:0.425 data_s:0.007 +INFO 2025-05-06 22:32:26 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.62 loss:0.008 grdn:0.213 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:33:51 ts/train.py:232 step:273K smpl:2M ep:11K epch:37.65 loss:0.007 grdn:0.207 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:35:17 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.68 loss:0.008 grdn:0.230 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:36:42 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.71 loss:0.009 grdn:0.243 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:38:08 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.73 loss:0.008 grdn:0.213 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:39:33 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.76 loss:0.008 grdn:0.213 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:40:59 ts/train.py:232 step:274K smpl:2M ep:11K epch:37.79 loss:0.009 grdn:0.223 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:42:24 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.82 loss:0.010 grdn:0.234 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:43:50 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.84 loss:0.010 grdn:0.240 lr:7.9e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 22:45:16 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.87 loss:0.010 grdn:0.239 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:46:41 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.90 loss:0.008 grdn:0.217 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:48:06 ts/train.py:232 step:275K smpl:2M ep:11K epch:37.93 loss:0.009 grdn:0.220 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:49:32 ts/train.py:232 step:276K smpl:2M ep:11K epch:37.95 loss:0.009 grdn:0.222 lr:7.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 22:50:57 ts/train.py:232 step:276K smpl:2M ep:11K epch:37.98 loss:0.009 grdn:0.226 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:52:23 ts/train.py:232 step:276K smpl:2M ep:11K epch:38.01 loss:0.009 grdn:0.229 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:53:48 ts/train.py:232 step:276K smpl:2M ep:11K epch:38.04 loss:0.009 grdn:0.233 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:55:14 ts/train.py:232 step:276K smpl:2M ep:11K epch:38.07 loss:0.010 grdn:0.244 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:56:39 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.09 loss:0.009 grdn:0.228 lr:7.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:58:05 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.12 loss:0.009 grdn:0.236 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 22:59:30 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.15 loss:0.009 grdn:0.229 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:00:56 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.18 loss:0.009 grdn:0.224 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:02:21 ts/train.py:232 step:277K smpl:2M ep:11K epch:38.20 loss:0.008 grdn:0.214 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:03:47 ts/train.py:232 step:278K smpl:2M ep:11K epch:38.23 loss:0.009 grdn:0.217 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:05:12 ts/train.py:232 step:278K smpl:2M ep:11K epch:38.26 loss:0.010 grdn:0.247 lr:7.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-06 23:06:37 ts/train.py:232 step:278K smpl:2M ep:11K epch:38.29 loss:0.010 grdn:0.239 lr:7.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-06 23:08:03 ts/train.py:232 step:278K smpl:2M ep:11K epch:38.31 loss:0.008 grdn:0.225 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:09:28 ts/train.py:232 step:278K smpl:2M ep:12K epch:38.34 loss:0.009 grdn:0.235 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:10:54 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.37 loss:0.008 grdn:0.215 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:12:19 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.40 loss:0.008 grdn:0.211 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:13:45 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.42 loss:0.008 grdn:0.206 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:15:10 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.45 loss:0.009 grdn:0.227 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:16:36 ts/train.py:232 step:279K smpl:2M ep:12K epch:38.48 loss:0.009 grdn:0.235 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:18:01 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.51 loss:0.009 grdn:0.231 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:19:27 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.53 loss:0.008 grdn:0.207 lr:7.8e-05 updt_s:0.426 data_s:0.000 +INFO 2025-05-06 23:20:54 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.56 loss:0.009 grdn:0.225 lr:7.8e-05 updt_s:0.425 data_s:0.010 +INFO 2025-05-06 23:22:19 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.59 loss:0.009 grdn:0.229 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:23:45 ts/train.py:232 step:280K smpl:2M ep:12K epch:38.62 loss:0.008 grdn:0.217 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:25:10 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.64 loss:0.008 grdn:0.216 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:26:36 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.67 loss:0.009 grdn:0.225 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:28:01 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.70 loss:0.009 grdn:0.225 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:29:27 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.73 loss:0.009 grdn:0.228 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:30:52 ts/train.py:232 step:281K smpl:2M ep:12K epch:38.75 loss:0.008 grdn:0.222 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:32:18 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.78 loss:0.007 grdn:0.197 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:33:43 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.81 loss:0.008 grdn:0.213 lr:7.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:35:08 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.84 loss:0.009 grdn:0.224 lr:7.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:36:34 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.86 loss:0.010 grdn:0.236 lr:7.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:37:59 ts/train.py:232 step:282K smpl:2M ep:12K epch:38.89 loss:0.009 grdn:0.240 lr:7.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:39:24 ts/train.py:232 step:283K smpl:2M ep:12K epch:38.92 loss:0.009 grdn:0.224 lr:7.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:40:50 ts/train.py:232 step:283K smpl:2M ep:12K epch:38.95 loss:0.008 grdn:0.207 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:42:15 ts/train.py:232 step:283K smpl:2M ep:12K epch:38.97 loss:0.007 grdn:0.201 lr:7.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:43:41 ts/train.py:232 step:283K smpl:2M ep:12K epch:39.00 loss:0.009 grdn:0.217 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:45:06 ts/train.py:232 step:283K smpl:2M ep:12K epch:39.03 loss:0.008 grdn:0.207 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:46:31 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.06 loss:0.008 grdn:0.214 lr:7.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:47:57 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.08 loss:0.008 grdn:0.225 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 23:49:22 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.11 loss:0.009 grdn:0.222 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:50:48 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.14 loss:0.009 grdn:0.219 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:52:13 ts/train.py:232 step:284K smpl:2M ep:12K epch:39.17 loss:0.009 grdn:0.233 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:53:38 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.19 loss:0.009 grdn:0.223 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:55:04 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.22 loss:0.008 grdn:0.207 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:56:29 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.25 loss:0.007 grdn:0.199 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:57:54 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.28 loss:0.008 grdn:0.217 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 23:59:20 ts/train.py:232 step:285K smpl:2M ep:12K epch:39.30 loss:0.009 grdn:0.233 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:00:45 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.33 loss:0.009 grdn:0.233 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:02:10 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.36 loss:0.007 grdn:0.205 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:03:36 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.39 loss:0.008 grdn:0.229 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:05:01 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.41 loss:0.008 grdn:0.213 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:06:26 ts/train.py:232 step:286K smpl:2M ep:12K epch:39.44 loss:0.008 grdn:0.215 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:07:52 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.47 loss:0.008 grdn:0.225 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:09:17 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.50 loss:0.010 grdn:0.236 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:10:44 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.52 loss:0.010 grdn:0.232 lr:7.7e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-07 00:12:10 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.55 loss:0.008 grdn:0.215 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:13:35 ts/train.py:232 step:287K smpl:2M ep:12K epch:39.58 loss:0.007 grdn:0.192 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:15:01 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.61 loss:0.009 grdn:0.225 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:16:26 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.64 loss:0.009 grdn:0.211 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:17:52 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.66 loss:0.008 grdn:0.217 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:19:17 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.69 loss:0.009 grdn:0.223 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:20:42 ts/train.py:232 step:288K smpl:2M ep:12K epch:39.72 loss:0.007 grdn:0.208 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:22:08 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.75 loss:0.009 grdn:0.232 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:23:33 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.77 loss:0.008 grdn:0.223 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:24:59 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.80 loss:0.010 grdn:0.253 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:26:24 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.83 loss:0.009 grdn:0.223 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:27:49 ts/train.py:232 step:289K smpl:2M ep:12K epch:39.86 loss:0.008 grdn:0.210 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:29:15 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.88 loss:0.008 grdn:0.213 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:30:40 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.91 loss:0.009 grdn:0.234 lr:7.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:32:06 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.94 loss:0.008 grdn:0.214 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:33:31 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.97 loss:0.009 grdn:0.229 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:34:57 ts/train.py:232 step:290K smpl:2M ep:12K epch:39.99 loss:0.008 grdn:0.219 lr:7.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:36:22 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.02 loss:0.008 grdn:0.224 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:37:47 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.05 loss:0.009 grdn:0.225 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:39:13 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.08 loss:0.008 grdn:0.216 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:40:39 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.10 loss:0.008 grdn:0.221 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:42:04 ts/train.py:232 step:291K smpl:2M ep:12K epch:40.13 loss:0.008 grdn:0.217 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:43:29 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.16 loss:0.008 grdn:0.216 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:44:55 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.19 loss:0.008 grdn:0.218 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:46:20 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.21 loss:0.009 grdn:0.236 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:47:46 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.24 loss:0.008 grdn:0.214 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:49:11 ts/train.py:232 step:292K smpl:2M ep:12K epch:40.27 loss:0.009 grdn:0.238 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:50:37 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.30 loss:0.008 grdn:0.215 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:52:02 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.32 loss:0.010 grdn:0.232 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:53:27 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.35 loss:0.008 grdn:0.198 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:54:53 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.38 loss:0.008 grdn:0.208 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 00:56:18 ts/train.py:232 step:293K smpl:2M ep:12K epch:40.41 loss:0.009 grdn:0.235 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:57:44 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.43 loss:0.008 grdn:0.220 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 00:59:09 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.46 loss:0.007 grdn:0.209 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:00:36 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.49 loss:0.009 grdn:0.235 lr:7.6e-05 updt_s:0.425 data_s:0.006 +INFO 2025-05-07 01:02:01 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.52 loss:0.008 grdn:0.214 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:03:26 ts/train.py:232 step:294K smpl:2M ep:12K epch:40.54 loss:0.008 grdn:0.220 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:04:52 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.57 loss:0.008 grdn:0.211 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:06:17 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.60 loss:0.007 grdn:0.209 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:07:42 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.63 loss:0.009 grdn:0.218 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:09:08 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.65 loss:0.007 grdn:0.197 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:10:33 ts/train.py:232 step:295K smpl:2M ep:12K epch:40.68 loss:0.009 grdn:0.230 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:11:58 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.71 loss:0.009 grdn:0.223 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:13:24 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.74 loss:0.007 grdn:0.203 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:14:49 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.76 loss:0.009 grdn:0.229 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:16:14 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.79 loss:0.009 grdn:0.222 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:17:40 ts/train.py:232 step:296K smpl:2M ep:12K epch:40.82 loss:0.009 grdn:0.239 lr:7.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:19:05 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.85 loss:0.009 grdn:0.225 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:20:30 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.87 loss:0.009 grdn:0.224 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:21:56 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.90 loss:0.008 grdn:0.224 lr:7.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:23:21 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.93 loss:0.009 grdn:0.223 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:24:47 ts/train.py:232 step:297K smpl:2M ep:12K epch:40.96 loss:0.009 grdn:0.221 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:26:12 ts/train.py:232 step:298K smpl:2M ep:12K epch:40.98 loss:0.008 grdn:0.222 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:27:37 ts/train.py:232 step:298K smpl:2M ep:12K epch:41.01 loss:0.009 grdn:0.230 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:29:03 ts/train.py:232 step:298K smpl:2M ep:12K epch:41.04 loss:0.007 grdn:0.214 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:30:28 ts/train.py:232 step:298K smpl:2M ep:12K epch:41.07 loss:0.009 grdn:0.244 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:31:54 ts/train.py:232 step:298K smpl:2M ep:12K epch:41.09 loss:0.009 grdn:0.225 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:33:19 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.12 loss:0.007 grdn:0.208 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:34:45 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.15 loss:0.007 grdn:0.212 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:36:10 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.18 loss:0.008 grdn:0.219 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:37:35 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.21 loss:0.009 grdn:0.239 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:39:01 ts/train.py:232 step:299K smpl:2M ep:12K epch:41.23 loss:0.007 grdn:0.207 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:40:26 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.26 loss:0.008 grdn:0.226 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:41:52 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.29 loss:0.008 grdn:0.213 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:43:17 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.32 loss:0.008 grdn:0.217 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:43:17 ts/train.py:241 Checkpoint policy after step 300000 +INFO 2025-05-07 01:44:46 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.34 loss:0.008 grdn:0.215 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:46:11 ts/train.py:232 step:300K smpl:2M ep:12K epch:41.37 loss:0.008 grdn:0.224 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:47:37 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.40 loss:0.009 grdn:0.217 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:49:02 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.43 loss:0.008 grdn:0.213 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 01:50:29 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.45 loss:0.009 grdn:0.223 lr:7.5e-05 updt_s:0.425 data_s:0.007 +INFO 2025-05-07 01:51:54 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.48 loss:0.008 grdn:0.210 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:53:19 ts/train.py:232 step:301K smpl:2M ep:12K epch:41.51 loss:0.008 grdn:0.216 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:54:44 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.54 loss:0.008 grdn:0.225 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:56:10 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.56 loss:0.009 grdn:0.244 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:57:35 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.59 loss:0.008 grdn:0.213 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 01:59:00 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.62 loss:0.008 grdn:0.214 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:00:26 ts/train.py:232 step:302K smpl:2M ep:12K epch:41.65 loss:0.007 grdn:0.209 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:01:51 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.67 loss:0.008 grdn:0.220 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:03:17 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.70 loss:0.009 grdn:0.225 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:04:42 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.73 loss:0.008 grdn:0.218 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:06:07 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.76 loss:0.008 grdn:0.208 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 02:07:33 ts/train.py:232 step:303K smpl:2M ep:13K epch:41.78 loss:0.008 grdn:0.228 lr:7.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 02:08:58 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.81 loss:0.009 grdn:0.235 lr:7.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:10:23 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.84 loss:0.010 grdn:0.242 lr:7.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 02:11:49 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.87 loss:0.009 grdn:0.230 lr:7.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 02:13:14 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.89 loss:0.008 grdn:0.228 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:14:40 ts/train.py:232 step:304K smpl:2M ep:13K epch:41.92 loss:0.008 grdn:0.225 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:16:05 ts/train.py:232 step:305K smpl:2M ep:13K epch:41.95 loss:0.007 grdn:0.203 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:17:30 ts/train.py:232 step:305K smpl:2M ep:13K epch:41.98 loss:0.007 grdn:0.207 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:18:55 ts/train.py:232 step:305K smpl:2M ep:13K epch:42.00 loss:0.008 grdn:0.219 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:20:21 ts/train.py:232 step:305K smpl:2M ep:13K epch:42.03 loss:0.008 grdn:0.211 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:21:46 ts/train.py:232 step:305K smpl:2M ep:13K epch:42.06 loss:0.009 grdn:0.235 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:23:11 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.09 loss:0.009 grdn:0.228 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:24:37 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.11 loss:0.008 grdn:0.226 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:26:02 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.14 loss:0.009 grdn:0.232 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:27:27 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.17 loss:0.008 grdn:0.214 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:28:52 ts/train.py:232 step:306K smpl:2M ep:13K epch:42.20 loss:0.008 grdn:0.217 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:30:18 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.22 loss:0.008 grdn:0.219 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:31:43 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.25 loss:0.008 grdn:0.220 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:33:08 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.28 loss:0.008 grdn:0.205 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:34:34 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.31 loss:0.007 grdn:0.216 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:35:59 ts/train.py:232 step:307K smpl:2M ep:13K epch:42.33 loss:0.009 grdn:0.233 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:37:25 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.36 loss:0.009 grdn:0.219 lr:7.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 02:38:50 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.39 loss:0.009 grdn:0.222 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:40:17 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.42 loss:0.008 grdn:0.225 lr:7.4e-05 updt_s:0.425 data_s:0.010 +INFO 2025-05-07 02:41:42 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.44 loss:0.008 grdn:0.211 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:43:08 ts/train.py:232 step:308K smpl:2M ep:13K epch:42.47 loss:0.008 grdn:0.229 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:44:33 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.50 loss:0.008 grdn:0.208 lr:7.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 02:45:59 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.53 loss:0.008 grdn:0.227 lr:7.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 02:47:24 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.55 loss:0.009 grdn:0.235 lr:7.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 02:48:50 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.58 loss:0.009 grdn:0.242 lr:7.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 02:50:15 ts/train.py:232 step:309K smpl:2M ep:13K epch:42.61 loss:0.007 grdn:0.211 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:51:40 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.64 loss:0.008 grdn:0.221 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:53:06 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.66 loss:0.008 grdn:0.212 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:54:31 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.69 loss:0.008 grdn:0.208 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:55:56 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.72 loss:0.007 grdn:0.211 lr:7.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:57:22 ts/train.py:232 step:310K smpl:2M ep:13K epch:42.75 loss:0.008 grdn:0.234 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 02:58:47 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.78 loss:0.008 grdn:0.210 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:00:12 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.80 loss:0.009 grdn:0.227 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:01:38 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.83 loss:0.008 grdn:0.228 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:03:03 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.86 loss:0.008 grdn:0.221 lr:7.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:04:29 ts/train.py:232 step:311K smpl:2M ep:13K epch:42.89 loss:0.007 grdn:0.202 lr:7.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:05:54 ts/train.py:232 step:312K smpl:2M ep:13K epch:42.91 loss:0.009 grdn:0.237 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:07:19 ts/train.py:232 step:312K smpl:2M ep:13K epch:42.94 loss:0.008 grdn:0.216 lr:7.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:08:45 ts/train.py:232 step:312K smpl:2M ep:13K epch:42.97 loss:0.007 grdn:0.205 lr:7.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:10:10 ts/train.py:232 step:312K smpl:2M ep:13K epch:43.00 loss:0.008 grdn:0.214 lr:7.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:11:36 ts/train.py:232 step:312K smpl:2M ep:13K epch:43.02 loss:0.008 grdn:0.213 lr:7.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:13:01 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.05 loss:0.008 grdn:0.218 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:14:27 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.08 loss:0.009 grdn:0.240 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:15:52 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.11 loss:0.007 grdn:0.207 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:17:17 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.13 loss:0.008 grdn:0.227 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:18:42 ts/train.py:232 step:313K smpl:3M ep:13K epch:43.16 loss:0.008 grdn:0.231 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:20:08 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.19 loss:0.008 grdn:0.228 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:21:33 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.22 loss:0.008 grdn:0.227 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:22:58 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.24 loss:0.007 grdn:0.215 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:24:24 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.27 loss:0.008 grdn:0.233 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:25:49 ts/train.py:232 step:314K smpl:3M ep:13K epch:43.30 loss:0.008 grdn:0.224 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:27:14 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.33 loss:0.008 grdn:0.214 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:28:39 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.35 loss:0.009 grdn:0.230 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:30:06 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.38 loss:0.008 grdn:0.227 lr:7.3e-05 updt_s:0.424 data_s:0.008 +INFO 2025-05-07 03:31:31 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.41 loss:0.008 grdn:0.226 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:32:56 ts/train.py:232 step:315K smpl:3M ep:13K epch:43.44 loss:0.008 grdn:0.213 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:34:22 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.46 loss:0.009 grdn:0.247 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:35:47 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.49 loss:0.008 grdn:0.215 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:37:12 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.52 loss:0.008 grdn:0.213 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:38:38 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.55 loss:0.007 grdn:0.203 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:40:03 ts/train.py:232 step:316K smpl:3M ep:13K epch:43.57 loss:0.008 grdn:0.220 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:41:28 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.60 loss:0.008 grdn:0.220 lr:7.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:42:54 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.63 loss:0.008 grdn:0.217 lr:7.2e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-07 03:44:19 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.66 loss:0.008 grdn:0.218 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:45:44 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.68 loss:0.007 grdn:0.214 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:47:10 ts/train.py:232 step:317K smpl:3M ep:13K epch:43.71 loss:0.008 grdn:0.219 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:48:35 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.74 loss:0.009 grdn:0.233 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:50:00 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.77 loss:0.009 grdn:0.226 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:51:26 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.79 loss:0.007 grdn:0.196 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:52:51 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.82 loss:0.007 grdn:0.207 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:54:17 ts/train.py:232 step:318K smpl:3M ep:13K epch:43.85 loss:0.007 grdn:0.216 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:55:42 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.88 loss:0.007 grdn:0.212 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 03:57:07 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.90 loss:0.008 grdn:0.215 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:58:33 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.93 loss:0.008 grdn:0.229 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 03:59:58 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.96 loss:0.008 grdn:0.222 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:01:24 ts/train.py:232 step:319K smpl:3M ep:13K epch:43.99 loss:0.009 grdn:0.237 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:02:49 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.01 loss:0.008 grdn:0.207 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:04:14 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.04 loss:0.009 grdn:0.231 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:05:39 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.07 loss:0.008 grdn:0.213 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:07:05 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.10 loss:0.007 grdn:0.208 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:08:30 ts/train.py:232 step:320K smpl:3M ep:13K epch:44.12 loss:0.008 grdn:0.213 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:09:55 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.15 loss:0.008 grdn:0.220 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:11:21 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.18 loss:0.008 grdn:0.217 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:12:46 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.21 loss:0.008 grdn:0.218 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:14:11 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.23 loss:0.008 grdn:0.224 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 04:15:37 ts/train.py:232 step:321K smpl:3M ep:13K epch:44.26 loss:0.008 grdn:0.235 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 04:17:02 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.29 loss:0.007 grdn:0.203 lr:7.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 04:18:28 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.32 loss:0.008 grdn:0.214 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:19:54 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.34 loss:0.009 grdn:0.235 lr:7.2e-05 updt_s:0.425 data_s:0.008 +INFO 2025-05-07 04:21:20 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.37 loss:0.007 grdn:0.214 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:22:45 ts/train.py:232 step:322K smpl:3M ep:13K epch:44.40 loss:0.008 grdn:0.223 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:24:10 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.43 loss:0.008 grdn:0.224 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:25:36 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.46 loss:0.008 grdn:0.216 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:27:01 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.48 loss:0.008 grdn:0.227 lr:7.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:28:26 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.51 loss:0.008 grdn:0.215 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:29:51 ts/train.py:232 step:323K smpl:3M ep:13K epch:44.54 loss:0.008 grdn:0.220 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:31:17 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.57 loss:0.009 grdn:0.234 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 04:32:42 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.59 loss:0.008 grdn:0.221 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 04:34:08 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.62 loss:0.007 grdn:0.203 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:35:33 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.65 loss:0.009 grdn:0.239 lr:7.1e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-07 04:36:58 ts/train.py:232 step:324K smpl:3M ep:13K epch:44.68 loss:0.008 grdn:0.228 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:38:24 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.70 loss:0.008 grdn:0.219 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:39:49 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.73 loss:0.008 grdn:0.222 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:41:14 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.76 loss:0.007 grdn:0.199 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:42:39 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.79 loss:0.008 grdn:0.228 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:44:05 ts/train.py:232 step:325K smpl:3M ep:13K epch:44.81 loss:0.007 grdn:0.221 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:45:30 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.84 loss:0.008 grdn:0.237 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:46:55 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.87 loss:0.007 grdn:0.202 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:48:20 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.90 loss:0.008 grdn:0.227 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:49:46 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.92 loss:0.008 grdn:0.226 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:51:11 ts/train.py:232 step:326K smpl:3M ep:13K epch:44.95 loss:0.008 grdn:0.215 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:52:37 ts/train.py:232 step:327K smpl:3M ep:13K epch:44.98 loss:0.010 grdn:0.257 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 04:54:02 ts/train.py:232 step:327K smpl:3M ep:14K epch:45.01 loss:0.008 grdn:0.225 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 04:55:27 ts/train.py:232 step:327K smpl:3M ep:14K epch:45.03 loss:0.008 grdn:0.217 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 04:56:53 ts/train.py:232 step:327K smpl:3M ep:14K epch:45.06 loss:0.007 grdn:0.209 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:58:18 ts/train.py:232 step:327K smpl:3M ep:14K epch:45.09 loss:0.006 grdn:0.200 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 04:59:43 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.12 loss:0.007 grdn:0.205 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:01:09 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.14 loss:0.007 grdn:0.210 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:02:34 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.17 loss:0.007 grdn:0.210 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:03:59 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.20 loss:0.008 grdn:0.214 lr:7.1e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:05:25 ts/train.py:232 step:328K smpl:3M ep:14K epch:45.23 loss:0.007 grdn:0.217 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:06:50 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.25 loss:0.009 grdn:0.231 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:08:16 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.28 loss:0.008 grdn:0.210 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:09:43 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.31 loss:0.008 grdn:0.203 lr:7.1e-05 updt_s:0.425 data_s:0.008 +INFO 2025-05-07 05:11:08 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.34 loss:0.009 grdn:0.231 lr:7.1e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:12:34 ts/train.py:232 step:329K smpl:3M ep:14K epch:45.36 loss:0.008 grdn:0.210 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:13:59 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.39 loss:0.008 grdn:0.220 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:15:25 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.42 loss:0.008 grdn:0.225 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:16:50 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.45 loss:0.008 grdn:0.209 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:16:50 ts/train.py:241 Checkpoint policy after step 330000 +INFO 2025-05-07 05:18:19 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.47 loss:0.008 grdn:0.218 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:19:44 ts/train.py:232 step:330K smpl:3M ep:14K epch:45.50 loss:0.008 grdn:0.224 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:21:09 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.53 loss:0.008 grdn:0.223 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:22:35 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.56 loss:0.008 grdn:0.217 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:24:00 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.58 loss:0.008 grdn:0.218 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:25:26 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.61 loss:0.008 grdn:0.217 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:26:51 ts/train.py:232 step:331K smpl:3M ep:14K epch:45.64 loss:0.008 grdn:0.224 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:28:17 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.67 loss:0.007 grdn:0.206 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:29:42 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.69 loss:0.008 grdn:0.234 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:31:08 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.72 loss:0.008 grdn:0.222 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:32:33 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.75 loss:0.008 grdn:0.215 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:33:58 ts/train.py:232 step:332K smpl:3M ep:14K epch:45.78 loss:0.007 grdn:0.209 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:35:24 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.80 loss:0.007 grdn:0.214 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:36:49 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.83 loss:0.008 grdn:0.227 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:38:15 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.86 loss:0.008 grdn:0.219 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:39:40 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.89 loss:0.009 grdn:0.225 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:41:05 ts/train.py:232 step:333K smpl:3M ep:14K epch:45.91 loss:0.008 grdn:0.222 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:42:31 ts/train.py:232 step:334K smpl:3M ep:14K epch:45.94 loss:0.008 grdn:0.210 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:43:56 ts/train.py:232 step:334K smpl:3M ep:14K epch:45.97 loss:0.008 grdn:0.217 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:45:22 ts/train.py:232 step:334K smpl:3M ep:14K epch:46.00 loss:0.008 grdn:0.229 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:46:47 ts/train.py:232 step:334K smpl:3M ep:14K epch:46.03 loss:0.008 grdn:0.213 lr:7.0e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:48:12 ts/train.py:232 step:334K smpl:3M ep:14K epch:46.05 loss:0.007 grdn:0.195 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:49:38 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.08 loss:0.008 grdn:0.224 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:51:03 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.11 loss:0.008 grdn:0.210 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:52:29 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.14 loss:0.008 grdn:0.213 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:53:54 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.16 loss:0.007 grdn:0.212 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:55:20 ts/train.py:232 step:335K smpl:3M ep:14K epch:46.19 loss:0.008 grdn:0.220 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:56:45 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.22 loss:0.009 grdn:0.240 lr:7.0e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 05:58:11 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.25 loss:0.009 grdn:0.235 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 05:59:38 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.27 loss:0.008 grdn:0.216 lr:6.9e-05 updt_s:0.425 data_s:0.011 +INFO 2025-05-07 06:01:03 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.30 loss:0.008 grdn:0.213 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:02:29 ts/train.py:232 step:336K smpl:3M ep:14K epch:46.33 loss:0.008 grdn:0.219 lr:6.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:03:54 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.36 loss:0.008 grdn:0.229 lr:6.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:05:20 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.38 loss:0.008 grdn:0.207 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:06:45 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.41 loss:0.010 grdn:0.253 lr:6.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:08:10 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.44 loss:0.007 grdn:0.202 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:09:36 ts/train.py:232 step:337K smpl:3M ep:14K epch:46.47 loss:0.007 grdn:0.216 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:11:01 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.49 loss:0.009 grdn:0.231 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:12:27 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.52 loss:0.009 grdn:0.235 lr:6.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:13:52 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.55 loss:0.007 grdn:0.219 lr:6.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:15:17 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.58 loss:0.007 grdn:0.218 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:16:43 ts/train.py:232 step:338K smpl:3M ep:14K epch:46.60 loss:0.008 grdn:0.211 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:18:08 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.63 loss:0.008 grdn:0.221 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:19:33 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.66 loss:0.008 grdn:0.230 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:20:59 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.69 loss:0.007 grdn:0.219 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:22:24 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.71 loss:0.006 grdn:0.195 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:23:49 ts/train.py:232 step:339K smpl:3M ep:14K epch:46.74 loss:0.008 grdn:0.218 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:25:15 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.77 loss:0.009 grdn:0.235 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:26:40 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.80 loss:0.008 grdn:0.216 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:28:05 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.82 loss:0.008 grdn:0.221 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:29:31 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.85 loss:0.007 grdn:0.226 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:30:56 ts/train.py:232 step:340K smpl:3M ep:14K epch:46.88 loss:0.007 grdn:0.212 lr:6.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:32:21 ts/train.py:232 step:341K smpl:3M ep:14K epch:46.91 loss:0.008 grdn:0.226 lr:6.9e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:33:47 ts/train.py:232 step:341K smpl:3M ep:14K epch:46.93 loss:0.007 grdn:0.202 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:35:12 ts/train.py:232 step:341K smpl:3M ep:14K epch:46.96 loss:0.007 grdn:0.222 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:36:37 ts/train.py:232 step:341K smpl:3M ep:14K epch:46.99 loss:0.008 grdn:0.227 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:38:03 ts/train.py:232 step:341K smpl:3M ep:14K epch:47.02 loss:0.007 grdn:0.196 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:39:28 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.04 loss:0.008 grdn:0.217 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:40:53 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.07 loss:0.007 grdn:0.213 lr:6.9e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:42:19 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.10 loss:0.008 grdn:0.220 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:43:44 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.13 loss:0.008 grdn:0.225 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:45:09 ts/train.py:232 step:342K smpl:3M ep:14K epch:47.15 loss:0.007 grdn:0.197 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:46:35 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.18 loss:0.007 grdn:0.213 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:48:00 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.21 loss:0.008 grdn:0.220 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 06:49:27 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.24 loss:0.007 grdn:0.206 lr:6.8e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-07 06:50:52 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.26 loss:0.008 grdn:0.232 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:52:18 ts/train.py:232 step:343K smpl:3M ep:14K epch:47.29 loss:0.008 grdn:0.235 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:53:43 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.32 loss:0.008 grdn:0.236 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:55:09 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.35 loss:0.007 grdn:0.215 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:56:34 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.37 loss:0.008 grdn:0.222 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:58:00 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.40 loss:0.008 grdn:0.223 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 06:59:25 ts/train.py:232 step:344K smpl:3M ep:14K epch:47.43 loss:0.007 grdn:0.214 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:00:51 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.46 loss:0.007 grdn:0.206 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:02:16 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.48 loss:0.007 grdn:0.208 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:03:41 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.51 loss:0.008 grdn:0.226 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:05:07 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.54 loss:0.007 grdn:0.209 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:06:32 ts/train.py:232 step:345K smpl:3M ep:14K epch:47.57 loss:0.008 grdn:0.223 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:07:58 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.60 loss:0.008 grdn:0.214 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:09:23 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.62 loss:0.009 grdn:0.241 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:10:49 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.65 loss:0.007 grdn:0.216 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:12:14 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.68 loss:0.008 grdn:0.227 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:13:39 ts/train.py:232 step:346K smpl:3M ep:14K epch:47.71 loss:0.007 grdn:0.217 lr:6.8e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:15:05 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.73 loss:0.007 grdn:0.209 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:16:30 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.76 loss:0.007 grdn:0.214 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:17:56 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.79 loss:0.008 grdn:0.225 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:19:21 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.82 loss:0.007 grdn:0.218 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:20:46 ts/train.py:232 step:347K smpl:3M ep:14K epch:47.84 loss:0.007 grdn:0.211 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:22:12 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.87 loss:0.008 grdn:0.221 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:23:37 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.90 loss:0.008 grdn:0.233 lr:6.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:25:03 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.93 loss:0.007 grdn:0.212 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:26:29 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.95 loss:0.008 grdn:0.230 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:27:54 ts/train.py:232 step:348K smpl:3M ep:14K epch:47.98 loss:0.008 grdn:0.222 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:29:19 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.01 loss:0.008 grdn:0.220 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:30:45 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.04 loss:0.008 grdn:0.231 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:32:10 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.06 loss:0.009 grdn:0.237 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:33:36 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.09 loss:0.008 grdn:0.227 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:35:01 ts/train.py:232 step:349K smpl:3M ep:14K epch:48.12 loss:0.008 grdn:0.215 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:36:27 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.15 loss:0.009 grdn:0.246 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:37:52 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.17 loss:0.007 grdn:0.221 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:39:19 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.20 loss:0.007 grdn:0.205 lr:6.7e-05 updt_s:0.425 data_s:0.008 +INFO 2025-05-07 07:40:45 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.23 loss:0.007 grdn:0.201 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:42:10 ts/train.py:232 step:350K smpl:3M ep:14K epch:48.26 loss:0.008 grdn:0.217 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:43:35 ts/train.py:232 step:351K smpl:3M ep:14K epch:48.28 loss:0.008 grdn:0.219 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:45:01 ts/train.py:232 step:351K smpl:3M ep:14K epch:48.31 loss:0.008 grdn:0.224 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:46:26 ts/train.py:232 step:351K smpl:3M ep:15K epch:48.34 loss:0.007 grdn:0.214 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:47:52 ts/train.py:232 step:351K smpl:3M ep:15K epch:48.37 loss:0.008 grdn:0.219 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:49:17 ts/train.py:232 step:351K smpl:3M ep:15K epch:48.39 loss:0.008 grdn:0.229 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:50:43 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.42 loss:0.008 grdn:0.214 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:52:08 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.45 loss:0.008 grdn:0.219 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:53:33 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.48 loss:0.007 grdn:0.205 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 07:54:59 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.50 loss:0.008 grdn:0.217 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:56:24 ts/train.py:232 step:352K smpl:3M ep:15K epch:48.53 loss:0.007 grdn:0.200 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:57:50 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.56 loss:0.008 grdn:0.219 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 07:59:15 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.59 loss:0.008 grdn:0.216 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:00:41 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.61 loss:0.008 grdn:0.219 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:02:06 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.64 loss:0.008 grdn:0.224 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 08:03:32 ts/train.py:232 step:353K smpl:3M ep:15K epch:48.67 loss:0.007 grdn:0.199 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:04:57 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.70 loss:0.008 grdn:0.217 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:06:22 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.72 loss:0.007 grdn:0.210 lr:6.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:07:48 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.75 loss:0.007 grdn:0.212 lr:6.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 08:09:13 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.78 loss:0.007 grdn:0.208 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:10:39 ts/train.py:232 step:354K smpl:3M ep:15K epch:48.81 loss:0.008 grdn:0.226 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:12:04 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.83 loss:0.007 grdn:0.195 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:13:30 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.86 loss:0.008 grdn:0.216 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:14:55 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.89 loss:0.007 grdn:0.215 lr:6.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 08:16:21 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.92 loss:0.009 grdn:0.221 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:17:46 ts/train.py:232 step:355K smpl:3M ep:15K epch:48.94 loss:0.007 grdn:0.214 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:19:12 ts/train.py:232 step:356K smpl:3M ep:15K epch:48.97 loss:0.009 grdn:0.239 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:20:37 ts/train.py:232 step:356K smpl:3M ep:15K epch:49.00 loss:0.008 grdn:0.224 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:22:03 ts/train.py:232 step:356K smpl:3M ep:15K epch:49.03 loss:0.007 grdn:0.212 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:23:28 ts/train.py:232 step:356K smpl:3M ep:15K epch:49.05 loss:0.008 grdn:0.225 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:24:54 ts/train.py:232 step:356K smpl:3M ep:15K epch:49.08 loss:0.006 grdn:0.205 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:26:19 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.11 loss:0.007 grdn:0.220 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:27:45 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.14 loss:0.007 grdn:0.206 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:29:12 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.17 loss:0.008 grdn:0.234 lr:6.6e-05 updt_s:0.425 data_s:0.010 +INFO 2025-05-07 08:30:37 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.19 loss:0.007 grdn:0.216 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:32:03 ts/train.py:232 step:357K smpl:3M ep:15K epch:49.22 loss:0.008 grdn:0.227 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:33:28 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.25 loss:0.007 grdn:0.214 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:34:54 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.28 loss:0.007 grdn:0.208 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:36:19 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.30 loss:0.008 grdn:0.215 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:37:45 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.33 loss:0.008 grdn:0.227 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:39:10 ts/train.py:232 step:358K smpl:3M ep:15K epch:49.36 loss:0.007 grdn:0.206 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:40:36 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.39 loss:0.008 grdn:0.213 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:42:01 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.41 loss:0.008 grdn:0.235 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:43:27 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.44 loss:0.007 grdn:0.210 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:44:52 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.47 loss:0.007 grdn:0.213 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:46:18 ts/train.py:232 step:359K smpl:3M ep:15K epch:49.50 loss:0.007 grdn:0.207 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:47:43 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.52 loss:0.008 grdn:0.225 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:49:08 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.55 loss:0.007 grdn:0.208 lr:6.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:50:34 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.58 loss:0.007 grdn:0.215 lr:6.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 08:50:34 ts/train.py:241 Checkpoint policy after step 360000 +INFO 2025-05-07 08:52:02 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.61 loss:0.007 grdn:0.216 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 08:53:28 ts/train.py:232 step:360K smpl:3M ep:15K epch:49.63 loss:0.007 grdn:0.211 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:54:53 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.66 loss:0.008 grdn:0.219 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:56:19 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.69 loss:0.007 grdn:0.229 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:57:44 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.72 loss:0.007 grdn:0.214 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 08:59:10 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.74 loss:0.007 grdn:0.223 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 09:00:35 ts/train.py:232 step:361K smpl:3M ep:15K epch:49.77 loss:0.008 grdn:0.222 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 09:02:01 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.80 loss:0.009 grdn:0.232 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 09:03:26 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.83 loss:0.007 grdn:0.214 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 09:04:52 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.85 loss:0.007 grdn:0.217 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 09:06:17 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.88 loss:0.008 grdn:0.226 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 09:07:42 ts/train.py:232 step:362K smpl:3M ep:15K epch:49.91 loss:0.007 grdn:0.219 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 09:09:08 ts/train.py:232 step:363K smpl:3M ep:15K epch:49.94 loss:0.007 grdn:0.208 lr:6.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 09:10:33 ts/train.py:232 step:363K smpl:3M ep:15K epch:49.96 loss:0.007 grdn:0.217 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:11:58 ts/train.py:232 step:363K smpl:3M ep:15K epch:49.99 loss:0.008 grdn:0.226 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:13:24 ts/train.py:232 step:363K smpl:3M ep:15K epch:50.02 loss:0.007 grdn:0.209 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:14:49 ts/train.py:232 step:363K smpl:3M ep:15K epch:50.05 loss:0.006 grdn:0.196 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:16:14 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.07 loss:0.008 grdn:0.225 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:17:40 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.10 loss:0.007 grdn:0.206 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:19:07 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.13 loss:0.008 grdn:0.220 lr:6.5e-05 updt_s:0.425 data_s:0.010 +INFO 2025-05-07 09:20:32 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.16 loss:0.008 grdn:0.231 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:21:57 ts/train.py:232 step:364K smpl:3M ep:15K epch:50.18 loss:0.008 grdn:0.227 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:23:23 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.21 loss:0.007 grdn:0.214 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:24:48 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.24 loss:0.008 grdn:0.228 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:26:13 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.27 loss:0.008 grdn:0.223 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:27:38 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.29 loss:0.007 grdn:0.210 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:29:03 ts/train.py:232 step:365K smpl:3M ep:15K epch:50.32 loss:0.007 grdn:0.216 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:30:28 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.35 loss:0.009 grdn:0.235 lr:6.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 09:31:54 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.38 loss:0.007 grdn:0.210 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:33:19 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.40 loss:0.006 grdn:0.204 lr:6.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:34:44 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.43 loss:0.006 grdn:0.203 lr:6.4e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-07 09:36:09 ts/train.py:232 step:366K smpl:3M ep:15K epch:50.46 loss:0.007 grdn:0.211 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:37:35 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.49 loss:0.007 grdn:0.198 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:39:00 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.51 loss:0.007 grdn:0.218 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:40:25 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.54 loss:0.007 grdn:0.218 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:41:50 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.57 loss:0.007 grdn:0.219 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:43:16 ts/train.py:232 step:367K smpl:3M ep:15K epch:50.60 loss:0.007 grdn:0.218 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:44:41 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.62 loss:0.007 grdn:0.204 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:46:06 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.65 loss:0.007 grdn:0.208 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:47:31 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.68 loss:0.008 grdn:0.223 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:48:57 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.71 loss:0.007 grdn:0.215 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:50:22 ts/train.py:232 step:368K smpl:3M ep:15K epch:50.74 loss:0.008 grdn:0.222 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:51:48 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.76 loss:0.008 grdn:0.218 lr:6.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 09:53:13 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.79 loss:0.007 grdn:0.202 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:54:38 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.82 loss:0.008 grdn:0.233 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:56:04 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.85 loss:0.006 grdn:0.200 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:57:29 ts/train.py:232 step:369K smpl:3M ep:15K epch:50.87 loss:0.007 grdn:0.208 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 09:58:54 ts/train.py:232 step:370K smpl:3M ep:15K epch:50.90 loss:0.007 grdn:0.206 lr:6.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:00:20 ts/train.py:232 step:370K smpl:3M ep:15K epch:50.93 loss:0.007 grdn:0.221 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:01:45 ts/train.py:232 step:370K smpl:3M ep:15K epch:50.96 loss:0.008 grdn:0.220 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:03:10 ts/train.py:232 step:370K smpl:3M ep:15K epch:50.98 loss:0.007 grdn:0.212 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:04:36 ts/train.py:232 step:370K smpl:3M ep:15K epch:51.01 loss:0.007 grdn:0.215 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:06:01 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.04 loss:0.008 grdn:0.237 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:07:26 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.07 loss:0.007 grdn:0.210 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:08:53 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.09 loss:0.008 grdn:0.223 lr:6.4e-05 updt_s:0.424 data_s:0.008 +INFO 2025-05-07 10:10:18 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.12 loss:0.006 grdn:0.207 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:11:43 ts/train.py:232 step:371K smpl:3M ep:15K epch:51.15 loss:0.007 grdn:0.219 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:13:09 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.18 loss:0.009 grdn:0.242 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:14:34 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.20 loss:0.007 grdn:0.213 lr:6.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:16:00 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.23 loss:0.007 grdn:0.219 lr:6.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:17:25 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.26 loss:0.008 grdn:0.227 lr:6.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:18:50 ts/train.py:232 step:372K smpl:3M ep:15K epch:51.29 loss:0.007 grdn:0.210 lr:6.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:20:16 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.31 loss:0.007 grdn:0.221 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:21:41 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.34 loss:0.008 grdn:0.230 lr:6.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-07 10:23:06 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.37 loss:0.007 grdn:0.210 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:24:32 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.40 loss:0.007 grdn:0.212 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:25:57 ts/train.py:232 step:373K smpl:3M ep:15K epch:51.42 loss:0.007 grdn:0.219 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:27:23 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.45 loss:0.007 grdn:0.220 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:28:48 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.48 loss:0.007 grdn:0.221 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:30:14 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.51 loss:0.007 grdn:0.218 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:31:39 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.53 loss:0.007 grdn:0.223 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:33:05 ts/train.py:232 step:374K smpl:3M ep:15K epch:51.56 loss:0.008 grdn:0.226 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:34:30 ts/train.py:232 step:375K smpl:3M ep:15K epch:51.59 loss:0.007 grdn:0.213 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:35:56 ts/train.py:232 step:375K smpl:3M ep:15K epch:51.62 loss:0.009 grdn:0.244 lr:6.3e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-07 10:37:21 ts/train.py:232 step:375K smpl:3M ep:15K epch:51.64 loss:0.006 grdn:0.193 lr:6.3e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-07 10:38:47 ts/train.py:232 step:375K smpl:3M ep:16K epch:51.67 loss:0.007 grdn:0.224 lr:6.3e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-07 10:40:13 ts/train.py:232 step:375K smpl:3M ep:16K epch:51.70 loss:0.007 grdn:0.204 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:41:38 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.73 loss:0.008 grdn:0.237 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:43:04 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.75 loss:0.007 grdn:0.216 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:44:29 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.78 loss:0.008 grdn:0.223 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:45:55 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.81 loss:0.008 grdn:0.218 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:47:20 ts/train.py:232 step:376K smpl:3M ep:16K epch:51.84 loss:0.008 grdn:0.229 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:48:46 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.86 loss:0.008 grdn:0.229 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:50:11 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.89 loss:0.007 grdn:0.221 lr:6.3e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-07 10:51:37 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.92 loss:0.007 grdn:0.219 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:53:03 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.95 loss:0.007 grdn:0.216 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:54:28 ts/train.py:232 step:377K smpl:3M ep:16K epch:51.97 loss:0.007 grdn:0.204 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:55:54 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.00 loss:0.007 grdn:0.205 lr:6.3e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-07 10:57:19 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.03 loss:0.007 grdn:0.218 lr:6.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 10:58:46 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.06 loss:0.007 grdn:0.216 lr:6.3e-05 updt_s:0.426 data_s:0.008 +INFO 2025-05-07 11:00:11 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.08 loss:0.007 grdn:0.213 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:01:37 ts/train.py:232 step:378K smpl:3M ep:16K epch:52.11 loss:0.008 grdn:0.223 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:03:02 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.14 loss:0.007 grdn:0.210 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:04:28 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.17 loss:0.008 grdn:0.241 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:05:53 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.19 loss:0.008 grdn:0.234 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:07:19 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.22 loss:0.006 grdn:0.207 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:08:45 ts/train.py:232 step:379K smpl:3M ep:16K epch:52.25 loss:0.007 grdn:0.226 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:10:10 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.28 loss:0.007 grdn:0.200 lr:6.2e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-07 11:11:36 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.31 loss:0.008 grdn:0.226 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:13:01 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.33 loss:0.007 grdn:0.226 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:14:27 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.36 loss:0.007 grdn:0.215 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:15:52 ts/train.py:232 step:380K smpl:3M ep:16K epch:52.39 loss:0.007 grdn:0.217 lr:6.2e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-07 11:17:18 ts/train.py:232 step:381K smpl:3M ep:16K epch:52.42 loss:0.007 grdn:0.220 lr:6.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-07 11:18:44 ts/train.py:232 step:381K smpl:3M ep:16K epch:52.44 loss:0.007 grdn:0.228 lr:6.2e-05 updt_s:0.427 data_s:0.001 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/logs/debug-internal.log b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/logs/debug-internal.log index 92598c742a153b961b64179845bcc4ecb500cc40..6450f58d0252a6a4885872d37011b5eeb9df7503 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/logs/debug-internal.log +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/logs/debug-internal.log @@ -8,3 +8,6 @@ {"time":"2025-05-05T17:38:08.29480052Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/ygbytty1/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} {"time":"2025-05-06T09:12:57.512959236Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/ygbytty1/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:49420->141.223.7.51:53: i/o timeout"} {"time":"2025-05-06T11:39:57.516626433Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/ygbytty1/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:59675->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T18:01:54.895827141Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} +{"time":"2025-05-06T18:02:07.71693416Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/ygbytty1/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-05-06T22:47:19.821719943Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/graphql","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/run-ygbytty1.wandb b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/run-ygbytty1.wandb index bda6bdaabca8ca6cef74959b9bb63682270acdb9..52afeac8d221cb98ed9f07e0454b6c9e1e747ad1 100644 --- a/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/run-ygbytty1.wandb +++ b/DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/run-ygbytty1.wandb @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5948bf35f5295ee6abcdd5905fd4d5ab780c49623727aa096add9b2e9eddfda4 -size 19333120 +oid sha256:ff9a20d17cb653675a463246cca4ecb9f9cc576ff407cbfa5eeedfc2a4bebb94 +size 36012032