diff --git a/.gitattributes b/.gitattributes index bf4e7e8d2ff52add7ad6ef7014afc72df32dc50a..d3a3b4355458623d6cbf0eb1367dcb90a7ac7231 100644 --- a/.gitattributes +++ b/.gitattributes @@ -45,3 +45,4 @@ DP_cube_downDims1_cropNo_freeze1_16_16_ema1_1e-4/wandb/run-20250503_130343-yd8eh DP_pengripA_downDims1_cropNo224_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134237-92tvlwhj/run-92tvlwhj.wandb filter=lfs diff=lfs merge=lfs -text DP_pengripA_downDims1_cropNo_freeze0_16_16_ema1_1e-4/wandb/run-20250505_134851-ygbytty1/run-ygbytty1.wandb filter=lfs diff=lfs merge=lfs -text DP_pengripA_downDims1_cropNo224_freeze0_32_32_ema1_1e-4/wandb/run-20250505_134633-or9d7oy0/run-or9d7oy0.wandb filter=lfs diff=lfs merge=lfs -text +DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5488425dd624a3dd290f108d262d26a1b1fbc16f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2f137792705a5881bc6d37c9e9a99ca09672e04ec89255b5bfb7d18e0329f25 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..7d4ed5a676fc441768347a116ad61d8539fe81fa --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.973484658309718e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..838a879be3f4f39d888473c358886b6d862eb915 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b05e7bc06c8cc8dc07a2495a2a1cff7a56d614710bfef7b038644fe42c879a65 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9e04fe1672f996b47f09bf4baaff5be6efe15cc4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd67e4a42de6eb1ce11d8410ced7669b428e0885f2c9ce62c562cbf24365c01d +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bd0cc4c8bbd1814446b0a6695f94da31b23961de --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 30000, + "verbose": false, + "_step_count": 30001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.973484658309718e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..f4945f660f45b332883dccfccf18d8b8815d916a --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/030000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 30000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1b43460632ae07024627519d9b8205598e84efdf --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f90ce66ce2b0102e2bc9576fddedecc1bbd79ee0cf10e3b0d79827f6296fd9f8 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..c4b9153044afcd3a374917ee7385a8edfd2c00ce --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.892425861749099e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b44d41640d8b4eb23cef24bb91fd2414072dd71f --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce508f91e607f00cab5dfdfd68adf8cd553a8644755e882420384b837a6aee1c +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7f47e390e86040fe8d1ced34153d1b0072b66eb9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2794ded3636042056b90e2eaefec878b6476625eddafe900d986a79fc908df40 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9c3d845454183abd4bb260e7962ae0003eff2976 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 60000, + "verbose": false, + "_step_count": 60001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.892425861749099e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d360b6037f1c8c2c85d38e951160eb02ace507dd --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/060000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 60000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..553f66de4186094e00ebc0c7fd8cb2b1ad190af5 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f7c9786c81437d208df5863753ca3f2ccbd377eebaaff10888efb60598886c6 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..844517b39b97d4b240356b58a29f5d5477ec3544 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.757705070247532e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f64827d11b0ef0e6303af1486a88b7f4ce8808f9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a23df7667b0ed054591c48401509e6dabe222f115feaba2fe8fad0616a734dc1 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2e5b54b71a067df6c2c9d48527500278cbaf7f47 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6a1a9093daad95a61851476c2545beb995845fd30a4f92ade45755e1e0142cd +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fce6df3fd82ed4234f9a3b0fd9fc8ec6fc021844 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 90000, + "verbose": false, + "_step_count": 90001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.757705070247532e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..cec965b83b7f817ac99559792f84cbb69a721bfd --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/090000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 90000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7082bb728232e1f50a6a92a8dc9451439b23299c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33fbf5cbab0c0769475ae8abfbc53747ad21a6c8294b5bbb1b8b05547ae4aa71 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..303785ff5bc367d42704d992bf6eed47bb7272a1 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.57079995288704e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f2dc859610d994fb0496fa57cfbd2bde9942e1e9 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50bd4f8949d02f79f02f941a409191820745ce58d0d3cdbe0e31529b7a7aa919 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..26af21ebb933b728b74c629ed43d23aba7b6cf72 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:286963990516d4c5433f27d94b9e1bed62f771293a3d0c4f844b9760e26e0084 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f978e60892059238d68988919b32bd99948d45d3 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 120000, + "verbose": false, + "_step_count": 120001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.57079995288704e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbb8719463d819c865c104597046a0165eea278 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/120000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 120000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..470dd9e34901e783a6193ab1d76c36b4c05304d2 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f143d446ba7c9b3d983b23fde31524099bf91b56bfcc5746be32add1c48da49 +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..ccf8d114f2cb5b174697d10148c2b182caaed9c6 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.33376055636136e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9982a0b1c0e2a46d6e8546593171f6b05ebd7515 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e0ae4f1bf0cddd2ae64f495ab6abb74c5e7a82bc79ed3aa0c2b6e78fdf3ff32 +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..466217869be6caa008e3a6f164e64cadec7f4dfb --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dcc879f0c329f560607aa45d21f19b2b59f5ae52d0bbcc20db61b81f7768ed1 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0767923b0a738183eb68f042ae20f5f89c2f9ddf --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 150000, + "verbose": false, + "_step_count": 150001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.33376055636136e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..941897fdfccbe971211529c488ac32a17d6dc0a0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/150000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 150000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..96490ef90ed69a6e77803c6f7d345db99b0b9b6c --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/config.json @@ -0,0 +1,96 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/model.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a074b11bf781aa800d42746a34e88f6731a11bda --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c43d139a6991442e677a7346d9bb74d1997498707cc0437420d9ee97ceec9f4a +size 369243880 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/train_config.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a555bfdedabf26cc6cf6ea91e6068445715e1363 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/pretrained_model/train_config.json @@ -0,0 +1,206 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripA", + "root": "/SSD/LSY/lerobot_dataset/pengripA", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 32, + "n_action_steps": 32, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 480, + 640 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": null, + "freeze_vision_backbone": false, + "use_ema": true, + "ema_lambda": 0.5 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "job_name": "DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 8, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_param_groups.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..a029d09f58b7b30c6b0782f9db5301ca5a7ca89b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.049186819281031e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f8d9c4bb4921cc2ced5464ad36282780402ee025 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aac1f263f83b2da082dde0913cfd7563944956f40980664ba4ea03ccd79ff79d +size 738026076 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/rng_state.safetensors b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..609aa1030c68fcff7c39801ccaa01e23524c4e44 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8646b32e848eb0b787fa62daa33bc4c9a67054a1f5b0bf6a06ff525db9369175 +size 15708 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/scheduler_state.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7e725acefb858f85655970cd16215b4aec1acdea --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 180000, + "verbose": false, + "_step_count": 180001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.049186819281031e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/training_step.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..3da013f3092359f6c4ebb97dfc69e8300dbb32e4 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/checkpoints/180000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 180000 +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..12cf918367d22cc11e03be8cc2518db67592db79 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug-internal.log @@ -0,0 +1,13 @@ +{"time":"2025-05-05T13:50:12.201235767Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-core.log"} +{"time":"2025-05-05T13:50:12.532812412Z","level":"INFO","msg":"created new stream","id":"fjhp50hn"} +{"time":"2025-05-05T13:50:12.532892102Z","level":"INFO","msg":"stream: started","id":"fjhp50hn"} +{"time":"2025-05-05T13:50:12.532970282Z","level":"INFO","msg":"sender: started","stream_id":"fjhp50hn"} +{"time":"2025-05-05T13:50:12.532968422Z","level":"INFO","msg":"writer: Do: started","stream_id":"fjhp50hn"} +{"time":"2025-05-05T13:50:12.533061181Z","level":"INFO","msg":"handler: started","stream_id":"fjhp50hn"} +{"time":"2025-05-05T13:50:13.03263614Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-05-05T22:01:13.826925473Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-05-06T09:24:18.037401348Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:60326->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T10:19:48.044760185Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:38057->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T10:30:03.039602464Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:41710->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:25:33.040402545Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:56185->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:44:48.040371676Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:43741->141.223.7.51:53: i/o timeout"} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f73f226450c4172998f055736fc48ba880ec89d5 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/debug.log @@ -0,0 +1,22 @@ +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Configure stats pid to 106588 +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug.log +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:init():781] calling init triggers +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'shylee/so100_pengripA', 'root': '/SSD/LSY/lerobot_dataset/pengripA', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'horizon': 32, 'n_action_steps': 32, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [480, 640], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': None, 'freeze_vision_backbone': False, 'use_ema': True, 'ema_lambda': 0.5}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4', 'job_name': 'DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 900000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 30000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:init():809] starting backend +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:init():813] sending inform_init request +2025-05-05 13:50:12,199 INFO MainThread:106588 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-05-05 13:50:12,199 INFO MainThread:106588 [wandb_init.py:init():823] backend started and connected +2025-05-05 13:50:12,201 INFO MainThread:106588 [wandb_init.py:init():915] updated telemetry +2025-05-05 13:50:12,385 INFO MainThread:106588 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-05-05 13:50:13,028 INFO MainThread:106588 [wandb_init.py:init():1014] starting run threads in backend +2025-05-05 13:50:13,119 INFO MainThread:106588 [wandb_run.py:_console_start():2454] atexit reg +2025-05-05 13:50:13,120 INFO MainThread:106588 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-05-05 13:50:13,120 INFO MainThread:106588 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-05-05 13:50:13,120 INFO MainThread:106588 [wandb_run.py:_redirect():2394] Redirects installed. +2025-05-05 13:50:13,121 INFO MainThread:106588 [wandb_init.py:init():1056] run started, returning control to user process diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..f00c484d7033d17647899613f1997fabf4132cb0 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/output.log @@ -0,0 +1,1051 @@ +Logs will be synced with wandb. +INFO 2025-05-05 13:50:13 ndb_utils.py:96 Track this run --> https://wandb.ai/marchmelo0923-postech/lerobot/runs/fjhp50hn +INFO 2025-05-05 13:50:13 ts/train.py:127 Creating dataset +Resolving data files: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 300/300 [00:00<00:00, 35921.41it/s] +INFO 2025-05-05 13:50:14 ts/train.py:138 Creating policy +INFO 2025-05-05 13:50:21 ts/train.py:144 Creating optimizer and scheduler +INFO 2025-05-05 13:50:21 ts/train.py:156 Output dir: /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4 +INFO 2025-05-05 13:50:21 ts/train.py:159 cfg.steps=900000 (900K) +INFO 2025-05-05 13:50:21 ts/train.py:160 dataset.num_frames=58090 (58K) +INFO 2025-05-05 13:50:21 ts/train.py:161 dataset.num_episodes=300 +INFO 2025-05-05 13:50:21 ts/train.py:162 num_learnable_params=92242982 (92M) +INFO 2025-05-05 13:50:21 ts/train.py:163 num_total_params=92243036 (92M) +INFO 2025-05-05 13:50:21 ts/train.py:202 Start offline training on a fixed dataset +INFO 2025-05-05 13:52:07 ts/train.py:232 step:200 smpl:2K ep:8 epch:0.03 loss:0.752 grdn:3.966 lr:2.0e-05 updt_s:0.517 data_s:0.011 +INFO 2025-05-05 13:53:33 ts/train.py:232 step:400 smpl:3K ep:17 epch:0.06 loss:0.158 grdn:2.980 lr:6.0e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 13:54:59 ts/train.py:232 step:600 smpl:5K ep:25 epch:0.08 loss:0.110 grdn:2.103 lr:9.5e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 13:56:26 ts/train.py:232 step:800 smpl:6K ep:33 epch:0.11 loss:0.069 grdn:1.286 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 13:57:52 ts/train.py:232 step:1K smpl:8K ep:41 epch:0.14 loss:0.058 grdn:1.119 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 13:59:19 ts/train.py:232 step:1K smpl:10K ep:50 epch:0.17 loss:0.055 grdn:1.006 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:00:45 ts/train.py:232 step:1K smpl:11K ep:58 epch:0.19 loss:0.050 grdn:0.903 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:02:12 ts/train.py:232 step:2K smpl:13K ep:66 epch:0.22 loss:0.043 grdn:0.815 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:03:38 ts/train.py:232 step:2K smpl:14K ep:74 epch:0.25 loss:0.043 grdn:0.818 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:05:05 ts/train.py:232 step:2K smpl:16K ep:83 epch:0.28 loss:0.039 grdn:0.722 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:06:31 ts/train.py:232 step:2K smpl:18K ep:91 epch:0.30 loss:0.040 grdn:0.722 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:07:58 ts/train.py:232 step:2K smpl:19K ep:99 epch:0.33 loss:0.040 grdn:0.698 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:09:24 ts/train.py:232 step:3K smpl:21K ep:107 epch:0.36 loss:0.040 grdn:0.666 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:10:51 ts/train.py:232 step:3K smpl:22K ep:116 epch:0.39 loss:0.034 grdn:0.593 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:12:17 ts/train.py:232 step:3K smpl:24K ep:124 epch:0.41 loss:0.036 grdn:0.625 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:13:44 ts/train.py:232 step:3K smpl:26K ep:132 epch:0.44 loss:0.035 grdn:0.563 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:15:10 ts/train.py:232 step:3K smpl:27K ep:140 epch:0.47 loss:0.033 grdn:0.536 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:16:37 ts/train.py:232 step:4K smpl:29K ep:149 epch:0.50 loss:0.032 grdn:0.514 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:18:03 ts/train.py:232 step:4K smpl:30K ep:157 epch:0.52 loss:0.032 grdn:0.532 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:19:30 ts/train.py:232 step:4K smpl:32K ep:165 epch:0.55 loss:0.032 grdn:0.530 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:20:56 ts/train.py:232 step:4K smpl:34K ep:174 epch:0.58 loss:0.031 grdn:0.498 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 14:22:23 ts/train.py:232 step:4K smpl:35K ep:182 epch:0.61 loss:0.030 grdn:0.469 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 14:23:49 ts/train.py:232 step:5K smpl:37K ep:190 epch:0.63 loss:0.030 grdn:0.468 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:25:16 ts/train.py:232 step:5K smpl:38K ep:198 epch:0.66 loss:0.031 grdn:0.484 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:26:42 ts/train.py:232 step:5K smpl:40K ep:207 epch:0.69 loss:0.029 grdn:0.430 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:28:09 ts/train.py:232 step:5K smpl:42K ep:215 epch:0.72 loss:0.029 grdn:0.454 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:29:35 ts/train.py:232 step:5K smpl:43K ep:223 epch:0.74 loss:0.029 grdn:0.454 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:31:02 ts/train.py:232 step:6K smpl:45K ep:231 epch:0.77 loss:0.027 grdn:0.421 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:32:28 ts/train.py:232 step:6K smpl:46K ep:240 epch:0.80 loss:0.031 grdn:0.458 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:33:55 ts/train.py:232 step:6K smpl:48K ep:248 epch:0.83 loss:0.027 grdn:0.397 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 14:35:22 ts/train.py:232 step:6K smpl:50K ep:256 epch:0.85 loss:0.025 grdn:0.385 lr:1.0e-04 updt_s:0.433 data_s:0.001 +INFO 2025-05-05 14:36:49 ts/train.py:232 step:6K smpl:51K ep:264 epch:0.88 loss:0.029 grdn:0.414 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 14:38:15 ts/train.py:232 step:7K smpl:53K ep:273 epch:0.91 loss:0.027 grdn:0.387 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 14:39:42 ts/train.py:232 step:7K smpl:54K ep:281 epch:0.94 loss:0.028 grdn:0.398 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 14:41:22 ts/train.py:232 step:7K smpl:56K ep:289 epch:0.96 loss:0.027 grdn:0.379 lr:1.0e-04 updt_s:0.492 data_s:0.009 +INFO 2025-05-05 14:42:49 ts/train.py:232 step:7K smpl:58K ep:297 epch:0.99 loss:0.027 grdn:0.378 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:44:15 ts/train.py:232 step:7K smpl:59K ep:306 epch:1.02 loss:0.027 grdn:0.390 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:45:42 ts/train.py:232 step:8K smpl:61K ep:314 epch:1.05 loss:0.026 grdn:0.385 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 14:47:09 ts/train.py:232 step:8K smpl:62K ep:322 epch:1.07 loss:0.024 grdn:0.352 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 14:48:35 ts/train.py:232 step:8K smpl:64K ep:331 epch:1.10 loss:0.027 grdn:0.361 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 14:50:02 ts/train.py:232 step:8K smpl:66K ep:339 epch:1.13 loss:0.025 grdn:0.357 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 14:51:29 ts/train.py:232 step:8K smpl:67K ep:347 epch:1.16 loss:0.027 grdn:0.364 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:52:55 ts/train.py:232 step:9K smpl:69K ep:355 epch:1.18 loss:0.027 grdn:0.360 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:54:22 ts/train.py:232 step:9K smpl:70K ep:364 epch:1.21 loss:0.026 grdn:0.358 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 14:55:49 ts/train.py:232 step:9K smpl:72K ep:372 epch:1.24 loss:0.025 grdn:0.338 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:57:15 ts/train.py:232 step:9K smpl:74K ep:380 epch:1.27 loss:0.025 grdn:0.320 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 14:58:42 ts/train.py:232 step:9K smpl:75K ep:388 epch:1.29 loss:0.024 grdn:0.316 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:00:08 ts/train.py:232 step:10K smpl:77K ep:397 epch:1.32 loss:0.024 grdn:0.318 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:01:35 ts/train.py:232 step:10K smpl:78K ep:405 epch:1.35 loss:0.026 grdn:0.338 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:03:01 ts/train.py:232 step:10K smpl:80K ep:413 epch:1.38 loss:0.024 grdn:0.321 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:04:28 ts/train.py:232 step:10K smpl:82K ep:421 epch:1.40 loss:0.024 grdn:0.316 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:05:55 ts/train.py:232 step:10K smpl:83K ep:430 epch:1.43 loss:0.025 grdn:0.334 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:07:21 ts/train.py:232 step:11K smpl:85K ep:438 epch:1.46 loss:0.022 grdn:0.295 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:08:48 ts/train.py:232 step:11K smpl:86K ep:446 epch:1.49 loss:0.022 grdn:0.307 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:10:14 ts/train.py:232 step:11K smpl:88K ep:454 epch:1.51 loss:0.025 grdn:0.325 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:11:41 ts/train.py:232 step:11K smpl:90K ep:463 epch:1.54 loss:0.026 grdn:0.310 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:13:08 ts/train.py:232 step:11K smpl:91K ep:471 epch:1.57 loss:0.025 grdn:0.324 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 15:14:34 ts/train.py:232 step:12K smpl:93K ep:479 epch:1.60 loss:0.025 grdn:0.308 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:16:01 ts/train.py:232 step:12K smpl:94K ep:488 epch:1.63 loss:0.022 grdn:0.294 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:17:28 ts/train.py:232 step:12K smpl:96K ep:496 epch:1.65 loss:0.023 grdn:0.316 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 15:18:54 ts/train.py:232 step:12K smpl:98K ep:504 epch:1.68 loss:0.023 grdn:0.275 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:20:21 ts/train.py:232 step:12K smpl:99K ep:512 epch:1.71 loss:0.022 grdn:0.288 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 15:21:48 ts/train.py:232 step:13K smpl:101K ep:521 epch:1.74 loss:0.023 grdn:0.301 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 15:23:14 ts/train.py:232 step:13K smpl:102K ep:529 epch:1.76 loss:0.021 grdn:0.284 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:24:41 ts/train.py:232 step:13K smpl:104K ep:537 epch:1.79 loss:0.023 grdn:0.294 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:26:07 ts/train.py:232 step:13K smpl:106K ep:545 epch:1.82 loss:0.024 grdn:0.292 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 15:27:34 ts/train.py:232 step:13K smpl:107K ep:554 epch:1.85 loss:0.022 grdn:0.283 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 15:29:01 ts/train.py:232 step:14K smpl:109K ep:562 epch:1.87 loss:0.022 grdn:0.303 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 15:30:27 ts/train.py:232 step:14K smpl:110K ep:570 epch:1.90 loss:0.021 grdn:0.278 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:31:55 ts/train.py:232 step:14K smpl:112K ep:578 epch:1.93 loss:0.021 grdn:0.279 lr:1.0e-04 updt_s:0.430 data_s:0.009 +INFO 2025-05-05 15:33:22 ts/train.py:232 step:14K smpl:114K ep:587 epch:1.96 loss:0.022 grdn:0.285 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:34:49 ts/train.py:232 step:14K smpl:115K ep:595 epch:1.98 loss:0.022 grdn:0.274 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:36:15 ts/train.py:232 step:15K smpl:117K ep:603 epch:2.01 loss:0.019 grdn:0.262 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:37:42 ts/train.py:232 step:15K smpl:118K ep:611 epch:2.04 loss:0.020 grdn:0.254 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:39:08 ts/train.py:232 step:15K smpl:120K ep:620 epch:2.07 loss:0.021 grdn:0.274 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 15:40:35 ts/train.py:232 step:15K smpl:122K ep:628 epch:2.09 loss:0.022 grdn:0.291 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 15:42:01 ts/train.py:232 step:15K smpl:123K ep:636 epch:2.12 loss:0.021 grdn:0.275 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:43:28 ts/train.py:232 step:16K smpl:125K ep:645 epch:2.15 loss:0.023 grdn:0.279 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:44:54 ts/train.py:232 step:16K smpl:126K ep:653 epch:2.18 loss:0.020 grdn:0.264 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:46:21 ts/train.py:232 step:16K smpl:128K ep:661 epch:2.20 loss:0.022 grdn:0.274 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:47:47 ts/train.py:232 step:16K smpl:130K ep:669 epch:2.23 loss:0.021 grdn:0.272 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:49:14 ts/train.py:232 step:16K smpl:131K ep:678 epch:2.26 loss:0.020 grdn:0.266 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 15:50:40 ts/train.py:232 step:17K smpl:133K ep:686 epch:2.29 loss:0.021 grdn:0.269 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:52:07 ts/train.py:232 step:17K smpl:134K ep:694 epch:2.31 loss:0.023 grdn:0.285 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:53:33 ts/train.py:232 step:17K smpl:136K ep:702 epch:2.34 loss:0.019 grdn:0.257 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:55:00 ts/train.py:232 step:17K smpl:138K ep:711 epch:2.37 loss:0.020 grdn:0.268 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:56:26 ts/train.py:232 step:17K smpl:139K ep:719 epch:2.40 loss:0.020 grdn:0.262 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:57:53 ts/train.py:232 step:18K smpl:141K ep:727 epch:2.42 loss:0.020 grdn:0.251 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 15:59:19 ts/train.py:232 step:18K smpl:142K ep:735 epch:2.45 loss:0.020 grdn:0.257 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:00:46 ts/train.py:232 step:18K smpl:144K ep:744 epch:2.48 loss:0.022 grdn:0.276 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:02:12 ts/train.py:232 step:18K smpl:146K ep:752 epch:2.51 loss:0.020 grdn:0.266 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:03:39 ts/train.py:232 step:18K smpl:147K ep:760 epch:2.53 loss:0.021 grdn:0.267 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:05:05 ts/train.py:232 step:19K smpl:149K ep:768 epch:2.56 loss:0.021 grdn:0.262 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:06:32 ts/train.py:232 step:19K smpl:150K ep:777 epch:2.59 loss:0.021 grdn:0.271 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:07:58 ts/train.py:232 step:19K smpl:152K ep:785 epch:2.62 loss:0.021 grdn:0.267 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:09:25 ts/train.py:232 step:19K smpl:154K ep:793 epch:2.64 loss:0.020 grdn:0.251 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:10:51 ts/train.py:232 step:19K smpl:155K ep:802 epch:2.67 loss:0.019 grdn:0.251 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:12:18 ts/train.py:232 step:20K smpl:157K ep:810 epch:2.70 loss:0.019 grdn:0.248 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:13:44 ts/train.py:232 step:20K smpl:158K ep:818 epch:2.73 loss:0.018 grdn:0.238 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:15:11 ts/train.py:232 step:20K smpl:160K ep:826 epch:2.75 loss:0.020 grdn:0.254 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:16:37 ts/train.py:232 step:20K smpl:162K ep:835 epch:2.78 loss:0.021 grdn:0.267 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:18:04 ts/train.py:232 step:20K smpl:163K ep:843 epch:2.81 loss:0.019 grdn:0.255 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 16:19:31 ts/train.py:232 step:21K smpl:165K ep:851 epch:2.84 loss:0.019 grdn:0.245 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:20:57 ts/train.py:232 step:21K smpl:166K ep:859 epch:2.86 loss:0.019 grdn:0.246 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:22:25 ts/train.py:232 step:21K smpl:168K ep:868 epch:2.89 loss:0.019 grdn:0.244 lr:1.0e-04 updt_s:0.431 data_s:0.009 +INFO 2025-05-05 16:23:52 ts/train.py:232 step:21K smpl:170K ep:876 epch:2.92 loss:0.019 grdn:0.257 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 16:25:19 ts/train.py:232 step:21K smpl:171K ep:884 epch:2.95 loss:0.020 grdn:0.257 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:26:45 ts/train.py:232 step:22K smpl:173K ep:892 epch:2.97 loss:0.019 grdn:0.240 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 16:28:12 ts/train.py:232 step:22K smpl:174K ep:901 epch:3.00 loss:0.019 grdn:0.240 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 16:29:39 ts/train.py:232 step:22K smpl:176K ep:909 epch:3.03 loss:0.019 grdn:0.254 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 16:31:05 ts/train.py:232 step:22K smpl:178K ep:917 epch:3.06 loss:0.019 grdn:0.254 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:32:32 ts/train.py:232 step:22K smpl:179K ep:925 epch:3.08 loss:0.018 grdn:0.246 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:33:58 ts/train.py:232 step:23K smpl:181K ep:934 epch:3.11 loss:0.018 grdn:0.240 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:35:25 ts/train.py:232 step:23K smpl:182K ep:942 epch:3.14 loss:0.018 grdn:0.231 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:36:51 ts/train.py:232 step:23K smpl:184K ep:950 epch:3.17 loss:0.018 grdn:0.241 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:38:18 ts/train.py:232 step:23K smpl:186K ep:959 epch:3.20 loss:0.017 grdn:0.238 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:39:44 ts/train.py:232 step:23K smpl:187K ep:967 epch:3.22 loss:0.018 grdn:0.245 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:41:11 ts/train.py:232 step:24K smpl:189K ep:975 epch:3.25 loss:0.018 grdn:0.238 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:42:37 ts/train.py:232 step:24K smpl:190K ep:983 epch:3.28 loss:0.018 grdn:0.237 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:44:04 ts/train.py:232 step:24K smpl:192K ep:992 epch:3.31 loss:0.019 grdn:0.247 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:45:30 ts/train.py:232 step:24K smpl:194K ep:1000 epch:3.33 loss:0.018 grdn:0.239 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 16:46:56 ts/train.py:232 step:24K smpl:195K ep:1K epch:3.36 loss:0.018 grdn:0.242 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 16:48:23 ts/train.py:232 step:25K smpl:197K ep:1K epch:3.39 loss:0.017 grdn:0.233 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 16:49:49 ts/train.py:232 step:25K smpl:198K ep:1K epch:3.42 loss:0.018 grdn:0.241 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 16:51:16 ts/train.py:232 step:25K smpl:200K ep:1K epch:3.44 loss:0.018 grdn:0.228 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 16:52:42 ts/train.py:232 step:25K smpl:202K ep:1K epch:3.47 loss:0.017 grdn:0.235 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:54:09 ts/train.py:232 step:25K smpl:203K ep:1K epch:3.50 loss:0.019 grdn:0.234 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:55:35 ts/train.py:232 step:26K smpl:205K ep:1K epch:3.53 loss:0.019 grdn:0.247 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:57:02 ts/train.py:232 step:26K smpl:206K ep:1K epch:3.55 loss:0.017 grdn:0.234 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:58:28 ts/train.py:232 step:26K smpl:208K ep:1K epch:3.58 loss:0.017 grdn:0.242 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 16:59:55 ts/train.py:232 step:26K smpl:210K ep:1K epch:3.61 loss:0.016 grdn:0.226 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:01:21 ts/train.py:232 step:26K smpl:211K ep:1K epch:3.64 loss:0.017 grdn:0.226 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:02:48 ts/train.py:232 step:27K smpl:213K ep:1K epch:3.66 loss:0.017 grdn:0.237 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:04:14 ts/train.py:232 step:27K smpl:214K ep:1K epch:3.69 loss:0.019 grdn:0.235 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:05:40 ts/train.py:232 step:27K smpl:216K ep:1K epch:3.72 loss:0.016 grdn:0.230 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:07:07 ts/train.py:232 step:27K smpl:218K ep:1K epch:3.75 loss:0.018 grdn:0.244 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:08:33 ts/train.py:232 step:27K smpl:219K ep:1K epch:3.77 loss:0.018 grdn:0.243 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:09:59 ts/train.py:232 step:28K smpl:221K ep:1K epch:3.80 loss:0.019 grdn:0.243 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:11:26 ts/train.py:232 step:28K smpl:222K ep:1K epch:3.83 loss:0.019 grdn:0.240 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:12:53 ts/train.py:232 step:28K smpl:224K ep:1K epch:3.86 loss:0.017 grdn:0.228 lr:1.0e-04 updt_s:0.429 data_s:0.009 +INFO 2025-05-05 17:14:20 ts/train.py:232 step:28K smpl:226K ep:1K epch:3.88 loss:0.017 grdn:0.225 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:15:46 ts/train.py:232 step:28K smpl:227K ep:1K epch:3.91 loss:0.017 grdn:0.224 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:17:12 ts/train.py:232 step:29K smpl:229K ep:1K epch:3.94 loss:0.018 grdn:0.231 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:18:39 ts/train.py:232 step:29K smpl:230K ep:1K epch:3.97 loss:0.018 grdn:0.237 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:20:05 ts/train.py:232 step:29K smpl:232K ep:1K epch:3.99 loss:0.018 grdn:0.236 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:21:32 ts/train.py:232 step:29K smpl:234K ep:1K epch:4.02 loss:0.017 grdn:0.230 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:22:58 ts/train.py:232 step:29K smpl:235K ep:1K epch:4.05 loss:0.018 grdn:0.235 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:24:24 ts/train.py:232 step:30K smpl:237K ep:1K epch:4.08 loss:0.018 grdn:0.233 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:25:51 ts/train.py:232 step:30K smpl:238K ep:1K epch:4.10 loss:0.016 grdn:0.227 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:27:17 ts/train.py:232 step:30K smpl:240K ep:1K epch:4.13 loss:0.017 grdn:0.230 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:27:17 ts/train.py:241 Checkpoint policy after step 30000 +INFO 2025-05-05 17:28:47 ts/train.py:232 step:30K smpl:242K ep:1K epch:4.16 loss:0.017 grdn:0.227 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:30:13 ts/train.py:232 step:30K smpl:243K ep:1K epch:4.19 loss:0.018 grdn:0.239 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:31:40 ts/train.py:232 step:31K smpl:245K ep:1K epch:4.21 loss:0.016 grdn:0.232 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:33:06 ts/train.py:232 step:31K smpl:246K ep:1K epch:4.24 loss:0.017 grdn:0.232 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:34:32 ts/train.py:232 step:31K smpl:248K ep:1K epch:4.27 loss:0.016 grdn:0.229 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:35:59 ts/train.py:232 step:31K smpl:250K ep:1K epch:4.30 loss:0.018 grdn:0.238 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:37:25 ts/train.py:232 step:31K smpl:251K ep:1K epch:4.32 loss:0.018 grdn:0.244 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 17:38:52 ts/train.py:232 step:32K smpl:253K ep:1K epch:4.35 loss:0.016 grdn:0.229 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:40:18 ts/train.py:232 step:32K smpl:254K ep:1K epch:4.38 loss:0.017 grdn:0.232 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:41:45 ts/train.py:232 step:32K smpl:256K ep:1K epch:4.41 loss:0.017 grdn:0.228 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:43:11 ts/train.py:232 step:32K smpl:258K ep:1K epch:4.43 loss:0.017 grdn:0.230 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:44:38 ts/train.py:232 step:32K smpl:259K ep:1K epch:4.46 loss:0.018 grdn:0.233 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:46:04 ts/train.py:232 step:33K smpl:261K ep:1K epch:4.49 loss:0.017 grdn:0.238 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:47:31 ts/train.py:232 step:33K smpl:262K ep:1K epch:4.52 loss:0.016 grdn:0.220 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:48:57 ts/train.py:232 step:33K smpl:264K ep:1K epch:4.54 loss:0.017 grdn:0.238 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:50:24 ts/train.py:232 step:33K smpl:266K ep:1K epch:4.57 loss:0.018 grdn:0.237 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:51:50 ts/train.py:232 step:33K smpl:267K ep:1K epch:4.60 loss:0.016 grdn:0.216 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:53:16 ts/train.py:232 step:34K smpl:269K ep:1K epch:4.63 loss:0.017 grdn:0.229 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:54:43 ts/train.py:232 step:34K smpl:270K ep:1K epch:4.65 loss:0.018 grdn:0.240 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 17:56:10 ts/train.py:232 step:34K smpl:272K ep:1K epch:4.68 loss:0.017 grdn:0.228 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 17:57:47 ts/train.py:232 step:34K smpl:274K ep:1K epch:4.71 loss:0.017 grdn:0.223 lr:1.0e-04 updt_s:0.486 data_s:0.001 +INFO 2025-05-05 17:59:14 ts/train.py:232 step:34K smpl:275K ep:1K epch:4.74 loss:0.017 grdn:0.236 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:00:40 ts/train.py:232 step:35K smpl:277K ep:1K epch:4.77 loss:0.017 grdn:0.229 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:02:07 ts/train.py:232 step:35K smpl:278K ep:1K epch:4.79 loss:0.018 grdn:0.231 lr:1.0e-04 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 18:03:35 ts/train.py:232 step:35K smpl:280K ep:1K epch:4.82 loss:0.016 grdn:0.215 lr:1.0e-04 updt_s:0.430 data_s:0.009 +INFO 2025-05-05 18:05:01 ts/train.py:232 step:35K smpl:282K ep:1K epch:4.85 loss:0.016 grdn:0.222 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:06:28 ts/train.py:232 step:35K smpl:283K ep:1K epch:4.88 loss:0.016 grdn:0.224 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:07:54 ts/train.py:232 step:36K smpl:285K ep:1K epch:4.90 loss:0.015 grdn:0.223 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:09:21 ts/train.py:232 step:36K smpl:286K ep:1K epch:4.93 loss:0.016 grdn:0.226 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:10:48 ts/train.py:232 step:36K smpl:288K ep:1K epch:4.96 loss:0.015 grdn:0.225 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:12:15 ts/train.py:232 step:36K smpl:290K ep:1K epch:4.99 loss:0.016 grdn:0.226 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:13:41 ts/train.py:232 step:36K smpl:291K ep:2K epch:5.01 loss:0.017 grdn:0.227 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:15:08 ts/train.py:232 step:37K smpl:293K ep:2K epch:5.04 loss:0.016 grdn:0.235 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:16:35 ts/train.py:232 step:37K smpl:294K ep:2K epch:5.07 loss:0.016 grdn:0.219 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:18:01 ts/train.py:232 step:37K smpl:296K ep:2K epch:5.10 loss:0.016 grdn:0.223 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:19:28 ts/train.py:232 step:37K smpl:298K ep:2K epch:5.12 loss:0.016 grdn:0.217 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:20:55 ts/train.py:232 step:37K smpl:299K ep:2K epch:5.15 loss:0.015 grdn:0.216 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:22:21 ts/train.py:232 step:38K smpl:301K ep:2K epch:5.18 loss:0.015 grdn:0.216 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:23:48 ts/train.py:232 step:38K smpl:302K ep:2K epch:5.21 loss:0.015 grdn:0.225 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:25:14 ts/train.py:232 step:38K smpl:304K ep:2K epch:5.23 loss:0.015 grdn:0.217 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:26:41 ts/train.py:232 step:38K smpl:306K ep:2K epch:5.26 loss:0.016 grdn:0.231 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:28:08 ts/train.py:232 step:38K smpl:307K ep:2K epch:5.29 loss:0.016 grdn:0.218 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:29:35 ts/train.py:232 step:39K smpl:309K ep:2K epch:5.32 loss:0.016 grdn:0.215 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:31:01 ts/train.py:232 step:39K smpl:310K ep:2K epch:5.34 loss:0.016 grdn:0.228 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:32:28 ts/train.py:232 step:39K smpl:312K ep:2K epch:5.37 loss:0.017 grdn:0.227 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:33:55 ts/train.py:232 step:39K smpl:314K ep:2K epch:5.40 loss:0.016 grdn:0.223 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:35:21 ts/train.py:232 step:39K smpl:315K ep:2K epch:5.43 loss:0.017 grdn:0.228 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:36:48 ts/train.py:232 step:40K smpl:317K ep:2K epch:5.45 loss:0.015 grdn:0.222 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:38:15 ts/train.py:232 step:40K smpl:318K ep:2K epch:5.48 loss:0.016 grdn:0.220 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:39:41 ts/train.py:232 step:40K smpl:320K ep:2K epch:5.51 loss:0.016 grdn:0.230 lr:1.0e-04 updt_s:0.432 data_s:0.001 +INFO 2025-05-05 18:41:08 ts/train.py:232 step:40K smpl:322K ep:2K epch:5.54 loss:0.017 grdn:0.230 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:42:34 ts/train.py:232 step:40K smpl:323K ep:2K epch:5.56 loss:0.016 grdn:0.223 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:44:01 ts/train.py:232 step:41K smpl:325K ep:2K epch:5.59 loss:0.015 grdn:0.212 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:45:27 ts/train.py:232 step:41K smpl:326K ep:2K epch:5.62 loss:0.015 grdn:0.209 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:46:54 ts/train.py:232 step:41K smpl:328K ep:2K epch:5.65 loss:0.017 grdn:0.230 lr:1.0e-04 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:48:20 ts/train.py:232 step:41K smpl:330K ep:2K epch:5.67 loss:0.016 grdn:0.214 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:49:47 ts/train.py:232 step:41K smpl:331K ep:2K epch:5.70 loss:0.015 grdn:0.222 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:51:13 ts/train.py:232 step:42K smpl:333K ep:2K epch:5.73 loss:0.016 grdn:0.219 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:52:40 ts/train.py:232 step:42K smpl:334K ep:2K epch:5.76 loss:0.016 grdn:0.213 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:54:08 ts/train.py:232 step:42K smpl:336K ep:2K epch:5.78 loss:0.015 grdn:0.219 lr:9.9e-05 updt_s:0.431 data_s:0.008 +INFO 2025-05-05 18:55:34 ts/train.py:232 step:42K smpl:338K ep:2K epch:5.81 loss:0.014 grdn:0.207 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:57:01 ts/train.py:232 step:42K smpl:339K ep:2K epch:5.84 loss:0.015 grdn:0.206 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:58:27 ts/train.py:232 step:43K smpl:341K ep:2K epch:5.87 loss:0.017 grdn:0.236 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 18:59:54 ts/train.py:232 step:43K smpl:342K ep:2K epch:5.89 loss:0.016 grdn:0.224 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:01:20 ts/train.py:232 step:43K smpl:344K ep:2K epch:5.92 loss:0.015 grdn:0.213 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:02:47 ts/train.py:232 step:43K smpl:346K ep:2K epch:5.95 loss:0.015 grdn:0.224 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:04:13 ts/train.py:232 step:43K smpl:347K ep:2K epch:5.98 loss:0.015 grdn:0.206 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:05:39 ts/train.py:232 step:44K smpl:349K ep:2K epch:6.00 loss:0.014 grdn:0.208 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:07:06 ts/train.py:232 step:44K smpl:350K ep:2K epch:6.03 loss:0.015 grdn:0.212 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:08:32 ts/train.py:232 step:44K smpl:352K ep:2K epch:6.06 loss:0.015 grdn:0.208 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:09:59 ts/train.py:232 step:44K smpl:354K ep:2K epch:6.09 loss:0.015 grdn:0.221 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:11:25 ts/train.py:232 step:44K smpl:355K ep:2K epch:6.11 loss:0.017 grdn:0.228 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:12:52 ts/train.py:232 step:45K smpl:357K ep:2K epch:6.14 loss:0.015 grdn:0.210 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:14:18 ts/train.py:232 step:45K smpl:358K ep:2K epch:6.17 loss:0.016 grdn:0.218 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:15:45 ts/train.py:232 step:45K smpl:360K ep:2K epch:6.20 loss:0.018 grdn:0.232 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:17:11 ts/train.py:232 step:45K smpl:362K ep:2K epch:6.22 loss:0.016 grdn:0.219 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:18:38 ts/train.py:232 step:45K smpl:363K ep:2K epch:6.25 loss:0.015 grdn:0.211 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:20:04 ts/train.py:232 step:46K smpl:365K ep:2K epch:6.28 loss:0.015 grdn:0.217 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:21:31 ts/train.py:232 step:46K smpl:366K ep:2K epch:6.31 loss:0.015 grdn:0.218 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:22:57 ts/train.py:232 step:46K smpl:368K ep:2K epch:6.33 loss:0.016 grdn:0.224 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:24:23 ts/train.py:232 step:46K smpl:370K ep:2K epch:6.36 loss:0.016 grdn:0.225 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:25:50 ts/train.py:232 step:46K smpl:371K ep:2K epch:6.39 loss:0.015 grdn:0.221 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:27:16 ts/train.py:232 step:47K smpl:373K ep:2K epch:6.42 loss:0.014 grdn:0.208 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:28:42 ts/train.py:232 step:47K smpl:374K ep:2K epch:6.45 loss:0.014 grdn:0.204 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:30:09 ts/train.py:232 step:47K smpl:376K ep:2K epch:6.47 loss:0.015 grdn:0.211 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:31:35 ts/train.py:232 step:47K smpl:378K ep:2K epch:6.50 loss:0.015 grdn:0.214 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:33:01 ts/train.py:232 step:47K smpl:379K ep:2K epch:6.53 loss:0.015 grdn:0.210 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:34:28 ts/train.py:232 step:48K smpl:381K ep:2K epch:6.56 loss:0.015 grdn:0.216 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:35:54 ts/train.py:232 step:48K smpl:382K ep:2K epch:6.58 loss:0.016 grdn:0.218 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:37:21 ts/train.py:232 step:48K smpl:384K ep:2K epch:6.61 loss:0.015 grdn:0.221 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:38:47 ts/train.py:232 step:48K smpl:386K ep:2K epch:6.64 loss:0.014 grdn:0.210 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:40:13 ts/train.py:232 step:48K smpl:387K ep:2K epch:6.67 loss:0.016 grdn:0.219 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:41:40 ts/train.py:232 step:49K smpl:389K ep:2K epch:6.69 loss:0.015 grdn:0.210 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:43:06 ts/train.py:232 step:49K smpl:390K ep:2K epch:6.72 loss:0.014 grdn:0.202 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:44:34 ts/train.py:232 step:49K smpl:392K ep:2K epch:6.75 loss:0.016 grdn:0.223 lr:9.9e-05 updt_s:0.430 data_s:0.008 +INFO 2025-05-05 19:46:01 ts/train.py:232 step:49K smpl:394K ep:2K epch:6.78 loss:0.014 grdn:0.212 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:47:27 ts/train.py:232 step:49K smpl:395K ep:2K epch:6.80 loss:0.015 grdn:0.210 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:48:54 ts/train.py:232 step:50K smpl:397K ep:2K epch:6.83 loss:0.014 grdn:0.213 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:50:20 ts/train.py:232 step:50K smpl:398K ep:2K epch:6.86 loss:0.014 grdn:0.209 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:51:46 ts/train.py:232 step:50K smpl:400K ep:2K epch:6.89 loss:0.015 grdn:0.214 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:53:13 ts/train.py:232 step:50K smpl:402K ep:2K epch:6.91 loss:0.016 grdn:0.227 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:54:39 ts/train.py:232 step:50K smpl:403K ep:2K epch:6.94 loss:0.013 grdn:0.198 lr:9.9e-05 updt_s:0.431 data_s:0.001 +INFO 2025-05-05 19:56:06 ts/train.py:232 step:51K smpl:405K ep:2K epch:6.97 loss:0.015 grdn:0.212 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:57:32 ts/train.py:232 step:51K smpl:406K ep:2K epch:7.00 loss:0.013 grdn:0.204 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 19:58:59 ts/train.py:232 step:51K smpl:408K ep:2K epch:7.02 loss:0.014 grdn:0.214 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:00:25 ts/train.py:232 step:51K smpl:410K ep:2K epch:7.05 loss:0.014 grdn:0.208 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:01:51 ts/train.py:232 step:51K smpl:411K ep:2K epch:7.08 loss:0.014 grdn:0.204 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:03:18 ts/train.py:232 step:52K smpl:413K ep:2K epch:7.11 loss:0.015 grdn:0.224 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:04:44 ts/train.py:232 step:52K smpl:414K ep:2K epch:7.13 loss:0.015 grdn:0.228 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:06:10 ts/train.py:232 step:52K smpl:416K ep:2K epch:7.16 loss:0.015 grdn:0.223 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:07:37 ts/train.py:232 step:52K smpl:418K ep:2K epch:7.19 loss:0.016 grdn:0.213 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:09:03 ts/train.py:232 step:52K smpl:419K ep:2K epch:7.22 loss:0.015 grdn:0.214 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:10:29 ts/train.py:232 step:53K smpl:421K ep:2K epch:7.24 loss:0.014 grdn:0.211 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:11:56 ts/train.py:232 step:53K smpl:422K ep:2K epch:7.27 loss:0.014 grdn:0.210 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:13:22 ts/train.py:232 step:53K smpl:424K ep:2K epch:7.30 loss:0.015 grdn:0.219 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:14:48 ts/train.py:232 step:53K smpl:426K ep:2K epch:7.33 loss:0.014 grdn:0.211 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:16:15 ts/train.py:232 step:53K smpl:427K ep:2K epch:7.35 loss:0.015 grdn:0.214 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:17:41 ts/train.py:232 step:54K smpl:429K ep:2K epch:7.38 loss:0.014 grdn:0.210 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:19:07 ts/train.py:232 step:54K smpl:430K ep:2K epch:7.41 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:20:34 ts/train.py:232 step:54K smpl:432K ep:2K epch:7.44 loss:0.015 grdn:0.220 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:22:00 ts/train.py:232 step:54K smpl:434K ep:2K epch:7.46 loss:0.013 grdn:0.204 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:23:26 ts/train.py:232 step:54K smpl:435K ep:2K epch:7.49 loss:0.013 grdn:0.207 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:24:53 ts/train.py:232 step:55K smpl:437K ep:2K epch:7.52 loss:0.014 grdn:0.206 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:26:19 ts/train.py:232 step:55K smpl:438K ep:2K epch:7.55 loss:0.015 grdn:0.212 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:27:45 ts/train.py:232 step:55K smpl:440K ep:2K epch:7.57 loss:0.015 grdn:0.224 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 20:29:12 ts/train.py:232 step:55K smpl:442K ep:2K epch:7.60 loss:0.015 grdn:0.208 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 20:30:38 ts/train.py:232 step:55K smpl:443K ep:2K epch:7.63 loss:0.015 grdn:0.217 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 20:32:04 ts/train.py:232 step:56K smpl:445K ep:2K epch:7.66 loss:0.014 grdn:0.220 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 20:33:30 ts/train.py:232 step:56K smpl:446K ep:2K epch:7.68 loss:0.015 grdn:0.213 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 20:34:58 ts/train.py:232 step:56K smpl:448K ep:2K epch:7.71 loss:0.015 grdn:0.215 lr:9.9e-05 updt_s:0.428 data_s:0.010 +INFO 2025-05-05 20:36:24 ts/train.py:232 step:56K smpl:450K ep:2K epch:7.74 loss:0.015 grdn:0.214 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 20:37:50 ts/train.py:232 step:56K smpl:451K ep:2K epch:7.77 loss:0.015 grdn:0.212 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 20:39:17 ts/train.py:232 step:57K smpl:453K ep:2K epch:7.79 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:40:43 ts/train.py:232 step:57K smpl:454K ep:2K epch:7.82 loss:0.014 grdn:0.206 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:42:10 ts/train.py:232 step:57K smpl:456K ep:2K epch:7.85 loss:0.013 grdn:0.204 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:43:36 ts/train.py:232 step:57K smpl:458K ep:2K epch:7.88 loss:0.014 grdn:0.220 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:45:02 ts/train.py:232 step:57K smpl:459K ep:2K epch:7.90 loss:0.014 grdn:0.204 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:46:29 ts/train.py:232 step:58K smpl:461K ep:2K epch:7.93 loss:0.015 grdn:0.222 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:47:55 ts/train.py:232 step:58K smpl:462K ep:2K epch:7.96 loss:0.014 grdn:0.218 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:49:21 ts/train.py:232 step:58K smpl:464K ep:2K epch:7.99 loss:0.015 grdn:0.225 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:50:48 ts/train.py:232 step:58K smpl:466K ep:2K epch:8.02 loss:0.014 grdn:0.203 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:52:14 ts/train.py:232 step:58K smpl:467K ep:2K epch:8.04 loss:0.014 grdn:0.210 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:53:41 ts/train.py:232 step:59K smpl:469K ep:2K epch:8.07 loss:0.014 grdn:0.211 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:55:07 ts/train.py:232 step:59K smpl:470K ep:2K epch:8.10 loss:0.014 grdn:0.211 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:56:33 ts/train.py:232 step:59K smpl:472K ep:2K epch:8.13 loss:0.013 grdn:0.196 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 20:58:00 ts/train.py:232 step:59K smpl:474K ep:2K epch:8.15 loss:0.013 grdn:0.210 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 20:59:26 ts/train.py:232 step:59K smpl:475K ep:2K epch:8.18 loss:0.014 grdn:0.209 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:00:52 ts/train.py:232 step:60K smpl:477K ep:2K epch:8.21 loss:0.014 grdn:0.208 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:02:18 ts/train.py:232 step:60K smpl:478K ep:2K epch:8.24 loss:0.014 grdn:0.215 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 21:03:45 ts/train.py:232 step:60K smpl:480K ep:2K epch:8.26 loss:0.014 grdn:0.207 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 21:03:45 ts/train.py:241 Checkpoint policy after step 60000 +INFO 2025-05-05 21:05:14 ts/train.py:232 step:60K smpl:482K ep:2K epch:8.29 loss:0.016 grdn:0.230 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:06:40 ts/train.py:232 step:60K smpl:483K ep:2K epch:8.32 loss:0.014 grdn:0.220 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:08:06 ts/train.py:232 step:61K smpl:485K ep:3K epch:8.35 loss:0.013 grdn:0.199 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:09:33 ts/train.py:232 step:61K smpl:486K ep:3K epch:8.37 loss:0.015 grdn:0.210 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:10:59 ts/train.py:232 step:61K smpl:488K ep:3K epch:8.40 loss:0.013 grdn:0.202 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:12:25 ts/train.py:232 step:61K smpl:490K ep:3K epch:8.43 loss:0.014 grdn:0.220 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:13:51 ts/train.py:232 step:61K smpl:491K ep:3K epch:8.46 loss:0.013 grdn:0.205 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:15:17 ts/train.py:232 step:62K smpl:493K ep:3K epch:8.48 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:16:43 ts/train.py:232 step:62K smpl:494K ep:3K epch:8.51 loss:0.014 grdn:0.214 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:18:09 ts/train.py:232 step:62K smpl:496K ep:3K epch:8.54 loss:0.013 grdn:0.208 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:19:36 ts/train.py:232 step:62K smpl:498K ep:3K epch:8.57 loss:0.013 grdn:0.204 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:21:02 ts/train.py:232 step:62K smpl:499K ep:3K epch:8.59 loss:0.014 grdn:0.214 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:22:28 ts/train.py:232 step:63K smpl:501K ep:3K epch:8.62 loss:0.012 grdn:0.195 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:23:54 ts/train.py:232 step:63K smpl:502K ep:3K epch:8.65 loss:0.012 grdn:0.202 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:25:21 ts/train.py:232 step:63K smpl:504K ep:3K epch:8.68 loss:0.014 grdn:0.217 lr:9.9e-05 updt_s:0.428 data_s:0.009 +INFO 2025-05-05 21:26:48 ts/train.py:232 step:63K smpl:506K ep:3K epch:8.70 loss:0.013 grdn:0.210 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:28:14 ts/train.py:232 step:63K smpl:507K ep:3K epch:8.73 loss:0.013 grdn:0.201 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:29:40 ts/train.py:232 step:64K smpl:509K ep:3K epch:8.76 loss:0.013 grdn:0.207 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:31:06 ts/train.py:232 step:64K smpl:510K ep:3K epch:8.79 loss:0.015 grdn:0.218 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:32:32 ts/train.py:232 step:64K smpl:512K ep:3K epch:8.81 loss:0.014 grdn:0.205 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:33:58 ts/train.py:232 step:64K smpl:514K ep:3K epch:8.84 loss:0.013 grdn:0.205 lr:9.9e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 21:35:24 ts/train.py:232 step:64K smpl:515K ep:3K epch:8.87 loss:0.012 grdn:0.195 lr:9.9e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 21:36:50 ts/train.py:232 step:65K smpl:517K ep:3K epch:8.90 loss:0.012 grdn:0.195 lr:9.9e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 21:38:16 ts/train.py:232 step:65K smpl:518K ep:3K epch:8.92 loss:0.013 grdn:0.213 lr:9.9e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 21:39:42 ts/train.py:232 step:65K smpl:520K ep:3K epch:8.95 loss:0.014 grdn:0.215 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:41:08 ts/train.py:232 step:65K smpl:522K ep:3K epch:8.98 loss:0.014 grdn:0.218 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:42:34 ts/train.py:232 step:65K smpl:523K ep:3K epch:9.01 loss:0.014 grdn:0.204 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:44:00 ts/train.py:232 step:66K smpl:525K ep:3K epch:9.03 loss:0.014 grdn:0.221 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:45:27 ts/train.py:232 step:66K smpl:526K ep:3K epch:9.06 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:46:53 ts/train.py:232 step:66K smpl:528K ep:3K epch:9.09 loss:0.013 grdn:0.202 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:48:19 ts/train.py:232 step:66K smpl:530K ep:3K epch:9.12 loss:0.013 grdn:0.209 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 21:49:45 ts/train.py:232 step:66K smpl:531K ep:3K epch:9.14 loss:0.014 grdn:0.217 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:51:11 ts/train.py:232 step:67K smpl:533K ep:3K epch:9.17 loss:0.013 grdn:0.209 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:52:38 ts/train.py:232 step:67K smpl:534K ep:3K epch:9.20 loss:0.014 grdn:0.201 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:54:04 ts/train.py:232 step:67K smpl:536K ep:3K epch:9.23 loss:0.013 grdn:0.204 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:55:30 ts/train.py:232 step:67K smpl:538K ep:3K epch:9.25 loss:0.012 grdn:0.191 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:56:56 ts/train.py:232 step:67K smpl:539K ep:3K epch:9.28 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:58:22 ts/train.py:232 step:68K smpl:541K ep:3K epch:9.31 loss:0.013 grdn:0.203 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 21:59:49 ts/train.py:232 step:68K smpl:542K ep:3K epch:9.34 loss:0.013 grdn:0.196 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 22:01:15 ts/train.py:232 step:68K smpl:544K ep:3K epch:9.36 loss:0.013 grdn:0.207 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 22:02:41 ts/train.py:232 step:68K smpl:546K ep:3K epch:9.39 loss:0.013 grdn:0.208 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 22:04:07 ts/train.py:232 step:68K smpl:547K ep:3K epch:9.42 loss:0.013 grdn:0.205 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 22:05:34 ts/train.py:232 step:69K smpl:549K ep:3K epch:9.45 loss:0.013 grdn:0.210 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 22:07:00 ts/train.py:232 step:69K smpl:550K ep:3K epch:9.47 loss:0.014 grdn:0.210 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 22:08:26 ts/train.py:232 step:69K smpl:552K ep:3K epch:9.50 loss:0.015 grdn:0.218 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 22:09:53 ts/train.py:232 step:69K smpl:554K ep:3K epch:9.53 loss:0.014 grdn:0.210 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 22:11:19 ts/train.py:232 step:69K smpl:555K ep:3K epch:9.56 loss:0.012 grdn:0.196 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 22:12:45 ts/train.py:232 step:70K smpl:557K ep:3K epch:9.59 loss:0.013 grdn:0.206 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 22:14:11 ts/train.py:232 step:70K smpl:558K ep:3K epch:9.61 loss:0.012 grdn:0.196 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 22:15:39 ts/train.py:232 step:70K smpl:560K ep:3K epch:9.64 loss:0.013 grdn:0.207 lr:9.9e-05 updt_s:0.429 data_s:0.009 +INFO 2025-05-05 22:17:05 ts/train.py:232 step:70K smpl:562K ep:3K epch:9.67 loss:0.013 grdn:0.197 lr:9.9e-05 updt_s:0.430 data_s:0.001 +INFO 2025-05-05 22:18:31 ts/train.py:232 step:70K smpl:563K ep:3K epch:9.70 loss:0.014 grdn:0.207 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 22:19:58 ts/train.py:232 step:71K smpl:565K ep:3K epch:9.72 loss:0.012 grdn:0.192 lr:9.9e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 22:21:24 ts/train.py:232 step:71K smpl:566K ep:3K epch:9.75 loss:0.012 grdn:0.204 lr:9.9e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:22:50 ts/train.py:232 step:71K smpl:568K ep:3K epch:9.78 loss:0.012 grdn:0.204 lr:9.8e-05 updt_s:0.429 data_s:0.001 +INFO 2025-05-05 22:24:16 ts/train.py:232 step:71K smpl:570K ep:3K epch:9.81 loss:0.013 grdn:0.213 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:25:42 ts/train.py:232 step:71K smpl:571K ep:3K epch:9.83 loss:0.013 grdn:0.212 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:27:08 ts/train.py:232 step:72K smpl:573K ep:3K epch:9.86 loss:0.013 grdn:0.211 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:28:33 ts/train.py:232 step:72K smpl:574K ep:3K epch:9.89 loss:0.014 grdn:0.214 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 22:29:59 ts/train.py:232 step:72K smpl:576K ep:3K epch:9.92 loss:0.012 grdn:0.199 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:31:25 ts/train.py:232 step:72K smpl:578K ep:3K epch:9.94 loss:0.013 grdn:0.211 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:32:51 ts/train.py:232 step:72K smpl:579K ep:3K epch:9.97 loss:0.013 grdn:0.199 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:34:17 ts/train.py:232 step:73K smpl:581K ep:3K epch:10.00 loss:0.014 grdn:0.214 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:35:43 ts/train.py:232 step:73K smpl:582K ep:3K epch:10.03 loss:0.013 grdn:0.201 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:37:09 ts/train.py:232 step:73K smpl:584K ep:3K epch:10.05 loss:0.012 grdn:0.198 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:38:35 ts/train.py:232 step:73K smpl:586K ep:3K epch:10.08 loss:0.011 grdn:0.198 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:40:01 ts/train.py:232 step:73K smpl:587K ep:3K epch:10.11 loss:0.011 grdn:0.195 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:41:27 ts/train.py:232 step:74K smpl:589K ep:3K epch:10.14 loss:0.013 grdn:0.217 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:42:53 ts/train.py:232 step:74K smpl:590K ep:3K epch:10.16 loss:0.012 grdn:0.196 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 22:44:19 ts/train.py:232 step:74K smpl:592K ep:3K epch:10.19 loss:0.013 grdn:0.198 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:45:45 ts/train.py:232 step:74K smpl:594K ep:3K epch:10.22 loss:0.013 grdn:0.210 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:47:11 ts/train.py:232 step:74K smpl:595K ep:3K epch:10.25 loss:0.012 grdn:0.193 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:48:37 ts/train.py:232 step:75K smpl:597K ep:3K epch:10.27 loss:0.013 grdn:0.210 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:50:02 ts/train.py:232 step:75K smpl:598K ep:3K epch:10.30 loss:0.013 grdn:0.206 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:51:28 ts/train.py:232 step:75K smpl:600K ep:3K epch:10.33 loss:0.013 grdn:0.217 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:52:54 ts/train.py:232 step:75K smpl:602K ep:3K epch:10.36 loss:0.012 grdn:0.207 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:54:20 ts/train.py:232 step:75K smpl:603K ep:3K epch:10.38 loss:0.014 grdn:0.217 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:55:46 ts/train.py:232 step:76K smpl:605K ep:3K epch:10.41 loss:0.012 grdn:0.208 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:57:12 ts/train.py:232 step:76K smpl:606K ep:3K epch:10.44 loss:0.012 grdn:0.205 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 22:58:38 ts/train.py:232 step:76K smpl:608K ep:3K epch:10.47 loss:0.012 grdn:0.200 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:00:04 ts/train.py:232 step:76K smpl:610K ep:3K epch:10.49 loss:0.013 grdn:0.198 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:01:30 ts/train.py:232 step:76K smpl:611K ep:3K epch:10.52 loss:0.013 grdn:0.217 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:02:56 ts/train.py:232 step:77K smpl:613K ep:3K epch:10.55 loss:0.014 grdn:0.210 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:04:22 ts/train.py:232 step:77K smpl:614K ep:3K epch:10.58 loss:0.012 grdn:0.201 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:05:49 ts/train.py:232 step:77K smpl:616K ep:3K epch:10.60 loss:0.011 grdn:0.199 lr:9.8e-05 updt_s:0.427 data_s:0.007 +INFO 2025-05-05 23:07:15 ts/train.py:232 step:77K smpl:618K ep:3K epch:10.63 loss:0.012 grdn:0.205 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:08:40 ts/train.py:232 step:77K smpl:619K ep:3K epch:10.66 loss:0.013 grdn:0.216 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:10:06 ts/train.py:232 step:78K smpl:621K ep:3K epch:10.69 loss:0.013 grdn:0.216 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:11:32 ts/train.py:232 step:78K smpl:622K ep:3K epch:10.71 loss:0.013 grdn:0.203 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:12:58 ts/train.py:232 step:78K smpl:624K ep:3K epch:10.74 loss:0.012 grdn:0.198 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:14:25 ts/train.py:232 step:78K smpl:626K ep:3K epch:10.77 loss:0.013 grdn:0.207 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:15:50 ts/train.py:232 step:78K smpl:627K ep:3K epch:10.80 loss:0.012 grdn:0.197 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:17:16 ts/train.py:232 step:79K smpl:629K ep:3K epch:10.82 loss:0.012 grdn:0.203 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:18:42 ts/train.py:232 step:79K smpl:630K ep:3K epch:10.85 loss:0.011 grdn:0.202 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:20:08 ts/train.py:232 step:79K smpl:632K ep:3K epch:10.88 loss:0.012 grdn:0.198 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:21:34 ts/train.py:232 step:79K smpl:634K ep:3K epch:10.91 loss:0.012 grdn:0.199 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:23:00 ts/train.py:232 step:79K smpl:635K ep:3K epch:10.93 loss:0.013 grdn:0.213 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:24:26 ts/train.py:232 step:80K smpl:637K ep:3K epch:10.96 loss:0.013 grdn:0.206 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:25:52 ts/train.py:232 step:80K smpl:638K ep:3K epch:10.99 loss:0.013 grdn:0.215 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:27:18 ts/train.py:232 step:80K smpl:640K ep:3K epch:11.02 loss:0.014 grdn:0.218 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:28:44 ts/train.py:232 step:80K smpl:642K ep:3K epch:11.04 loss:0.012 grdn:0.197 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:30:10 ts/train.py:232 step:80K smpl:643K ep:3K epch:11.07 loss:0.013 grdn:0.204 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:31:36 ts/train.py:232 step:81K smpl:645K ep:3K epch:11.10 loss:0.012 grdn:0.209 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:33:02 ts/train.py:232 step:81K smpl:646K ep:3K epch:11.13 loss:0.012 grdn:0.205 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:34:28 ts/train.py:232 step:81K smpl:648K ep:3K epch:11.16 loss:0.013 grdn:0.209 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:35:53 ts/train.py:232 step:81K smpl:650K ep:3K epch:11.18 loss:0.013 grdn:0.209 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:37:19 ts/train.py:232 step:81K smpl:651K ep:3K epch:11.21 loss:0.011 grdn:0.186 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:38:45 ts/train.py:232 step:82K smpl:653K ep:3K epch:11.24 loss:0.013 grdn:0.197 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:40:11 ts/train.py:232 step:82K smpl:654K ep:3K epch:11.27 loss:0.013 grdn:0.206 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:41:37 ts/train.py:232 step:82K smpl:656K ep:3K epch:11.29 loss:0.011 grdn:0.186 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:43:03 ts/train.py:232 step:82K smpl:658K ep:3K epch:11.32 loss:0.011 grdn:0.193 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:44:29 ts/train.py:232 step:82K smpl:659K ep:3K epch:11.35 loss:0.013 grdn:0.214 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:45:55 ts/train.py:232 step:83K smpl:661K ep:3K epch:11.38 loss:0.012 grdn:0.206 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:47:20 ts/train.py:232 step:83K smpl:662K ep:3K epch:11.40 loss:0.013 grdn:0.206 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:48:46 ts/train.py:232 step:83K smpl:664K ep:3K epch:11.43 loss:0.012 grdn:0.196 lr:9.8e-05 updt_s:0.428 data_s:0.001 +INFO 2025-05-05 23:50:12 ts/train.py:232 step:83K smpl:666K ep:3K epch:11.46 loss:0.013 grdn:0.210 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:51:38 ts/train.py:232 step:83K smpl:667K ep:3K epch:11.49 loss:0.012 grdn:0.200 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:53:04 ts/train.py:232 step:84K smpl:669K ep:3K epch:11.51 loss:0.013 grdn:0.207 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:54:29 ts/train.py:232 step:84K smpl:670K ep:3K epch:11.54 loss:0.012 grdn:0.201 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:55:56 ts/train.py:232 step:84K smpl:672K ep:3K epch:11.57 loss:0.014 grdn:0.218 lr:9.8e-05 updt_s:0.426 data_s:0.008 +INFO 2025-05-05 23:57:22 ts/train.py:232 step:84K smpl:674K ep:3K epch:11.60 loss:0.013 grdn:0.215 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-05 23:58:48 ts/train.py:232 step:84K smpl:675K ep:3K epch:11.62 loss:0.011 grdn:0.198 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:00:13 ts/train.py:232 step:85K smpl:677K ep:3K epch:11.65 loss:0.012 grdn:0.195 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:01:39 ts/train.py:232 step:85K smpl:678K ep:4K epch:11.68 loss:0.011 grdn:0.200 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:03:05 ts/train.py:232 step:85K smpl:680K ep:4K epch:11.71 loss:0.012 grdn:0.199 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:04:31 ts/train.py:232 step:85K smpl:682K ep:4K epch:11.73 loss:0.012 grdn:0.208 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:05:56 ts/train.py:232 step:85K smpl:683K ep:4K epch:11.76 loss:0.012 grdn:0.204 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:07:22 ts/train.py:232 step:86K smpl:685K ep:4K epch:11.79 loss:0.012 grdn:0.200 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:08:47 ts/train.py:232 step:86K smpl:686K ep:4K epch:11.82 loss:0.013 grdn:0.212 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:10:13 ts/train.py:232 step:86K smpl:688K ep:4K epch:11.84 loss:0.012 grdn:0.206 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:11:39 ts/train.py:232 step:86K smpl:690K ep:4K epch:11.87 loss:0.013 grdn:0.219 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:13:04 ts/train.py:232 step:86K smpl:691K ep:4K epch:11.90 loss:0.013 grdn:0.211 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:14:30 ts/train.py:232 step:87K smpl:693K ep:4K epch:11.93 loss:0.011 grdn:0.191 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:15:55 ts/train.py:232 step:87K smpl:694K ep:4K epch:11.95 loss:0.012 grdn:0.207 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:17:21 ts/train.py:232 step:87K smpl:696K ep:4K epch:11.98 loss:0.013 grdn:0.214 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:18:46 ts/train.py:232 step:87K smpl:698K ep:4K epch:12.01 loss:0.011 grdn:0.193 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:20:12 ts/train.py:232 step:87K smpl:699K ep:4K epch:12.04 loss:0.012 grdn:0.211 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:21:38 ts/train.py:232 step:88K smpl:701K ep:4K epch:12.06 loss:0.012 grdn:0.194 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:23:03 ts/train.py:232 step:88K smpl:702K ep:4K epch:12.09 loss:0.013 grdn:0.204 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:24:29 ts/train.py:232 step:88K smpl:704K ep:4K epch:12.12 loss:0.011 grdn:0.199 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:25:55 ts/train.py:232 step:88K smpl:706K ep:4K epch:12.15 loss:0.013 grdn:0.212 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:27:20 ts/train.py:232 step:88K smpl:707K ep:4K epch:12.17 loss:0.012 grdn:0.203 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:28:46 ts/train.py:232 step:89K smpl:709K ep:4K epch:12.20 loss:0.012 grdn:0.205 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:30:11 ts/train.py:232 step:89K smpl:710K ep:4K epch:12.23 loss:0.011 grdn:0.198 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:31:37 ts/train.py:232 step:89K smpl:712K ep:4K epch:12.26 loss:0.012 grdn:0.199 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:33:02 ts/train.py:232 step:89K smpl:714K ep:4K epch:12.28 loss:0.012 grdn:0.199 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:34:28 ts/train.py:232 step:89K smpl:715K ep:4K epch:12.31 loss:0.011 grdn:0.195 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:35:53 ts/train.py:232 step:90K smpl:717K ep:4K epch:12.34 loss:0.012 grdn:0.202 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:37:19 ts/train.py:232 step:90K smpl:718K ep:4K epch:12.37 loss:0.012 grdn:0.202 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:38:45 ts/train.py:232 step:90K smpl:720K ep:4K epch:12.39 loss:0.012 grdn:0.197 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:38:45 ts/train.py:241 Checkpoint policy after step 90000 +INFO 2025-05-06 00:40:13 ts/train.py:232 step:90K smpl:722K ep:4K epch:12.42 loss:0.013 grdn:0.212 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:41:39 ts/train.py:232 step:90K smpl:723K ep:4K epch:12.45 loss:0.012 grdn:0.203 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:43:05 ts/train.py:232 step:91K smpl:725K ep:4K epch:12.48 loss:0.012 grdn:0.202 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:44:30 ts/train.py:232 step:91K smpl:726K ep:4K epch:12.50 loss:0.011 grdn:0.194 lr:9.8e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:45:57 ts/train.py:232 step:91K smpl:728K ep:4K epch:12.53 loss:0.012 grdn:0.199 lr:9.8e-05 updt_s:0.426 data_s:0.009 +INFO 2025-05-06 00:47:23 ts/train.py:232 step:91K smpl:730K ep:4K epch:12.56 loss:0.012 grdn:0.202 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:48:49 ts/train.py:232 step:91K smpl:731K ep:4K epch:12.59 loss:0.012 grdn:0.207 lr:9.8e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:50:14 ts/train.py:232 step:92K smpl:733K ep:4K epch:12.61 loss:0.012 grdn:0.209 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:51:40 ts/train.py:232 step:92K smpl:734K ep:4K epch:12.64 loss:0.012 grdn:0.202 lr:9.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:53:06 ts/train.py:232 step:92K smpl:736K ep:4K epch:12.67 loss:0.012 grdn:0.197 lr:9.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 00:54:31 ts/train.py:232 step:92K smpl:738K ep:4K epch:12.70 loss:0.011 grdn:0.190 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:55:57 ts/train.py:232 step:92K smpl:739K ep:4K epch:12.73 loss:0.011 grdn:0.193 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:57:22 ts/train.py:232 step:93K smpl:741K ep:4K epch:12.75 loss:0.012 grdn:0.202 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 00:58:48 ts/train.py:232 step:93K smpl:742K ep:4K epch:12.78 loss:0.011 grdn:0.199 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:00:13 ts/train.py:232 step:93K smpl:744K ep:4K epch:12.81 loss:0.012 grdn:0.202 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:01:39 ts/train.py:232 step:93K smpl:746K ep:4K epch:12.84 loss:0.011 grdn:0.194 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:03:04 ts/train.py:232 step:93K smpl:747K ep:4K epch:12.86 loss:0.011 grdn:0.197 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:04:29 ts/train.py:232 step:94K smpl:749K ep:4K epch:12.89 loss:0.011 grdn:0.197 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:05:55 ts/train.py:232 step:94K smpl:750K ep:4K epch:12.92 loss:0.011 grdn:0.191 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:07:20 ts/train.py:232 step:94K smpl:752K ep:4K epch:12.95 loss:0.011 grdn:0.198 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:08:46 ts/train.py:232 step:94K smpl:754K ep:4K epch:12.97 loss:0.012 grdn:0.212 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:10:11 ts/train.py:232 step:94K smpl:755K ep:4K epch:13.00 loss:0.012 grdn:0.209 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:11:37 ts/train.py:232 step:95K smpl:757K ep:4K epch:13.03 loss:0.012 grdn:0.201 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:13:02 ts/train.py:232 step:95K smpl:758K ep:4K epch:13.06 loss:0.011 grdn:0.201 lr:9.7e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 01:14:28 ts/train.py:232 step:95K smpl:760K ep:4K epch:13.08 loss:0.012 grdn:0.208 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:15:53 ts/train.py:232 step:95K smpl:762K ep:4K epch:13.11 loss:0.012 grdn:0.194 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:17:19 ts/train.py:232 step:95K smpl:763K ep:4K epch:13.14 loss:0.011 grdn:0.192 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:18:45 ts/train.py:232 step:96K smpl:765K ep:4K epch:13.17 loss:0.012 grdn:0.198 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:20:10 ts/train.py:232 step:96K smpl:766K ep:4K epch:13.19 loss:0.013 grdn:0.209 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:21:36 ts/train.py:232 step:96K smpl:768K ep:4K epch:13.22 loss:0.011 grdn:0.188 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:23:01 ts/train.py:232 step:96K smpl:770K ep:4K epch:13.25 loss:0.012 grdn:0.207 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:24:27 ts/train.py:232 step:96K smpl:771K ep:4K epch:13.28 loss:0.012 grdn:0.198 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:25:52 ts/train.py:232 step:97K smpl:773K ep:4K epch:13.30 loss:0.011 grdn:0.189 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:27:18 ts/train.py:232 step:97K smpl:774K ep:4K epch:13.33 loss:0.012 grdn:0.207 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:28:43 ts/train.py:232 step:97K smpl:776K ep:4K epch:13.36 loss:0.011 grdn:0.187 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:30:09 ts/train.py:232 step:97K smpl:778K ep:4K epch:13.39 loss:0.011 grdn:0.196 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:31:34 ts/train.py:232 step:97K smpl:779K ep:4K epch:13.41 loss:0.012 grdn:0.200 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:33:00 ts/train.py:232 step:98K smpl:781K ep:4K epch:13.44 loss:0.012 grdn:0.205 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:34:25 ts/train.py:232 step:98K smpl:782K ep:4K epch:13.47 loss:0.011 grdn:0.191 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:35:52 ts/train.py:232 step:98K smpl:784K ep:4K epch:13.50 loss:0.011 grdn:0.192 lr:9.7e-05 updt_s:0.425 data_s:0.008 +INFO 2025-05-06 01:37:18 ts/train.py:232 step:98K smpl:786K ep:4K epch:13.52 loss:0.012 grdn:0.207 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:38:43 ts/train.py:232 step:98K smpl:787K ep:4K epch:13.55 loss:0.012 grdn:0.196 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:40:09 ts/train.py:232 step:99K smpl:789K ep:4K epch:13.58 loss:0.011 grdn:0.212 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:41:34 ts/train.py:232 step:99K smpl:790K ep:4K epch:13.61 loss:0.012 grdn:0.201 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:43:00 ts/train.py:232 step:99K smpl:792K ep:4K epch:13.63 loss:0.012 grdn:0.200 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:44:25 ts/train.py:232 step:99K smpl:794K ep:4K epch:13.66 loss:0.012 grdn:0.201 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:45:50 ts/train.py:232 step:99K smpl:795K ep:4K epch:13.69 loss:0.012 grdn:0.199 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:47:16 ts/train.py:232 step:100K smpl:797K ep:4K epch:13.72 loss:0.011 grdn:0.189 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:48:41 ts/train.py:232 step:100K smpl:798K ep:4K epch:13.74 loss:0.012 grdn:0.211 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:50:07 ts/train.py:232 step:100K smpl:800K ep:4K epch:13.77 loss:0.012 grdn:0.203 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:51:32 ts/train.py:232 step:100K smpl:802K ep:4K epch:13.80 loss:0.011 grdn:0.190 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:52:58 ts/train.py:232 step:100K smpl:803K ep:4K epch:13.83 loss:0.012 grdn:0.199 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:54:23 ts/train.py:232 step:101K smpl:805K ep:4K epch:13.85 loss:0.011 grdn:0.196 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 01:55:48 ts/train.py:232 step:101K smpl:806K ep:4K epch:13.88 loss:0.012 grdn:0.210 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:57:14 ts/train.py:232 step:101K smpl:808K ep:4K epch:13.91 loss:0.011 grdn:0.192 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 01:58:39 ts/train.py:232 step:101K smpl:810K ep:4K epch:13.94 loss:0.011 grdn:0.191 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:00:05 ts/train.py:232 step:101K smpl:811K ep:4K epch:13.96 loss:0.012 grdn:0.206 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:01:30 ts/train.py:232 step:102K smpl:813K ep:4K epch:13.99 loss:0.012 grdn:0.207 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:02:56 ts/train.py:232 step:102K smpl:814K ep:4K epch:14.02 loss:0.013 grdn:0.205 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:04:21 ts/train.py:232 step:102K smpl:816K ep:4K epch:14.05 loss:0.010 grdn:0.188 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:05:46 ts/train.py:232 step:102K smpl:818K ep:4K epch:14.07 loss:0.011 grdn:0.196 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:07:12 ts/train.py:232 step:102K smpl:819K ep:4K epch:14.10 loss:0.010 grdn:0.201 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:08:37 ts/train.py:232 step:103K smpl:821K ep:4K epch:14.13 loss:0.013 grdn:0.206 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:10:03 ts/train.py:232 step:103K smpl:822K ep:4K epch:14.16 loss:0.011 grdn:0.186 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:11:28 ts/train.py:232 step:103K smpl:824K ep:4K epch:14.18 loss:0.011 grdn:0.198 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:12:53 ts/train.py:232 step:103K smpl:826K ep:4K epch:14.21 loss:0.012 grdn:0.205 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:14:19 ts/train.py:232 step:103K smpl:827K ep:4K epch:14.24 loss:0.012 grdn:0.203 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:15:44 ts/train.py:232 step:104K smpl:829K ep:4K epch:14.27 loss:0.011 grdn:0.198 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:17:10 ts/train.py:232 step:104K smpl:830K ep:4K epch:14.30 loss:0.010 grdn:0.187 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:18:35 ts/train.py:232 step:104K smpl:832K ep:4K epch:14.32 loss:0.011 grdn:0.199 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:20:01 ts/train.py:232 step:104K smpl:834K ep:4K epch:14.35 loss:0.011 grdn:0.206 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:21:26 ts/train.py:232 step:104K smpl:835K ep:4K epch:14.38 loss:0.011 grdn:0.196 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:22:52 ts/train.py:232 step:105K smpl:837K ep:4K epch:14.41 loss:0.010 grdn:0.192 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:24:17 ts/train.py:232 step:105K smpl:838K ep:4K epch:14.43 loss:0.012 grdn:0.220 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:25:44 ts/train.py:232 step:105K smpl:840K ep:4K epch:14.46 loss:0.012 grdn:0.200 lr:9.7e-05 updt_s:0.426 data_s:0.009 +INFO 2025-05-06 02:27:09 ts/train.py:232 step:105K smpl:842K ep:4K epch:14.49 loss:0.011 grdn:0.200 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:28:35 ts/train.py:232 step:105K smpl:843K ep:4K epch:14.52 loss:0.012 grdn:0.208 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:30:00 ts/train.py:232 step:106K smpl:845K ep:4K epch:14.54 loss:0.011 grdn:0.196 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:31:26 ts/train.py:232 step:106K smpl:846K ep:4K epch:14.57 loss:0.011 grdn:0.201 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:32:51 ts/train.py:232 step:106K smpl:848K ep:4K epch:14.60 loss:0.012 grdn:0.209 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:34:17 ts/train.py:232 step:106K smpl:850K ep:4K epch:14.63 loss:0.011 grdn:0.195 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:35:42 ts/train.py:232 step:106K smpl:851K ep:4K epch:14.65 loss:0.011 grdn:0.203 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:37:07 ts/train.py:232 step:107K smpl:853K ep:4K epch:14.68 loss:0.011 grdn:0.197 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:38:32 ts/train.py:232 step:107K smpl:854K ep:4K epch:14.71 loss:0.012 grdn:0.209 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:39:58 ts/train.py:232 step:107K smpl:856K ep:4K epch:14.74 loss:0.012 grdn:0.210 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:41:23 ts/train.py:232 step:107K smpl:858K ep:4K epch:14.76 loss:0.011 grdn:0.206 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:42:49 ts/train.py:232 step:107K smpl:859K ep:4K epch:14.79 loss:0.012 grdn:0.197 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:44:14 ts/train.py:232 step:108K smpl:861K ep:4K epch:14.82 loss:0.012 grdn:0.203 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:45:39 ts/train.py:232 step:108K smpl:862K ep:4K epch:14.85 loss:0.011 grdn:0.197 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:47:05 ts/train.py:232 step:108K smpl:864K ep:4K epch:14.87 loss:0.011 grdn:0.199 lr:9.7e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:48:30 ts/train.py:232 step:108K smpl:866K ep:4K epch:14.90 loss:0.010 grdn:0.193 lr:9.7e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:49:55 ts/train.py:232 step:108K smpl:867K ep:4K epch:14.93 loss:0.011 grdn:0.204 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:51:21 ts/train.py:232 step:109K smpl:869K ep:4K epch:14.96 loss:0.012 grdn:0.199 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:52:46 ts/train.py:232 step:109K smpl:870K ep:4K epch:14.98 loss:0.010 grdn:0.189 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:54:12 ts/train.py:232 step:109K smpl:872K ep:5K epch:15.01 loss:0.011 grdn:0.196 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:55:37 ts/train.py:232 step:109K smpl:874K ep:5K epch:15.04 loss:0.011 grdn:0.198 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:57:02 ts/train.py:232 step:109K smpl:875K ep:5K epch:15.07 loss:0.010 grdn:0.192 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 02:58:28 ts/train.py:232 step:110K smpl:877K ep:5K epch:15.09 loss:0.011 grdn:0.195 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 02:59:53 ts/train.py:232 step:110K smpl:878K ep:5K epch:15.12 loss:0.011 grdn:0.201 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:01:18 ts/train.py:232 step:110K smpl:880K ep:5K epch:15.15 loss:0.011 grdn:0.199 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:02:44 ts/train.py:232 step:110K smpl:882K ep:5K epch:15.18 loss:0.011 grdn:0.199 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:04:09 ts/train.py:232 step:110K smpl:883K ep:5K epch:15.20 loss:0.011 grdn:0.198 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:05:34 ts/train.py:232 step:111K smpl:885K ep:5K epch:15.23 loss:0.010 grdn:0.198 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:07:00 ts/train.py:232 step:111K smpl:886K ep:5K epch:15.26 loss:0.011 grdn:0.203 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:08:25 ts/train.py:232 step:111K smpl:888K ep:5K epch:15.29 loss:0.011 grdn:0.197 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:09:50 ts/train.py:232 step:111K smpl:890K ep:5K epch:15.31 loss:0.011 grdn:0.197 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:11:16 ts/train.py:232 step:111K smpl:891K ep:5K epch:15.34 loss:0.011 grdn:0.194 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:12:41 ts/train.py:232 step:112K smpl:893K ep:5K epch:15.37 loss:0.011 grdn:0.196 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:14:07 ts/train.py:232 step:112K smpl:894K ep:5K epch:15.40 loss:0.011 grdn:0.191 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:15:34 ts/train.py:232 step:112K smpl:896K ep:5K epch:15.42 loss:0.009 grdn:0.182 lr:9.6e-05 updt_s:0.424 data_s:0.010 +INFO 2025-05-06 03:16:59 ts/train.py:232 step:112K smpl:898K ep:5K epch:15.45 loss:0.012 grdn:0.209 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:18:24 ts/train.py:232 step:112K smpl:899K ep:5K epch:15.48 loss:0.011 grdn:0.190 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:19:50 ts/train.py:232 step:113K smpl:901K ep:5K epch:15.51 loss:0.010 grdn:0.187 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:21:15 ts/train.py:232 step:113K smpl:902K ep:5K epch:15.53 loss:0.010 grdn:0.190 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:22:40 ts/train.py:232 step:113K smpl:904K ep:5K epch:15.56 loss:0.012 grdn:0.201 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 03:24:06 ts/train.py:232 step:113K smpl:906K ep:5K epch:15.59 loss:0.011 grdn:0.202 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 03:25:31 ts/train.py:232 step:113K smpl:907K ep:5K epch:15.62 loss:0.011 grdn:0.201 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:26:56 ts/train.py:232 step:114K smpl:909K ep:5K epch:15.64 loss:0.011 grdn:0.194 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:28:22 ts/train.py:232 step:114K smpl:910K ep:5K epch:15.67 loss:0.011 grdn:0.196 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:29:47 ts/train.py:232 step:114K smpl:912K ep:5K epch:15.70 loss:0.012 grdn:0.210 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:31:13 ts/train.py:232 step:114K smpl:914K ep:5K epch:15.73 loss:0.012 grdn:0.202 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:32:38 ts/train.py:232 step:114K smpl:915K ep:5K epch:15.75 loss:0.012 grdn:0.202 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:34:03 ts/train.py:232 step:115K smpl:917K ep:5K epch:15.78 loss:0.010 grdn:0.184 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 03:35:29 ts/train.py:232 step:115K smpl:918K ep:5K epch:15.81 loss:0.010 grdn:0.200 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:36:54 ts/train.py:232 step:115K smpl:920K ep:5K epch:15.84 loss:0.011 grdn:0.194 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:38:19 ts/train.py:232 step:115K smpl:922K ep:5K epch:15.87 loss:0.010 grdn:0.192 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:39:45 ts/train.py:232 step:115K smpl:923K ep:5K epch:15.89 loss:0.010 grdn:0.194 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 03:41:10 ts/train.py:232 step:116K smpl:925K ep:5K epch:15.92 loss:0.011 grdn:0.200 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 03:42:36 ts/train.py:232 step:116K smpl:926K ep:5K epch:15.95 loss:0.012 grdn:0.194 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:44:01 ts/train.py:232 step:116K smpl:928K ep:5K epch:15.98 loss:0.012 grdn:0.206 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 03:45:27 ts/train.py:232 step:116K smpl:930K ep:5K epch:16.00 loss:0.010 grdn:0.184 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 03:46:52 ts/train.py:232 step:116K smpl:931K ep:5K epch:16.03 loss:0.011 grdn:0.193 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 03:48:17 ts/train.py:232 step:117K smpl:933K ep:5K epch:16.06 loss:0.011 grdn:0.199 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:49:43 ts/train.py:232 step:117K smpl:934K ep:5K epch:16.09 loss:0.010 grdn:0.197 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:51:08 ts/train.py:232 step:117K smpl:936K ep:5K epch:16.11 loss:0.011 grdn:0.196 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:52:34 ts/train.py:232 step:117K smpl:938K ep:5K epch:16.14 loss:0.011 grdn:0.203 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:53:59 ts/train.py:232 step:117K smpl:939K ep:5K epch:16.17 loss:0.011 grdn:0.195 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:55:24 ts/train.py:232 step:118K smpl:941K ep:5K epch:16.20 loss:0.011 grdn:0.195 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:56:50 ts/train.py:232 step:118K smpl:942K ep:5K epch:16.22 loss:0.013 grdn:0.207 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:58:15 ts/train.py:232 step:118K smpl:944K ep:5K epch:16.25 loss:0.011 grdn:0.201 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 03:59:40 ts/train.py:232 step:118K smpl:946K ep:5K epch:16.28 loss:0.012 grdn:0.216 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:01:06 ts/train.py:232 step:118K smpl:947K ep:5K epch:16.31 loss:0.011 grdn:0.197 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:02:31 ts/train.py:232 step:119K smpl:949K ep:5K epch:16.33 loss:0.011 grdn:0.196 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:03:56 ts/train.py:232 step:119K smpl:950K ep:5K epch:16.36 loss:0.010 grdn:0.191 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:05:23 ts/train.py:232 step:119K smpl:952K ep:5K epch:16.39 loss:0.010 grdn:0.198 lr:9.6e-05 updt_s:0.425 data_s:0.008 +INFO 2025-05-06 04:06:48 ts/train.py:232 step:119K smpl:954K ep:5K epch:16.42 loss:0.010 grdn:0.193 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:08:14 ts/train.py:232 step:119K smpl:955K ep:5K epch:16.44 loss:0.011 grdn:0.198 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:09:39 ts/train.py:232 step:120K smpl:957K ep:5K epch:16.47 loss:0.011 grdn:0.200 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:11:04 ts/train.py:232 step:120K smpl:958K ep:5K epch:16.50 loss:0.010 grdn:0.194 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:12:29 ts/train.py:232 step:120K smpl:960K ep:5K epch:16.53 loss:0.012 grdn:0.208 lr:9.6e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 04:12:29 ts/train.py:241 Checkpoint policy after step 120000 +INFO 2025-05-06 04:13:58 ts/train.py:232 step:120K smpl:962K ep:5K epch:16.55 loss:0.010 grdn:0.199 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:15:23 ts/train.py:232 step:120K smpl:963K ep:5K epch:16.58 loss:0.011 grdn:0.199 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:16:49 ts/train.py:232 step:121K smpl:965K ep:5K epch:16.61 loss:0.010 grdn:0.188 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:18:14 ts/train.py:232 step:121K smpl:966K ep:5K epch:16.64 loss:0.011 grdn:0.199 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:19:39 ts/train.py:232 step:121K smpl:968K ep:5K epch:16.66 loss:0.010 grdn:0.185 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:21:05 ts/train.py:232 step:121K smpl:970K ep:5K epch:16.69 loss:0.011 grdn:0.203 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:22:30 ts/train.py:232 step:121K smpl:971K ep:5K epch:16.72 loss:0.012 grdn:0.200 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:23:56 ts/train.py:232 step:122K smpl:973K ep:5K epch:16.75 loss:0.011 grdn:0.203 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:25:21 ts/train.py:232 step:122K smpl:974K ep:5K epch:16.77 loss:0.011 grdn:0.201 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:26:47 ts/train.py:232 step:122K smpl:976K ep:5K epch:16.80 loss:0.011 grdn:0.200 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:28:12 ts/train.py:232 step:122K smpl:978K ep:5K epch:16.83 loss:0.011 grdn:0.205 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:29:37 ts/train.py:232 step:122K smpl:979K ep:5K epch:16.86 loss:0.010 grdn:0.187 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:31:03 ts/train.py:232 step:123K smpl:981K ep:5K epch:16.88 loss:0.011 grdn:0.207 lr:9.6e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:32:28 ts/train.py:232 step:123K smpl:982K ep:5K epch:16.91 loss:0.010 grdn:0.186 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:33:54 ts/train.py:232 step:123K smpl:984K ep:5K epch:16.94 loss:0.011 grdn:0.198 lr:9.6e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:35:19 ts/train.py:232 step:123K smpl:986K ep:5K epch:16.97 loss:0.011 grdn:0.200 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:36:45 ts/train.py:232 step:123K smpl:987K ep:5K epch:16.99 loss:0.011 grdn:0.200 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:38:10 ts/train.py:232 step:124K smpl:989K ep:5K epch:17.02 loss:0.010 grdn:0.194 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:39:35 ts/train.py:232 step:124K smpl:990K ep:5K epch:17.05 loss:0.010 grdn:0.194 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:41:01 ts/train.py:232 step:124K smpl:992K ep:5K epch:17.08 loss:0.011 grdn:0.204 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:42:26 ts/train.py:232 step:124K smpl:994K ep:5K epch:17.10 loss:0.011 grdn:0.205 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:43:52 ts/train.py:232 step:124K smpl:995K ep:5K epch:17.13 loss:0.011 grdn:0.199 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:45:17 ts/train.py:232 step:125K smpl:997K ep:5K epch:17.16 loss:0.010 grdn:0.191 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:46:43 ts/train.py:232 step:125K smpl:998K ep:5K epch:17.19 loss:0.011 grdn:0.193 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:48:08 ts/train.py:232 step:125K smpl:1M ep:5K epch:17.21 loss:0.011 grdn:0.205 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:49:34 ts/train.py:232 step:125K smpl:1M ep:5K epch:17.24 loss:0.010 grdn:0.192 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:50:59 ts/train.py:232 step:125K smpl:1M ep:5K epch:17.27 loss:0.011 grdn:0.193 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:52:25 ts/train.py:232 step:126K smpl:1M ep:5K epch:17.30 loss:0.012 grdn:0.208 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 04:53:50 ts/train.py:232 step:126K smpl:1M ep:5K epch:17.32 loss:0.011 grdn:0.206 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:55:17 ts/train.py:232 step:126K smpl:1M ep:5K epch:17.35 loss:0.010 grdn:0.187 lr:9.5e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-06 04:56:42 ts/train.py:232 step:126K smpl:1M ep:5K epch:17.38 loss:0.011 grdn:0.207 lr:9.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 04:58:08 ts/train.py:232 step:126K smpl:1M ep:5K epch:17.41 loss:0.009 grdn:0.193 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 04:59:33 ts/train.py:232 step:127K smpl:1M ep:5K epch:17.44 loss:0.011 grdn:0.196 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:00:59 ts/train.py:232 step:127K smpl:1M ep:5K epch:17.46 loss:0.010 grdn:0.202 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:02:24 ts/train.py:232 step:127K smpl:1M ep:5K epch:17.49 loss:0.011 grdn:0.196 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:03:49 ts/train.py:232 step:127K smpl:1M ep:5K epch:17.52 loss:0.010 grdn:0.189 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:05:15 ts/train.py:232 step:127K smpl:1M ep:5K epch:17.55 loss:0.010 grdn:0.193 lr:9.5e-05 updt_s:0.425 data_s:0.000 +INFO 2025-05-06 05:06:40 ts/train.py:232 step:128K smpl:1M ep:5K epch:17.57 loss:0.010 grdn:0.197 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:08:05 ts/train.py:232 step:128K smpl:1M ep:5K epch:17.60 loss:0.009 grdn:0.184 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:09:31 ts/train.py:232 step:128K smpl:1M ep:5K epch:17.63 loss:0.011 grdn:0.193 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:10:56 ts/train.py:232 step:128K smpl:1M ep:5K epch:17.66 loss:0.010 grdn:0.197 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:12:21 ts/train.py:232 step:128K smpl:1M ep:5K epch:17.68 loss:0.012 grdn:0.211 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:13:47 ts/train.py:232 step:129K smpl:1M ep:5K epch:17.71 loss:0.011 grdn:0.198 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:15:12 ts/train.py:232 step:129K smpl:1M ep:5K epch:17.74 loss:0.010 grdn:0.194 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:16:37 ts/train.py:232 step:129K smpl:1M ep:5K epch:17.77 loss:0.011 grdn:0.193 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:18:02 ts/train.py:232 step:129K smpl:1M ep:5K epch:17.79 loss:0.011 grdn:0.198 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:19:28 ts/train.py:232 step:129K smpl:1M ep:5K epch:17.82 loss:0.010 grdn:0.192 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:20:53 ts/train.py:232 step:130K smpl:1M ep:5K epch:17.85 loss:0.010 grdn:0.188 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:22:18 ts/train.py:232 step:130K smpl:1M ep:5K epch:17.88 loss:0.011 grdn:0.201 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:23:43 ts/train.py:232 step:130K smpl:1M ep:5K epch:17.90 loss:0.011 grdn:0.201 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:25:09 ts/train.py:232 step:130K smpl:1M ep:5K epch:17.93 loss:0.009 grdn:0.189 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:26:34 ts/train.py:232 step:130K smpl:1M ep:5K epch:17.96 loss:0.010 grdn:0.193 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:27:59 ts/train.py:232 step:131K smpl:1M ep:5K epch:17.99 loss:0.010 grdn:0.192 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:29:25 ts/train.py:232 step:131K smpl:1M ep:5K epch:18.01 loss:0.010 grdn:0.197 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:30:50 ts/train.py:232 step:131K smpl:1M ep:5K epch:18.04 loss:0.011 grdn:0.194 lr:9.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 05:32:15 ts/train.py:232 step:131K smpl:1M ep:5K epch:18.07 loss:0.010 grdn:0.187 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:33:40 ts/train.py:232 step:131K smpl:1M ep:5K epch:18.10 loss:0.011 grdn:0.198 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:35:05 ts/train.py:232 step:132K smpl:1M ep:5K epch:18.12 loss:0.010 grdn:0.191 lr:9.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 05:36:31 ts/train.py:232 step:132K smpl:1M ep:5K epch:18.15 loss:0.009 grdn:0.182 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:37:56 ts/train.py:232 step:132K smpl:1M ep:5K epch:18.18 loss:0.011 grdn:0.191 lr:9.5e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 05:39:21 ts/train.py:232 step:132K smpl:1M ep:5K epch:18.21 loss:0.010 grdn:0.197 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:40:47 ts/train.py:232 step:132K smpl:1M ep:5K epch:18.23 loss:0.010 grdn:0.193 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 05:42:12 ts/train.py:232 step:133K smpl:1M ep:5K epch:18.26 loss:0.010 grdn:0.199 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:43:37 ts/train.py:232 step:133K smpl:1M ep:5K epch:18.29 loss:0.009 grdn:0.184 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:45:04 ts/train.py:232 step:133K smpl:1M ep:5K epch:18.32 loss:0.010 grdn:0.197 lr:9.5e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-06 05:46:29 ts/train.py:232 step:133K smpl:1M ep:6K epch:18.34 loss:0.011 grdn:0.198 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:47:55 ts/train.py:232 step:133K smpl:1M ep:6K epch:18.37 loss:0.011 grdn:0.195 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 05:49:20 ts/train.py:232 step:134K smpl:1M ep:6K epch:18.40 loss:0.010 grdn:0.195 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:50:46 ts/train.py:232 step:134K smpl:1M ep:6K epch:18.43 loss:0.010 grdn:0.199 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:52:11 ts/train.py:232 step:134K smpl:1M ep:6K epch:18.45 loss:0.010 grdn:0.189 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:53:36 ts/train.py:232 step:134K smpl:1M ep:6K epch:18.48 loss:0.010 grdn:0.185 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:55:02 ts/train.py:232 step:134K smpl:1M ep:6K epch:18.51 loss:0.010 grdn:0.195 lr:9.5e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 05:56:27 ts/train.py:232 step:135K smpl:1M ep:6K epch:18.54 loss:0.011 grdn:0.204 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 05:57:53 ts/train.py:232 step:135K smpl:1M ep:6K epch:18.56 loss:0.010 grdn:0.191 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 05:59:18 ts/train.py:232 step:135K smpl:1M ep:6K epch:18.59 loss:0.010 grdn:0.193 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:00:44 ts/train.py:232 step:135K smpl:1M ep:6K epch:18.62 loss:0.009 grdn:0.190 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:02:09 ts/train.py:232 step:135K smpl:1M ep:6K epch:18.65 loss:0.011 grdn:0.198 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:03:35 ts/train.py:232 step:136K smpl:1M ep:6K epch:18.67 loss:0.010 grdn:0.187 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:05:00 ts/train.py:232 step:136K smpl:1M ep:6K epch:18.70 loss:0.010 grdn:0.195 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:06:26 ts/train.py:232 step:136K smpl:1M ep:6K epch:18.73 loss:0.010 grdn:0.191 lr:9.5e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:07:51 ts/train.py:232 step:136K smpl:1M ep:6K epch:18.76 loss:0.009 grdn:0.190 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:09:17 ts/train.py:232 step:136K smpl:1M ep:6K epch:18.78 loss:0.009 grdn:0.186 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:10:42 ts/train.py:232 step:137K smpl:1M ep:6K epch:18.81 loss:0.010 grdn:0.194 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:12:08 ts/train.py:232 step:137K smpl:1M ep:6K epch:18.84 loss:0.009 grdn:0.181 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:13:33 ts/train.py:232 step:137K smpl:1M ep:6K epch:18.87 loss:0.010 grdn:0.206 lr:9.4e-05 updt_s:0.427 data_s:0.001 +INFO 2025-05-06 06:14:59 ts/train.py:232 step:137K smpl:1M ep:6K epch:18.89 loss:0.010 grdn:0.204 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:16:24 ts/train.py:232 step:137K smpl:1M ep:6K epch:18.92 loss:0.011 grdn:0.202 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:17:50 ts/train.py:232 step:138K smpl:1M ep:6K epch:18.95 loss:0.010 grdn:0.195 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:19:15 ts/train.py:232 step:138K smpl:1M ep:6K epch:18.98 loss:0.010 grdn:0.191 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:20:41 ts/train.py:232 step:138K smpl:1M ep:6K epch:19.00 loss:0.010 grdn:0.190 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:22:06 ts/train.py:232 step:138K smpl:1M ep:6K epch:19.03 loss:0.010 grdn:0.204 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:23:32 ts/train.py:232 step:138K smpl:1M ep:6K epch:19.06 loss:0.010 grdn:0.193 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:24:57 ts/train.py:232 step:139K smpl:1M ep:6K epch:19.09 loss:0.011 grdn:0.196 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:26:22 ts/train.py:232 step:139K smpl:1M ep:6K epch:19.12 loss:0.010 grdn:0.199 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:27:48 ts/train.py:232 step:139K smpl:1M ep:6K epch:19.14 loss:0.010 grdn:0.187 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:29:13 ts/train.py:232 step:139K smpl:1M ep:6K epch:19.17 loss:0.011 grdn:0.206 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:30:39 ts/train.py:232 step:139K smpl:1M ep:6K epch:19.20 loss:0.010 grdn:0.195 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:32:04 ts/train.py:232 step:140K smpl:1M ep:6K epch:19.23 loss:0.011 grdn:0.200 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:33:30 ts/train.py:232 step:140K smpl:1M ep:6K epch:19.25 loss:0.010 grdn:0.189 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:34:57 ts/train.py:232 step:140K smpl:1M ep:6K epch:19.28 loss:0.009 grdn:0.185 lr:9.4e-05 updt_s:0.426 data_s:0.007 +INFO 2025-05-06 06:36:22 ts/train.py:232 step:140K smpl:1M ep:6K epch:19.31 loss:0.010 grdn:0.190 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:37:47 ts/train.py:232 step:140K smpl:1M ep:6K epch:19.34 loss:0.010 grdn:0.188 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:39:13 ts/train.py:232 step:141K smpl:1M ep:6K epch:19.36 loss:0.011 grdn:0.212 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:40:38 ts/train.py:232 step:141K smpl:1M ep:6K epch:19.39 loss:0.009 grdn:0.191 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:42:04 ts/train.py:232 step:141K smpl:1M ep:6K epch:19.42 loss:0.011 grdn:0.209 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:43:29 ts/train.py:232 step:141K smpl:1M ep:6K epch:19.45 loss:0.010 grdn:0.184 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:44:55 ts/train.py:232 step:141K smpl:1M ep:6K epch:19.47 loss:0.009 grdn:0.184 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:46:20 ts/train.py:232 step:142K smpl:1M ep:6K epch:19.50 loss:0.010 grdn:0.194 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:47:45 ts/train.py:232 step:142K smpl:1M ep:6K epch:19.53 loss:0.009 grdn:0.194 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:49:11 ts/train.py:232 step:142K smpl:1M ep:6K epch:19.56 loss:0.011 grdn:0.207 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:50:36 ts/train.py:232 step:142K smpl:1M ep:6K epch:19.58 loss:0.010 grdn:0.193 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:52:02 ts/train.py:232 step:142K smpl:1M ep:6K epch:19.61 loss:0.009 grdn:0.187 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 06:53:27 ts/train.py:232 step:143K smpl:1M ep:6K epch:19.64 loss:0.010 grdn:0.185 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:54:53 ts/train.py:232 step:143K smpl:1M ep:6K epch:19.67 loss:0.010 grdn:0.186 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:56:18 ts/train.py:232 step:143K smpl:1M ep:6K epch:19.69 loss:0.011 grdn:0.205 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:57:43 ts/train.py:232 step:143K smpl:1M ep:6K epch:19.72 loss:0.010 grdn:0.192 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 06:59:09 ts/train.py:232 step:143K smpl:1M ep:6K epch:19.75 loss:0.010 grdn:0.193 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:00:34 ts/train.py:232 step:144K smpl:1M ep:6K epch:19.78 loss:0.010 grdn:0.186 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:02:00 ts/train.py:232 step:144K smpl:1M ep:6K epch:19.80 loss:0.009 grdn:0.180 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:03:25 ts/train.py:232 step:144K smpl:1M ep:6K epch:19.83 loss:0.010 grdn:0.187 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:04:51 ts/train.py:232 step:144K smpl:1M ep:6K epch:19.86 loss:0.010 grdn:0.192 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:06:16 ts/train.py:232 step:144K smpl:1M ep:6K epch:19.89 loss:0.009 grdn:0.180 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:07:42 ts/train.py:232 step:145K smpl:1M ep:6K epch:19.91 loss:0.010 grdn:0.206 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:09:07 ts/train.py:232 step:145K smpl:1M ep:6K epch:19.94 loss:0.010 grdn:0.201 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:10:32 ts/train.py:232 step:145K smpl:1M ep:6K epch:19.97 loss:0.009 grdn:0.188 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:11:58 ts/train.py:232 step:145K smpl:1M ep:6K epch:20.00 loss:0.010 grdn:0.197 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:13:23 ts/train.py:232 step:145K smpl:1M ep:6K epch:20.02 loss:0.010 grdn:0.187 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:14:49 ts/train.py:232 step:146K smpl:1M ep:6K epch:20.05 loss:0.010 grdn:0.204 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:16:14 ts/train.py:232 step:146K smpl:1M ep:6K epch:20.08 loss:0.009 grdn:0.190 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:17:40 ts/train.py:232 step:146K smpl:1M ep:6K epch:20.11 loss:0.010 grdn:0.195 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:19:05 ts/train.py:232 step:146K smpl:1M ep:6K epch:20.13 loss:0.010 grdn:0.179 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:20:30 ts/train.py:232 step:146K smpl:1M ep:6K epch:20.16 loss:0.010 grdn:0.202 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:21:56 ts/train.py:232 step:147K smpl:1M ep:6K epch:20.19 loss:0.009 grdn:0.188 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:23:21 ts/train.py:232 step:147K smpl:1M ep:6K epch:20.22 loss:0.009 grdn:0.185 lr:9.4e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:24:48 ts/train.py:232 step:147K smpl:1M ep:6K epch:20.24 loss:0.010 grdn:0.194 lr:9.4e-05 updt_s:0.425 data_s:0.008 +INFO 2025-05-06 07:26:13 ts/train.py:232 step:147K smpl:1M ep:6K epch:20.27 loss:0.010 grdn:0.189 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:27:39 ts/train.py:232 step:147K smpl:1M ep:6K epch:20.30 loss:0.009 grdn:0.185 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:29:04 ts/train.py:232 step:148K smpl:1M ep:6K epch:20.33 loss:0.009 grdn:0.195 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:30:29 ts/train.py:232 step:148K smpl:1M ep:6K epch:20.35 loss:0.010 grdn:0.185 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:31:55 ts/train.py:232 step:148K smpl:1M ep:6K epch:20.38 loss:0.009 grdn:0.177 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:33:20 ts/train.py:232 step:148K smpl:1M ep:6K epch:20.41 loss:0.009 grdn:0.184 lr:9.4e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:34:45 ts/train.py:232 step:148K smpl:1M ep:6K epch:20.44 loss:0.009 grdn:0.181 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:36:11 ts/train.py:232 step:149K smpl:1M ep:6K epch:20.46 loss:0.011 grdn:0.199 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:37:36 ts/train.py:232 step:149K smpl:1M ep:6K epch:20.49 loss:0.010 grdn:0.198 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:39:01 ts/train.py:232 step:149K smpl:1M ep:6K epch:20.52 loss:0.010 grdn:0.203 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:40:27 ts/train.py:232 step:149K smpl:1M ep:6K epch:20.55 loss:0.010 grdn:0.200 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:41:52 ts/train.py:232 step:149K smpl:1M ep:6K epch:20.57 loss:0.009 grdn:0.180 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:43:17 ts/train.py:232 step:150K smpl:1M ep:6K epch:20.60 loss:0.010 grdn:0.192 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:44:43 ts/train.py:232 step:150K smpl:1M ep:6K epch:20.63 loss:0.008 grdn:0.178 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:46:08 ts/train.py:232 step:150K smpl:1M ep:6K epch:20.66 loss:0.009 grdn:0.185 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:46:08 ts/train.py:241 Checkpoint policy after step 150000 +INFO 2025-05-06 07:47:37 ts/train.py:232 step:150K smpl:1M ep:6K epch:20.69 loss:0.010 grdn:0.198 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:49:02 ts/train.py:232 step:150K smpl:1M ep:6K epch:20.71 loss:0.010 grdn:0.190 lr:9.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:50:28 ts/train.py:232 step:151K smpl:1M ep:6K epch:20.74 loss:0.010 grdn:0.194 lr:9.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:51:53 ts/train.py:232 step:151K smpl:1M ep:6K epch:20.77 loss:0.010 grdn:0.197 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:53:18 ts/train.py:232 step:151K smpl:1M ep:6K epch:20.80 loss:0.011 grdn:0.195 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:54:44 ts/train.py:232 step:151K smpl:1M ep:6K epch:20.82 loss:0.010 grdn:0.189 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:56:09 ts/train.py:232 step:151K smpl:1M ep:6K epch:20.85 loss:0.011 grdn:0.200 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 07:57:35 ts/train.py:232 step:152K smpl:1M ep:6K epch:20.88 loss:0.010 grdn:0.190 lr:9.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 07:59:00 ts/train.py:232 step:152K smpl:1M ep:6K epch:20.91 loss:0.009 grdn:0.183 lr:9.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 08:00:25 ts/train.py:232 step:152K smpl:1M ep:6K epch:20.93 loss:0.011 grdn:0.208 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:01:51 ts/train.py:232 step:152K smpl:1M ep:6K epch:20.96 loss:0.009 grdn:0.181 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:03:16 ts/train.py:232 step:152K smpl:1M ep:6K epch:20.99 loss:0.010 grdn:0.190 lr:9.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 08:04:42 ts/train.py:232 step:153K smpl:1M ep:6K epch:21.02 loss:0.009 grdn:0.185 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:06:07 ts/train.py:232 step:153K smpl:1M ep:6K epch:21.04 loss:0.010 grdn:0.191 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:07:32 ts/train.py:232 step:153K smpl:1M ep:6K epch:21.07 loss:0.010 grdn:0.194 lr:9.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 08:08:58 ts/train.py:232 step:153K smpl:1M ep:6K epch:21.10 loss:0.010 grdn:0.194 lr:9.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 08:10:23 ts/train.py:232 step:153K smpl:1M ep:6K epch:21.13 loss:0.010 grdn:0.204 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:11:49 ts/train.py:232 step:154K smpl:1M ep:6K epch:21.15 loss:0.009 grdn:0.192 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:13:14 ts/train.py:232 step:154K smpl:1M ep:6K epch:21.18 loss:0.011 grdn:0.208 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:14:40 ts/train.py:232 step:154K smpl:1M ep:6K epch:21.21 loss:0.010 grdn:0.195 lr:9.3e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-06 08:16:05 ts/train.py:232 step:154K smpl:1M ep:6K epch:21.24 loss:0.011 grdn:0.194 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:17:31 ts/train.py:232 step:154K smpl:1M ep:6K epch:21.26 loss:0.010 grdn:0.199 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:18:56 ts/train.py:232 step:155K smpl:1M ep:6K epch:21.29 loss:0.010 grdn:0.191 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:20:21 ts/train.py:232 step:155K smpl:1M ep:6K epch:21.32 loss:0.008 grdn:0.177 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:21:46 ts/train.py:232 step:155K smpl:1M ep:6K epch:21.35 loss:0.009 grdn:0.180 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:23:11 ts/train.py:232 step:155K smpl:1M ep:6K epch:21.37 loss:0.010 grdn:0.189 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:24:37 ts/train.py:232 step:155K smpl:1M ep:6K epch:21.40 loss:0.009 grdn:0.188 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:26:02 ts/train.py:232 step:156K smpl:1M ep:6K epch:21.43 loss:0.010 grdn:0.193 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:27:27 ts/train.py:232 step:156K smpl:1M ep:6K epch:21.46 loss:0.009 grdn:0.195 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:28:53 ts/train.py:232 step:156K smpl:1M ep:6K epch:21.48 loss:0.010 grdn:0.196 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:30:18 ts/train.py:232 step:156K smpl:1M ep:6K epch:21.51 loss:0.009 grdn:0.184 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:31:43 ts/train.py:232 step:156K smpl:1M ep:6K epch:21.54 loss:0.010 grdn:0.193 lr:9.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 08:33:09 ts/train.py:232 step:157K smpl:1M ep:6K epch:21.57 loss:0.010 grdn:0.201 lr:9.3e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 08:34:34 ts/train.py:232 step:157K smpl:1M ep:6K epch:21.59 loss:0.010 grdn:0.196 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:35:59 ts/train.py:232 step:157K smpl:1M ep:6K epch:21.62 loss:0.009 grdn:0.181 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:37:25 ts/train.py:232 step:157K smpl:1M ep:6K epch:21.65 loss:0.009 grdn:0.188 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:38:50 ts/train.py:232 step:157K smpl:1M ep:7K epch:21.68 loss:0.010 grdn:0.197 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:40:15 ts/train.py:232 step:158K smpl:1M ep:7K epch:21.70 loss:0.009 grdn:0.185 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:41:40 ts/train.py:232 step:158K smpl:1M ep:7K epch:21.73 loss:0.008 grdn:0.179 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:43:06 ts/train.py:232 step:158K smpl:1M ep:7K epch:21.76 loss:0.010 grdn:0.186 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:44:31 ts/train.py:232 step:158K smpl:1M ep:7K epch:21.79 loss:0.010 grdn:0.192 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:45:56 ts/train.py:232 step:158K smpl:1M ep:7K epch:21.81 loss:0.010 grdn:0.189 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:47:21 ts/train.py:232 step:159K smpl:1M ep:7K epch:21.84 loss:0.009 grdn:0.189 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:48:46 ts/train.py:232 step:159K smpl:1M ep:7K epch:21.87 loss:0.010 grdn:0.196 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:50:12 ts/train.py:232 step:159K smpl:1M ep:7K epch:21.90 loss:0.009 grdn:0.192 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:51:37 ts/train.py:232 step:159K smpl:1M ep:7K epch:21.92 loss:0.009 grdn:0.182 lr:9.3e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 08:53:02 ts/train.py:232 step:159K smpl:1M ep:7K epch:21.95 loss:0.010 grdn:0.192 lr:9.3e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:54:27 ts/train.py:232 step:160K smpl:1M ep:7K epch:21.98 loss:0.009 grdn:0.195 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:55:52 ts/train.py:232 step:160K smpl:1M ep:7K epch:22.01 loss:0.009 grdn:0.189 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:57:18 ts/train.py:232 step:160K smpl:1M ep:7K epch:22.03 loss:0.010 grdn:0.201 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 08:58:43 ts/train.py:232 step:160K smpl:1M ep:7K epch:22.06 loss:0.009 grdn:0.190 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 09:00:08 ts/train.py:232 step:160K smpl:1M ep:7K epch:22.09 loss:0.010 grdn:0.195 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:01:33 ts/train.py:232 step:161K smpl:1M ep:7K epch:22.12 loss:0.010 grdn:0.197 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:02:59 ts/train.py:232 step:161K smpl:1M ep:7K epch:22.14 loss:0.010 grdn:0.198 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:04:25 ts/train.py:232 step:161K smpl:1M ep:7K epch:22.17 loss:0.008 grdn:0.174 lr:9.2e-05 updt_s:0.424 data_s:0.007 +INFO 2025-05-06 09:05:50 ts/train.py:232 step:161K smpl:1M ep:7K epch:22.20 loss:0.009 grdn:0.185 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 09:07:16 ts/train.py:232 step:161K smpl:1M ep:7K epch:22.23 loss:0.009 grdn:0.191 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:08:41 ts/train.py:232 step:162K smpl:1M ep:7K epch:22.26 loss:0.010 grdn:0.188 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:10:06 ts/train.py:232 step:162K smpl:1M ep:7K epch:22.28 loss:0.011 grdn:0.199 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:11:31 ts/train.py:232 step:162K smpl:1M ep:7K epch:22.31 loss:0.011 grdn:0.199 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:12:56 ts/train.py:232 step:162K smpl:1M ep:7K epch:22.34 loss:0.009 grdn:0.189 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:14:22 ts/train.py:232 step:162K smpl:1M ep:7K epch:22.37 loss:0.010 grdn:0.200 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:15:47 ts/train.py:232 step:163K smpl:1M ep:7K epch:22.39 loss:0.009 grdn:0.184 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:17:12 ts/train.py:232 step:163K smpl:1M ep:7K epch:22.42 loss:0.010 grdn:0.197 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:18:38 ts/train.py:232 step:163K smpl:1M ep:7K epch:22.45 loss:0.010 grdn:0.188 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:20:03 ts/train.py:232 step:163K smpl:1M ep:7K epch:22.48 loss:0.010 grdn:0.190 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:21:28 ts/train.py:232 step:163K smpl:1M ep:7K epch:22.50 loss:0.010 grdn:0.198 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:22:54 ts/train.py:232 step:164K smpl:1M ep:7K epch:22.53 loss:0.009 grdn:0.182 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:24:19 ts/train.py:232 step:164K smpl:1M ep:7K epch:22.56 loss:0.009 grdn:0.186 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:25:44 ts/train.py:232 step:164K smpl:1M ep:7K epch:22.59 loss:0.010 grdn:0.194 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:27:09 ts/train.py:232 step:164K smpl:1M ep:7K epch:22.61 loss:0.008 grdn:0.180 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 09:28:35 ts/train.py:232 step:164K smpl:1M ep:7K epch:22.64 loss:0.010 grdn:0.196 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:30:00 ts/train.py:232 step:165K smpl:1M ep:7K epch:22.67 loss:0.009 grdn:0.195 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:31:25 ts/train.py:232 step:165K smpl:1M ep:7K epch:22.70 loss:0.009 grdn:0.187 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:32:51 ts/train.py:232 step:165K smpl:1M ep:7K epch:22.72 loss:0.008 grdn:0.182 lr:9.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 09:34:16 ts/train.py:232 step:165K smpl:1M ep:7K epch:22.75 loss:0.010 grdn:0.187 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:35:42 ts/train.py:232 step:165K smpl:1M ep:7K epch:22.78 loss:0.010 grdn:0.194 lr:9.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 09:37:07 ts/train.py:232 step:166K smpl:1M ep:7K epch:22.81 loss:0.009 grdn:0.190 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:38:32 ts/train.py:232 step:166K smpl:1M ep:7K epch:22.83 loss:0.009 grdn:0.179 lr:9.2e-05 updt_s:0.426 data_s:0.001 +INFO 2025-05-06 09:39:58 ts/train.py:232 step:166K smpl:1M ep:7K epch:22.86 loss:0.010 grdn:0.191 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:41:23 ts/train.py:232 step:166K smpl:1M ep:7K epch:22.89 loss:0.010 grdn:0.198 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:42:48 ts/train.py:232 step:166K smpl:1M ep:7K epch:22.92 loss:0.009 grdn:0.192 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:44:14 ts/train.py:232 step:167K smpl:1M ep:7K epch:22.94 loss:0.010 grdn:0.195 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:45:39 ts/train.py:232 step:167K smpl:1M ep:7K epch:22.97 loss:0.010 grdn:0.192 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:47:05 ts/train.py:232 step:167K smpl:1M ep:7K epch:23.00 loss:0.009 grdn:0.193 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:48:30 ts/train.py:232 step:167K smpl:1M ep:7K epch:23.03 loss:0.010 grdn:0.192 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:49:56 ts/train.py:232 step:167K smpl:1M ep:7K epch:23.05 loss:0.009 grdn:0.186 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:51:21 ts/train.py:232 step:168K smpl:1M ep:7K epch:23.08 loss:0.010 grdn:0.203 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:52:46 ts/train.py:232 step:168K smpl:1M ep:7K epch:23.11 loss:0.010 grdn:0.184 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:54:13 ts/train.py:232 step:168K smpl:1M ep:7K epch:23.14 loss:0.009 grdn:0.178 lr:9.2e-05 updt_s:0.425 data_s:0.009 +INFO 2025-05-06 09:55:39 ts/train.py:232 step:168K smpl:1M ep:7K epch:23.16 loss:0.009 grdn:0.186 lr:9.2e-05 updt_s:0.425 data_s:0.001 +INFO 2025-05-06 09:57:04 ts/train.py:232 step:168K smpl:1M ep:7K epch:23.19 loss:0.009 grdn:0.188 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 09:58:29 ts/train.py:232 step:169K smpl:1M ep:7K epch:23.22 loss:0.009 grdn:0.191 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 09:59:54 ts/train.py:232 step:169K smpl:1M ep:7K epch:23.25 loss:0.008 grdn:0.178 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:01:19 ts/train.py:232 step:169K smpl:1M ep:7K epch:23.27 loss:0.009 grdn:0.199 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:02:44 ts/train.py:232 step:169K smpl:1M ep:7K epch:23.30 loss:0.009 grdn:0.192 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:04:09 ts/train.py:232 step:169K smpl:1M ep:7K epch:23.33 loss:0.010 grdn:0.198 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:05:34 ts/train.py:232 step:170K smpl:1M ep:7K epch:23.36 loss:0.009 grdn:0.190 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:07:00 ts/train.py:232 step:170K smpl:1M ep:7K epch:23.38 loss:0.009 grdn:0.186 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:08:25 ts/train.py:232 step:170K smpl:1M ep:7K epch:23.41 loss:0.009 grdn:0.183 lr:9.2e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:09:50 ts/train.py:232 step:170K smpl:1M ep:7K epch:23.44 loss:0.009 grdn:0.181 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:11:15 ts/train.py:232 step:170K smpl:1M ep:7K epch:23.47 loss:0.009 grdn:0.191 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:12:40 ts/train.py:232 step:171K smpl:1M ep:7K epch:23.49 loss:0.009 grdn:0.179 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:14:05 ts/train.py:232 step:171K smpl:1M ep:7K epch:23.52 loss:0.009 grdn:0.187 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:15:30 ts/train.py:232 step:171K smpl:1M ep:7K epch:23.55 loss:0.010 grdn:0.196 lr:9.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 10:16:55 ts/train.py:232 step:171K smpl:1M ep:7K epch:23.58 loss:0.010 grdn:0.196 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:18:20 ts/train.py:232 step:171K smpl:1M ep:7K epch:23.60 loss:0.009 grdn:0.186 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:19:45 ts/train.py:232 step:172K smpl:1M ep:7K epch:23.63 loss:0.009 grdn:0.198 lr:9.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 10:21:10 ts/train.py:232 step:172K smpl:1M ep:7K epch:23.66 loss:0.009 grdn:0.191 lr:9.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 10:22:35 ts/train.py:232 step:172K smpl:1M ep:7K epch:23.69 loss:0.008 grdn:0.183 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:24:00 ts/train.py:232 step:172K smpl:1M ep:7K epch:23.71 loss:0.010 grdn:0.189 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:25:25 ts/train.py:232 step:172K smpl:1M ep:7K epch:23.74 loss:0.010 grdn:0.193 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:26:50 ts/train.py:232 step:173K smpl:1M ep:7K epch:23.77 loss:0.009 grdn:0.186 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:28:15 ts/train.py:232 step:173K smpl:1M ep:7K epch:23.80 loss:0.010 grdn:0.191 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:29:40 ts/train.py:232 step:173K smpl:1M ep:7K epch:23.83 loss:0.009 grdn:0.186 lr:9.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 10:31:06 ts/train.py:232 step:173K smpl:1M ep:7K epch:23.85 loss:0.009 grdn:0.187 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:32:31 ts/train.py:232 step:173K smpl:1M ep:7K epch:23.88 loss:0.010 grdn:0.191 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:33:56 ts/train.py:232 step:174K smpl:1M ep:7K epch:23.91 loss:0.009 grdn:0.184 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:35:21 ts/train.py:232 step:174K smpl:1M ep:7K epch:23.94 loss:0.009 grdn:0.196 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:36:46 ts/train.py:232 step:174K smpl:1M ep:7K epch:23.96 loss:0.008 grdn:0.177 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:38:11 ts/train.py:232 step:174K smpl:1M ep:7K epch:23.99 loss:0.009 grdn:0.191 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:39:36 ts/train.py:232 step:174K smpl:1M ep:7K epch:24.02 loss:0.009 grdn:0.188 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:41:01 ts/train.py:232 step:175K smpl:1M ep:7K epch:24.05 loss:0.009 grdn:0.187 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:42:27 ts/train.py:232 step:175K smpl:1M ep:7K epch:24.07 loss:0.009 grdn:0.189 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:43:53 ts/train.py:232 step:175K smpl:1M ep:7K epch:24.10 loss:0.008 grdn:0.181 lr:9.1e-05 updt_s:0.423 data_s:0.009 +INFO 2025-05-06 10:45:18 ts/train.py:232 step:175K smpl:1M ep:7K epch:24.13 loss:0.010 grdn:0.190 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:46:43 ts/train.py:232 step:175K smpl:1M ep:7K epch:24.16 loss:0.009 grdn:0.183 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:48:08 ts/train.py:232 step:176K smpl:1M ep:7K epch:24.18 loss:0.009 grdn:0.188 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:49:33 ts/train.py:232 step:176K smpl:1M ep:7K epch:24.21 loss:0.009 grdn:0.196 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:50:58 ts/train.py:232 step:176K smpl:1M ep:7K epch:24.24 loss:0.010 grdn:0.197 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:52:23 ts/train.py:232 step:176K smpl:1M ep:7K epch:24.27 loss:0.009 grdn:0.197 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:53:48 ts/train.py:232 step:176K smpl:1M ep:7K epch:24.29 loss:0.010 grdn:0.196 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:55:14 ts/train.py:232 step:177K smpl:1M ep:7K epch:24.32 loss:0.009 grdn:0.184 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:56:39 ts/train.py:232 step:177K smpl:1M ep:7K epch:24.35 loss:0.010 grdn:0.189 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:58:04 ts/train.py:232 step:177K smpl:1M ep:7K epch:24.38 loss:0.009 grdn:0.191 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 10:59:29 ts/train.py:232 step:177K smpl:1M ep:7K epch:24.40 loss:0.010 grdn:0.191 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:00:54 ts/train.py:232 step:177K smpl:1M ep:7K epch:24.43 loss:0.010 grdn:0.201 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:02:19 ts/train.py:232 step:178K smpl:1M ep:7K epch:24.46 loss:0.009 grdn:0.192 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:03:44 ts/train.py:232 step:178K smpl:1M ep:7K epch:24.49 loss:0.009 grdn:0.199 lr:9.1e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 11:05:09 ts/train.py:232 step:178K smpl:1M ep:7K epch:24.51 loss:0.009 grdn:0.183 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:06:34 ts/train.py:232 step:178K smpl:1M ep:7K epch:24.54 loss:0.008 grdn:0.181 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:07:59 ts/train.py:232 step:178K smpl:1M ep:7K epch:24.57 loss:0.010 grdn:0.191 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:09:24 ts/train.py:232 step:179K smpl:1M ep:7K epch:24.60 loss:0.008 grdn:0.185 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:10:49 ts/train.py:232 step:179K smpl:1M ep:7K epch:24.62 loss:0.009 grdn:0.183 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:12:14 ts/train.py:232 step:179K smpl:1M ep:7K epch:24.65 loss:0.008 grdn:0.178 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:13:40 ts/train.py:232 step:179K smpl:1M ep:7K epch:24.68 loss:0.009 grdn:0.199 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:15:05 ts/train.py:232 step:179K smpl:1M ep:7K epch:24.71 loss:0.009 grdn:0.194 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:16:30 ts/train.py:232 step:180K smpl:1M ep:7K epch:24.73 loss:0.008 grdn:0.184 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:17:55 ts/train.py:232 step:180K smpl:1M ep:7K epch:24.76 loss:0.009 grdn:0.191 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:19:20 ts/train.py:232 step:180K smpl:1M ep:7K epch:24.79 loss:0.009 grdn:0.194 lr:9.1e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:19:20 ts/train.py:241 Checkpoint policy after step 180000 +INFO 2025-05-06 11:20:49 ts/train.py:232 step:180K smpl:1M ep:7K epch:24.82 loss:0.009 grdn:0.191 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:22:14 ts/train.py:232 step:180K smpl:1M ep:7K epch:24.84 loss:0.010 grdn:0.193 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:23:39 ts/train.py:232 step:181K smpl:1M ep:7K epch:24.87 loss:0.010 grdn:0.191 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:25:04 ts/train.py:232 step:181K smpl:1M ep:7K epch:24.90 loss:0.009 grdn:0.178 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:26:29 ts/train.py:232 step:181K smpl:1M ep:7K epch:24.93 loss:0.009 grdn:0.187 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:27:54 ts/train.py:232 step:181K smpl:1M ep:7K epch:24.95 loss:0.009 grdn:0.190 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:29:19 ts/train.py:232 step:181K smpl:1M ep:7K epch:24.98 loss:0.010 grdn:0.199 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:30:44 ts/train.py:232 step:182K smpl:1M ep:8K epch:25.01 loss:0.009 grdn:0.185 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:32:10 ts/train.py:232 step:182K smpl:1M ep:8K epch:25.04 loss:0.009 grdn:0.185 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:33:36 ts/train.py:232 step:182K smpl:1M ep:8K epch:25.06 loss:0.008 grdn:0.167 lr:9.0e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-06 11:35:01 ts/train.py:232 step:182K smpl:1M ep:8K epch:25.09 loss:0.009 grdn:0.193 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:36:26 ts/train.py:232 step:182K smpl:1M ep:8K epch:25.12 loss:0.009 grdn:0.188 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:37:51 ts/train.py:232 step:183K smpl:1M ep:8K epch:25.15 loss:0.009 grdn:0.198 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:39:16 ts/train.py:232 step:183K smpl:1M ep:8K epch:25.17 loss:0.009 grdn:0.186 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:40:41 ts/train.py:232 step:183K smpl:1M ep:8K epch:25.20 loss:0.009 grdn:0.196 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:42:06 ts/train.py:232 step:183K smpl:1M ep:8K epch:25.23 loss:0.009 grdn:0.191 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:43:32 ts/train.py:232 step:183K smpl:1M ep:8K epch:25.26 loss:0.009 grdn:0.186 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:44:57 ts/train.py:232 step:184K smpl:1M ep:8K epch:25.28 loss:0.009 grdn:0.191 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:46:22 ts/train.py:232 step:184K smpl:1M ep:8K epch:25.31 loss:0.009 grdn:0.194 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:47:47 ts/train.py:232 step:184K smpl:1M ep:8K epch:25.34 loss:0.009 grdn:0.190 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:49:12 ts/train.py:232 step:184K smpl:1M ep:8K epch:25.37 loss:0.009 grdn:0.190 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:50:37 ts/train.py:232 step:184K smpl:1M ep:8K epch:25.40 loss:0.009 grdn:0.176 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:52:02 ts/train.py:232 step:185K smpl:1M ep:8K epch:25.42 loss:0.009 grdn:0.180 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:53:27 ts/train.py:232 step:185K smpl:1M ep:8K epch:25.45 loss:0.008 grdn:0.170 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:54:52 ts/train.py:232 step:185K smpl:1M ep:8K epch:25.48 loss:0.009 grdn:0.199 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:56:18 ts/train.py:232 step:185K smpl:1M ep:8K epch:25.51 loss:0.010 grdn:0.195 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:57:43 ts/train.py:232 step:185K smpl:1M ep:8K epch:25.53 loss:0.009 grdn:0.183 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 11:59:08 ts/train.py:232 step:186K smpl:1M ep:8K epch:25.56 loss:0.009 grdn:0.189 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:00:33 ts/train.py:232 step:186K smpl:1M ep:8K epch:25.59 loss:0.009 grdn:0.190 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:01:58 ts/train.py:232 step:186K smpl:1M ep:8K epch:25.62 loss:0.009 grdn:0.195 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:03:23 ts/train.py:232 step:186K smpl:1M ep:8K epch:25.64 loss:0.009 grdn:0.182 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:04:48 ts/train.py:232 step:186K smpl:1M ep:8K epch:25.67 loss:0.008 grdn:0.178 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:06:13 ts/train.py:232 step:187K smpl:1M ep:8K epch:25.70 loss:0.009 grdn:0.185 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:07:38 ts/train.py:232 step:187K smpl:1M ep:8K epch:25.73 loss:0.008 grdn:0.180 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:09:03 ts/train.py:232 step:187K smpl:1M ep:8K epch:25.75 loss:0.009 grdn:0.180 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:10:28 ts/train.py:232 step:187K smpl:1M ep:8K epch:25.78 loss:0.009 grdn:0.186 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:11:53 ts/train.py:232 step:187K smpl:1M ep:8K epch:25.81 loss:0.009 grdn:0.188 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:13:18 ts/train.py:232 step:188K smpl:2M ep:8K epch:25.84 loss:0.009 grdn:0.186 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:14:44 ts/train.py:232 step:188K smpl:2M ep:8K epch:25.86 loss:0.008 grdn:0.189 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:16:09 ts/train.py:232 step:188K smpl:2M ep:8K epch:25.89 loss:0.008 grdn:0.184 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:17:34 ts/train.py:232 step:188K smpl:2M ep:8K epch:25.92 loss:0.009 grdn:0.190 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:18:59 ts/train.py:232 step:188K smpl:2M ep:8K epch:25.95 loss:0.009 grdn:0.193 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:20:24 ts/train.py:232 step:189K smpl:2M ep:8K epch:25.97 loss:0.008 grdn:0.189 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:21:49 ts/train.py:232 step:189K smpl:2M ep:8K epch:26.00 loss:0.010 grdn:0.200 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:23:15 ts/train.py:232 step:189K smpl:2M ep:8K epch:26.03 loss:0.009 grdn:0.190 lr:9.0e-05 updt_s:0.423 data_s:0.008 +INFO 2025-05-06 12:24:40 ts/train.py:232 step:189K smpl:2M ep:8K epch:26.06 loss:0.009 grdn:0.178 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:26:05 ts/train.py:232 step:189K smpl:2M ep:8K epch:26.08 loss:0.009 grdn:0.191 lr:9.0e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:27:30 ts/train.py:232 step:190K smpl:2M ep:8K epch:26.11 loss:0.008 grdn:0.177 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:28:55 ts/train.py:232 step:190K smpl:2M ep:8K epch:26.14 loss:0.010 grdn:0.201 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:30:21 ts/train.py:232 step:190K smpl:2M ep:8K epch:26.17 loss:0.009 grdn:0.186 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:31:45 ts/train.py:232 step:190K smpl:2M ep:8K epch:26.19 loss:0.009 grdn:0.193 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:33:11 ts/train.py:232 step:190K smpl:2M ep:8K epch:26.22 loss:0.010 grdn:0.206 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:34:35 ts/train.py:232 step:191K smpl:2M ep:8K epch:26.25 loss:0.008 grdn:0.175 lr:8.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 12:36:01 ts/train.py:232 step:191K smpl:2M ep:8K epch:26.28 loss:0.008 grdn:0.173 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:37:26 ts/train.py:232 step:191K smpl:2M ep:8K epch:26.30 loss:0.008 grdn:0.181 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:38:51 ts/train.py:232 step:191K smpl:2M ep:8K epch:26.33 loss:0.009 grdn:0.193 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:40:16 ts/train.py:232 step:191K smpl:2M ep:8K epch:26.36 loss:0.010 grdn:0.191 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:41:41 ts/train.py:232 step:192K smpl:2M ep:8K epch:26.39 loss:0.009 grdn:0.185 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:43:06 ts/train.py:232 step:192K smpl:2M ep:8K epch:26.41 loss:0.008 grdn:0.184 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:44:31 ts/train.py:232 step:192K smpl:2M ep:8K epch:26.44 loss:0.008 grdn:0.182 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:45:56 ts/train.py:232 step:192K smpl:2M ep:8K epch:26.47 loss:0.009 grdn:0.192 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:47:21 ts/train.py:232 step:192K smpl:2M ep:8K epch:26.50 loss:0.010 grdn:0.202 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:48:46 ts/train.py:232 step:193K smpl:2M ep:8K epch:26.52 loss:0.008 grdn:0.180 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:50:12 ts/train.py:232 step:193K smpl:2M ep:8K epch:26.55 loss:0.009 grdn:0.190 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:51:37 ts/train.py:232 step:193K smpl:2M ep:8K epch:26.58 loss:0.009 grdn:0.184 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:53:02 ts/train.py:232 step:193K smpl:2M ep:8K epch:26.61 loss:0.009 grdn:0.183 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:54:27 ts/train.py:232 step:193K smpl:2M ep:8K epch:26.63 loss:0.009 grdn:0.190 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:55:52 ts/train.py:232 step:194K smpl:2M ep:8K epch:26.66 loss:0.009 grdn:0.184 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 12:57:17 ts/train.py:232 step:194K smpl:2M ep:8K epch:26.69 loss:0.008 grdn:0.186 lr:8.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 12:58:42 ts/train.py:232 step:194K smpl:2M ep:8K epch:26.72 loss:0.009 grdn:0.185 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:00:07 ts/train.py:232 step:194K smpl:2M ep:8K epch:26.74 loss:0.009 grdn:0.188 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:01:32 ts/train.py:232 step:194K smpl:2M ep:8K epch:26.77 loss:0.009 grdn:0.190 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:02:57 ts/train.py:232 step:195K smpl:2M ep:8K epch:26.80 loss:0.010 grdn:0.200 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:04:22 ts/train.py:232 step:195K smpl:2M ep:8K epch:26.83 loss:0.009 grdn:0.189 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:05:47 ts/train.py:232 step:195K smpl:2M ep:8K epch:26.85 loss:0.009 grdn:0.184 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:07:12 ts/train.py:232 step:195K smpl:2M ep:8K epch:26.88 loss:0.009 grdn:0.191 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:08:37 ts/train.py:232 step:195K smpl:2M ep:8K epch:26.91 loss:0.009 grdn:0.191 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:10:02 ts/train.py:232 step:196K smpl:2M ep:8K epch:26.94 loss:0.008 grdn:0.179 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:11:27 ts/train.py:232 step:196K smpl:2M ep:8K epch:26.97 loss:0.009 grdn:0.179 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:12:53 ts/train.py:232 step:196K smpl:2M ep:8K epch:26.99 loss:0.010 grdn:0.206 lr:8.9e-05 updt_s:0.424 data_s:0.007 +INFO 2025-05-06 13:14:18 ts/train.py:232 step:196K smpl:2M ep:8K epch:27.02 loss:0.008 grdn:0.183 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:15:44 ts/train.py:232 step:196K smpl:2M ep:8K epch:27.05 loss:0.009 grdn:0.186 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:17:09 ts/train.py:232 step:197K smpl:2M ep:8K epch:27.08 loss:0.009 grdn:0.183 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:18:34 ts/train.py:232 step:197K smpl:2M ep:8K epch:27.10 loss:0.010 grdn:0.206 lr:8.9e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 13:19:59 ts/train.py:232 step:197K smpl:2M ep:8K epch:27.13 loss:0.009 grdn:0.191 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:21:24 ts/train.py:232 step:197K smpl:2M ep:8K epch:27.16 loss:0.009 grdn:0.187 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:22:49 ts/train.py:232 step:197K smpl:2M ep:8K epch:27.19 loss:0.009 grdn:0.187 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:24:14 ts/train.py:232 step:198K smpl:2M ep:8K epch:27.21 loss:0.008 grdn:0.182 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:25:39 ts/train.py:232 step:198K smpl:2M ep:8K epch:27.24 loss:0.009 grdn:0.185 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:27:04 ts/train.py:232 step:198K smpl:2M ep:8K epch:27.27 loss:0.009 grdn:0.192 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:28:29 ts/train.py:232 step:198K smpl:2M ep:8K epch:27.30 loss:0.009 grdn:0.182 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:29:54 ts/train.py:232 step:198K smpl:2M ep:8K epch:27.32 loss:0.008 grdn:0.177 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:31:19 ts/train.py:232 step:199K smpl:2M ep:8K epch:27.35 loss:0.008 grdn:0.187 lr:8.9e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:32:44 ts/train.py:232 step:199K smpl:2M ep:8K epch:27.38 loss:0.009 grdn:0.193 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:34:09 ts/train.py:232 step:199K smpl:2M ep:8K epch:27.41 loss:0.009 grdn:0.192 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:35:35 ts/train.py:232 step:199K smpl:2M ep:8K epch:27.43 loss:0.009 grdn:0.181 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:37:00 ts/train.py:232 step:199K smpl:2M ep:8K epch:27.46 loss:0.009 grdn:0.188 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:38:25 ts/train.py:232 step:200K smpl:2M ep:8K epch:27.49 loss:0.009 grdn:0.195 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:39:50 ts/train.py:232 step:200K smpl:2M ep:8K epch:27.52 loss:0.009 grdn:0.180 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:41:15 ts/train.py:232 step:200K smpl:2M ep:8K epch:27.54 loss:0.009 grdn:0.195 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:42:40 ts/train.py:232 step:200K smpl:2M ep:8K epch:27.57 loss:0.008 grdn:0.192 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:44:05 ts/train.py:232 step:200K smpl:2M ep:8K epch:27.60 loss:0.008 grdn:0.192 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:45:30 ts/train.py:232 step:201K smpl:2M ep:8K epch:27.63 loss:0.008 grdn:0.186 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:46:55 ts/train.py:232 step:201K smpl:2M ep:8K epch:27.65 loss:0.010 grdn:0.198 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:48:21 ts/train.py:232 step:201K smpl:2M ep:8K epch:27.68 loss:0.009 grdn:0.189 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:49:46 ts/train.py:232 step:201K smpl:2M ep:8K epch:27.71 loss:0.009 grdn:0.193 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:51:11 ts/train.py:232 step:201K smpl:2M ep:8K epch:27.74 loss:0.009 grdn:0.187 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:52:36 ts/train.py:232 step:202K smpl:2M ep:8K epch:27.76 loss:0.008 grdn:0.182 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:54:01 ts/train.py:232 step:202K smpl:2M ep:8K epch:27.79 loss:0.009 grdn:0.190 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:55:26 ts/train.py:232 step:202K smpl:2M ep:8K epch:27.82 loss:0.008 grdn:0.182 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 13:56:51 ts/train.py:232 step:202K smpl:2M ep:8K epch:27.85 loss:0.008 grdn:0.179 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 13:58:16 ts/train.py:232 step:202K smpl:2M ep:8K epch:27.87 loss:0.010 grdn:0.196 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 13:59:41 ts/train.py:232 step:203K smpl:2M ep:8K epch:27.90 loss:0.009 grdn:0.187 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:01:06 ts/train.py:232 step:203K smpl:2M ep:8K epch:27.93 loss:0.009 grdn:0.188 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:02:33 ts/train.py:232 step:203K smpl:2M ep:8K epch:27.96 loss:0.009 grdn:0.187 lr:8.8e-05 updt_s:0.423 data_s:0.009 +INFO 2025-05-06 14:03:58 ts/train.py:232 step:203K smpl:2M ep:8K epch:27.98 loss:0.008 grdn:0.187 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:05:23 ts/train.py:232 step:203K smpl:2M ep:8K epch:28.01 loss:0.009 grdn:0.196 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:06:48 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.04 loss:0.008 grdn:0.183 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:08:13 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.07 loss:0.009 grdn:0.193 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:09:38 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.09 loss:0.008 grdn:0.171 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:11:03 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.12 loss:0.008 grdn:0.186 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:12:28 ts/train.py:232 step:204K smpl:2M ep:8K epch:28.15 loss:0.009 grdn:0.201 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:13:53 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.18 loss:0.009 grdn:0.197 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:15:19 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.20 loss:0.009 grdn:0.204 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:16:44 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.23 loss:0.009 grdn:0.187 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:18:09 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.26 loss:0.009 grdn:0.186 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:19:34 ts/train.py:232 step:205K smpl:2M ep:8K epch:28.29 loss:0.009 grdn:0.191 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:20:59 ts/train.py:232 step:206K smpl:2M ep:8K epch:28.31 loss:0.009 grdn:0.198 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:22:24 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.34 loss:0.009 grdn:0.194 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:23:49 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.37 loss:0.009 grdn:0.187 lr:8.8e-05 updt_s:0.424 data_s:0.001 +INFO 2025-05-06 14:25:14 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.40 loss:0.008 grdn:0.185 lr:8.8e-05 updt_s:0.423 data_s:0.001 +INFO 2025-05-06 14:26:39 ts/train.py:232 step:206K smpl:2M ep:9K epch:28.42 loss:0.009 grdn:0.190 lr:8.8e-05 updt_s:0.424 data_s:0.001 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/requirements.txt b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8fc771efb7757d31b751b471861a827f2ce353fc --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/requirements.txt @@ -0,0 +1,146 @@ +sympy==1.13.1 +torch==2.6.0 +nvidia-cuda-cupti-cu12==12.4.127 +gitdb==4.0.12 +nvidia-cusolver-cu12==11.6.1.9 +packaging==24.2 +numba==0.61.2 +safetensors==0.5.3 +nvidia-cusparselt-cu12==0.6.2 +pynput==1.8.1 +fsspec==2024.12.0 +filelock==3.18.0 +pydantic==2.11.3 +cloudpickle==3.1.1 +pillow==11.2.1 +lerobot==0.1.0 +inquirerpy==0.3.4 +charset-normalizer==3.4.1 +PySocks==1.7.1 +pycparser==2.22 +nvidia-cublas-cu12==12.4.5.8 +propcache==0.3.1 +GitPython==3.1.44 +llvmlite==0.44.0 +protobuf==5.29.4 +async-timeout==5.0.1 +draccus==0.10.0 +soupsieve==2.6 +Werkzeug==3.1.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +av==14.3.0 +huggingface-hub==0.30.2 +jsonlines==4.0.0 +setproctitle==1.3.5 +imageio==2.37.0 +aiohttp==3.11.16 +einops==0.8.1 +imageio-ffmpeg==0.6.0 +pytz==2025.2 +pymunk==6.11.1 +typing-inspect==0.9.0 +PyYAML==6.0.2 +cffi==1.17.1 +feetech-servo-sdk==1.0.0 +pyserial==3.5 +mypy-extensions==1.0.0 +MarkupSafe==3.0.2 +zipp==3.21.0 +typing_extensions==4.13.2 +setuptools==78.1.0 +python-xlib==0.33 +tzdata==2025.2 +requests==2.32.3 +pip==25.0.1 +mpmath==1.3.0 +annotated-types==0.7.0 +nvidia-cuda-runtime-cu12==12.4.127 +prompt_toolkit==3.0.51 +evdev==1.9.1 +dill==0.3.8 +nvidia-cufft-cu12==11.2.1.3 +python-dateutil==2.9.0.post0 +nvidia-nvjitlink-cu12==12.4.127 +attrs==25.3.0 +orderly-set==5.4.0 +regex==2024.11.6 +tqdm==4.67.1 +pyzmq==26.4.0 +numpy==2.2.4 +nvidia-cudnn-cu12==9.1.0.70 +toml==0.10.2 +fasteners==0.19 +wheel==0.45.1 +xxhash==3.5.0 +platformdirs==4.3.7 +nvidia-curand-cu12==10.3.5.147 +sentry-sdk==2.26.1 +yarl==1.20.0 +Flask==3.1.0 +smmap==5.0.2 +nvidia-cusparse-cu12==12.3.1.170 +docker-pycreds==0.4.0 +importlib_metadata==8.6.1 +Farama-Notifications==0.0.4 +pyyaml-include==1.4.1 +termcolor==3.0.1 +zarr==2.18.3 +mergedeep==1.3.4 +wandb==0.19.9 +urllib3==2.4.0 +pandas==2.2.3 +beautifulsoup4==4.13.4 +six==1.17.0 +hf_transfer==0.1.9 +antlr4-python3-runtime==4.9.3 +click==8.1.8 +psutil==7.0.0 +omegaconf==2.3.0 +wcwidth==0.2.13 +certifi==2025.1.31 +typing-inspection==0.4.0 +itsdangerous==2.2.0 +diffusers==0.33.1 +opencv-python-headless==4.11.0.86 +torchvision==0.21.0 +pydantic_core==2.33.1 +networkx==3.4.2 +rerun-sdk==0.22.1 +cmake==4.0.0 +numcodecs==0.13.1 +gymnasium==0.29.1 +multiprocess==0.70.16 +pfzy==0.3.4 +idna==3.10 +asciitree==0.3.3 +triton==3.2.0 +multidict==6.4.3 +frozenlist==1.5.0 +gdown==5.2.0 +deepdiff==8.4.2 +aiohappyeyeballs==2.6.1 +blinker==1.9.0 +Jinja2==3.1.6 +aiosignal==1.3.2 +pyarrow==19.0.1 +nvidia-nccl-cu12==2.21.5 +h5py==3.13.0 +TorchCodec==0.2.1 +datasets==3.5.0 +nvidia-nvtx-cu12==12.4.127 +packaging==24.2 +importlib_metadata==8.0.0 +autocommand==2.2.2 +typing_extensions==4.12.2 +zipp==3.19.2 +platformdirs==4.2.2 +jaraco.collections==5.1.0 +jaraco.text==3.12.1 +jaraco.context==5.3.0 +wheel==0.45.1 +typeguard==4.3.0 +tomli==2.0.1 +more-itertools==10.3.0 +backports.tarfile==1.2.0 +jaraco.functools==4.0.1 +inflect==7.3.1 diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/wandb-metadata.json b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d4c4e6f4bf15fbde11faa47e24771efda867f8cb --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/files/wandb-metadata.json @@ -0,0 +1,109 @@ +{ + "os": "Linux-6.8.0-58-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.17", + "startedAt": "2025-05-05T13:50:12.200342Z", + "args": [ + "--dataset.repo_id=shylee/so100_pengripA", + "--policy.type=diffusion", + "--policy.device=cuda", + "--dataset.root=/SSD/LSY/lerobot_dataset/pengripA", + "--wandb.enable=true", + "--save_freq=30000", + "--steps=900000", + "--policy.noise_scheduler_type=DDIM", + "--policy.num_inference_steps=10", + "--batch_size=8", + "--policy.down_dims=[256,512,1024]", + "--policy.vision_backbone=resnet50", + "--policy.pretrained_backbone_weights=ResNet50_Weights.IMAGENET1K_V1", + "--policy.use_group_norm=false", + "--policy.crop_shape=[480,640]", + "--policy.crop_is_random=false", + "--policy.freeze_vision_backbone=false", + "--policy.horizon=32", + "--policy.n_action_steps=32", + "--policy.optimizer_lr=0.0001", + "--policy.use_ema=true", + "--policy.ema_lambda=0.5", + "--output_dir=/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "--job_name=DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4" + ], + "program": "/NAS/LSY/lerobot/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "git": { + "remote": "https://github.com/huggingface/lerobot.git", + "commit": "42bf1e8b9df3d38b3898b24e46b5e0386910c466" + }, + "root": "/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4", + "host": "8b1bacebd36d", + "executable": "/opt/conda/envs/lerobot/bin/python", + "codePathLocal": "lerobot/scripts/train.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA RTX 6000 Ada Generation", + "gpu_count": 8, + "disk": { + "/": { + "total": "3778772082688", + "used": "3371102519296" + } + }, + "memory": { + "total": "540891144192" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + }, + { + "name": "NVIDIA RTX 6000 Ada Generation", + "memoryTotal": "51527024640", + "cudaCores": 18176, + "architecture": "Ada" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-core.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b50695ef0cfebf5e276b7e882d110d6b57ed749b --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-core.log @@ -0,0 +1,6 @@ +{"time":"2025-05-05T13:50:12.003436101Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphb8xw9nq/port-106588.txt","pid":106588,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-05-05T13:50:12.00431301Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":106588} +{"time":"2025-05-05T13:50:12.00430766Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37201,"Zone":""}} +{"time":"2025-05-05T13:50:12.193687277Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:60862"} +{"time":"2025-05-05T13:50:12.201074417Z","level":"INFO","msg":"handleInformInit: received","streamId":"fjhp50hn","id":"127.0.0.1:60862"} +{"time":"2025-05-05T13:50:12.532899082Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"fjhp50hn","id":"127.0.0.1:60862"} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..12cf918367d22cc11e03be8cc2518db67592db79 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log @@ -0,0 +1,13 @@ +{"time":"2025-05-05T13:50:12.201235767Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-core.log"} +{"time":"2025-05-05T13:50:12.532812412Z","level":"INFO","msg":"created new stream","id":"fjhp50hn"} +{"time":"2025-05-05T13:50:12.532892102Z","level":"INFO","msg":"stream: started","id":"fjhp50hn"} +{"time":"2025-05-05T13:50:12.532970282Z","level":"INFO","msg":"sender: started","stream_id":"fjhp50hn"} +{"time":"2025-05-05T13:50:12.532968422Z","level":"INFO","msg":"writer: Do: started","stream_id":"fjhp50hn"} +{"time":"2025-05-05T13:50:12.533061181Z","level":"INFO","msg":"handler: started","stream_id":"fjhp50hn"} +{"time":"2025-05-05T13:50:13.03263614Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-05-05T22:01:13.826925473Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-05-06T09:24:18.037401348Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:60326->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T10:19:48.044760185Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:38057->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T10:30:03.039602464Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:41710->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:25:33.040402545Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:56185->141.223.7.51:53: i/o timeout"} +{"time":"2025-05-06T12:44:48.040371676Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/fjhp50hn/file_stream\": dial tcp: lookup api.wandb.ai on 141.223.7.51:53: read udp 172.17.0.4:43741->141.223.7.51:53: i/o timeout"} diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug.log b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f73f226450c4172998f055736fc48ba880ec89d5 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug.log @@ -0,0 +1,22 @@ +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Configure stats pid to 106588 +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug.log +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/logs/debug-internal.log +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:init():781] calling init triggers +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'shylee/so100_pengripA', 'root': '/SSD/LSY/lerobot_dataset/pengripA', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'horizon': 32, 'n_action_steps': 32, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [480, 640], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': None, 'freeze_vision_backbone': False, 'use_ema': True, 'ema_lambda': 0.5}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4', 'job_name': 'DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 900000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 30000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:init():809] starting backend +2025-05-05 13:50:12,196 INFO MainThread:106588 [wandb_init.py:init():813] sending inform_init request +2025-05-05 13:50:12,199 INFO MainThread:106588 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-05-05 13:50:12,199 INFO MainThread:106588 [wandb_init.py:init():823] backend started and connected +2025-05-05 13:50:12,201 INFO MainThread:106588 [wandb_init.py:init():915] updated telemetry +2025-05-05 13:50:12,385 INFO MainThread:106588 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-05-05 13:50:13,028 INFO MainThread:106588 [wandb_init.py:init():1014] starting run threads in backend +2025-05-05 13:50:13,119 INFO MainThread:106588 [wandb_run.py:_console_start():2454] atexit reg +2025-05-05 13:50:13,120 INFO MainThread:106588 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-05-05 13:50:13,120 INFO MainThread:106588 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-05-05 13:50:13,120 INFO MainThread:106588 [wandb_run.py:_redirect():2394] Redirects installed. +2025-05-05 13:50:13,121 INFO MainThread:106588 [wandb_init.py:init():1056] run started, returning control to user process diff --git a/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb new file mode 100644 index 0000000000000000000000000000000000000000..f948fc4a32a2436624707a9cf4e63ff78424d972 --- /dev/null +++ b/DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-fjhp50hn/run-fjhp50hn.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d888a831344268b1be8c52b9993e710067272ba5ae0436d3b10c613c95bcf1ff +size 19431424