diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/pretrained_model/config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..efcca9315d08b67619deec97821406d8cc63f1c7 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/pretrained_model/config.json @@ -0,0 +1,101 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/pretrained_model/model.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..073d070cf171c1b4c528c630c546d44142869786 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e3b73198b54095c8df31d3be4b219b871afb32484a76eacc299f4e5a9a39df9 +size 369241872 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/pretrained_model/train_config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd7cee29257c2b5fec8a3939559e95bed146e6cd --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripF", + "root": "/SSD/LSY/lerobot_dataset/pengripF", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/optimizer_param_groups.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..6926324a229b5f3a741c4acd6c9f9055873ff93e --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.55941723863299e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/optimizer_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c9c80215875888c3d6f847b1a84468e1ff3cc334 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:953ae1c8d421a57b956f96f7fc2f603de86e7533e2a2f06c49cdd3e16b610d52 +size 738026076 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/rng_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7e15edf269a6df8001c4d77189c9afeb88c03b06 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22ae220e7fc0111f9c343ab862908420b1761af706738e811ede5ed481d32e2f +size 15708 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/scheduler_state.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..57f7f32291879d54dd6884b139a3353e9208cf67 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 720000, + "verbose": false, + "_step_count": 720001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.55941723863299e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/training_step.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..721e07e355e18f869a5dd8aa6f83c651ec7c3772 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/720000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 720000 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/pretrained_model/config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..efcca9315d08b67619deec97821406d8cc63f1c7 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/pretrained_model/config.json @@ -0,0 +1,101 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/pretrained_model/model.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aa93bbbfdd9694827f15972336d985d090cacd91 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:628a81acb9b4992a37b825c5f0b7688c11e4f37c4692ee900b78e68fc2c124c3 +size 369241872 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/pretrained_model/train_config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd7cee29257c2b5fec8a3939559e95bed146e6cd --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripF", + "root": "/SSD/LSY/lerobot_dataset/pengripF", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/optimizer_param_groups.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..910528e41cdfbb139e48f5656e7a9c98cbc08d9e --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.706007892278027e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/optimizer_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a4e251ad3340d5df201d594f36cbe029aac885ad --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7edd80c844976076676b4706d633728cb87acbc0cbc8785753ecbcd942a8848 +size 738026076 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/rng_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..28607b8e1c5ba9996d6772bcae834c6cc9b57a11 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e249ed34dc51f65c60a5733595e8a9437cc8f8a5d656e9a015862b8a9a1d378e +size 15708 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/scheduler_state.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..531a163e77edd5cec217b3d8dfdf81d78aac5b48 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 750000, + "verbose": false, + "_step_count": 750001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.706007892278027e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/training_step.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..362e20eb49f9138dd347449af4f6da372b3c74ac --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/750000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 750000 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/pretrained_model/config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..efcca9315d08b67619deec97821406d8cc63f1c7 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/pretrained_model/config.json @@ -0,0 +1,101 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/pretrained_model/model.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..387127cbf5c8f9ff7bd3ba6b34885b1b72ec9719 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5526ddbfb96cc0f41495e2387a378a3f86955cb651a84ce68fccd830b13c33f6 +size 369241872 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/pretrained_model/train_config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd7cee29257c2b5fec8a3939559e95bed146e6cd --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripF", + "root": "/SSD/LSY/lerobot_dataset/pengripF", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/optimizer_param_groups.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9c890340d82de989b405601d8eb6f70d68373 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 4.327463582432278e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/optimizer_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..91c0c093a66a99f88b1a3746acff0153935bd25f --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc2766a609eb284683080ac65d760e29e98420d10e8c38f3d9abcde496526964 +size 738026076 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/rng_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5144a71a4ae9e3236959440467ecdcae1bfbd688 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de0bb88a254d6a5ea1ef98e3c7c84b8e64088421592d3f3cebcd89bea8208510 +size 15708 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/scheduler_state.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1fed77bf3648d162e332234925925207b7372ef2 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 780000, + "verbose": false, + "_step_count": 780001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 4.327463582432278e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/training_step.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..e908e4d92cd192332ca7f02b2314283190a92940 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/780000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 780000 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/pretrained_model/config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..efcca9315d08b67619deec97821406d8cc63f1c7 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/pretrained_model/config.json @@ -0,0 +1,101 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/pretrained_model/model.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..94295dacad6019eef3caeea788b56da331bde93e --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:784ae51eb4eaa915f345518b527ec5a1f0bbdc1f52bd069e4a937c914e45f3b9 +size 369241872 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/pretrained_model/train_config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd7cee29257c2b5fec8a3939559e95bed146e6cd --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripF", + "root": "/SSD/LSY/lerobot_dataset/pengripF", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/optimizer_param_groups.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..24fd12b096a8b3cc8374a56fa3f1444ddd1e79a7 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.449873091292648e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/optimizer_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f8510dcf7744829530eaa4ea65520222f3cbeca3 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b50f0d614e295253ddc16e9a0220d38735ad3d806df289bfbf10c856f5dc551c +size 738026076 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/rng_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..39559bfcbf7bb1f02b0851f210792cc7d06cdd19 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e388e7e32b812492a196d74a2c5a149bd2aa5a1d1dc24b1500b372171479e119 +size 15708 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/scheduler_state.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ccfa1dee81e2d12eb7db6aab02f6d04e953e7d64 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 810000, + "verbose": false, + "_step_count": 810001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.449873091292648e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/training_step.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dd21436a578c3f7692acb15fcf45406552aea847 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/810000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 810000 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/pretrained_model/config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..efcca9315d08b67619deec97821406d8cc63f1c7 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/pretrained_model/config.json @@ -0,0 +1,101 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/pretrained_model/model.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aec6d261353686cd568e2aa0361d35ea34b62078 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cb521bf481de45069aa18a0962827c2184127b48b68f83fea9abd19bf9f64b5 +size 369241872 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/pretrained_model/train_config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd7cee29257c2b5fec8a3939559e95bed146e6cd --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripF", + "root": "/SSD/LSY/lerobot_dataset/pengripF", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/optimizer_param_groups.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..d7d9b557f0d16e9ca21004cc468242db56e55f03 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 1.0938305482918298e-06, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/optimizer_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e2cea881c69f53b99e8848fd0a191ad81fc12b2a --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a7eec2688daa6ac517aabad78c056b3e20075af274cd079f79d87347d5919cf +size 738026076 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/rng_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0d5ab3cb56f3673d752c1868408e63e24a4feb7b --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3880dd917197a6eab6035eeb470540790b3f7ab26f37af3752ea7a4e9c2897df +size 15708 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/scheduler_state.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..34b4a3103a1dd0587363f83d1665fd3745f5768f --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 840000, + "verbose": false, + "_step_count": 840001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 1.0938305482918298e-06 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/training_step.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dcc98c000f570a21f87e055135c99c45d8095ae6 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/840000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 840000 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/pretrained_model/config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..efcca9315d08b67619deec97821406d8cc63f1c7 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/pretrained_model/config.json @@ -0,0 +1,101 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/pretrained_model/model.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..68984935a37ae4b456ea450a2090fc87e462d30f --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb4968b2036773c2b8e1d22ca8bffb1194355448af9b7cffcf2478fe35052b43 +size 369241872 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/pretrained_model/train_config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd7cee29257c2b5fec8a3939559e95bed146e6cd --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripF", + "root": "/SSD/LSY/lerobot_dataset/pengripF", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/optimizer_param_groups.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..ff95a438a66d456a3bc8a3c9f97111b5203e78a7 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 2.742095458231675e-07, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/optimizer_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9c64c23d7ae57623f32719f11292472c3932b580 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da086504de0861e1a3cd50cbd633825e4575165405d7705f80f3338ec02cd5eb +size 738026076 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/rng_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bfa403606c9594f4486fac6c010a2859d0f65e25 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc7925e48a37762052a19423d0bdf91785289747f140002e681dbbd4787ede4c +size 15708 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/scheduler_state.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8b59af52612a6931b227893f3a6e8bb90aaf033d --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 870000, + "verbose": false, + "_step_count": 870001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 2.742095458231675e-07 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/training_step.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d7dc2eb64e5995132035fbf3233a84acd6dbdfee --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/870000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 870000 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/pretrained_model/config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..efcca9315d08b67619deec97821406d8cc63f1c7 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/pretrained_model/config.json @@ -0,0 +1,101 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/pretrained_model/model.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..03fe817329344e811da529223cc38dd654ad87dd --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:762e13356108402ef49d68f4f3e73989308ac3ef25817527e93d7972ad11cd26 +size 369241872 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/pretrained_model/train_config.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..dd7cee29257c2b5fec8a3939559e95bed146e6cd --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/pretrained_model/train_config.json @@ -0,0 +1,211 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripF", + "root": "/SSD/LSY/lerobot_dataset/pengripF", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.98, + "use_pcg": false, + "pcg_weight": 0.1 + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 30000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/optimizer_param_groups.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..c27412d579214db5ef0199d94aca48d5fbd9aecc --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 0.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/optimizer_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..467ec46e6c10dcad3e3978bc4c7d2bd41c75ca07 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a04f4d17edcfbe3de2ea1c58b939121147a4d5fe18363f3de027fce14e152489 +size 738026076 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/rng_state.safetensors b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..48816241e0bf9ad885a2ae9f7aee85ce262a7434 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d958dbfeba6b3f10a2f68131f282421f55b08f4ad94c761c9a345f9f0de38e17 +size 15708 diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/scheduler_state.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f820026d12b5046df9a66ee511c37ba824ab0033 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 900000, + "verbose": false, + "_step_count": 900001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 0.0 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/training_step.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..6e08a40d8f4b15183bbf2305222048be81ca55ff --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/900000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 900000 +} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug-internal.log b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug-internal.log index b261b7a09f53560365718d3e08a49ef1eb07e7a3..3847323a6b72e5124a1aa9dbbf2fd9f584e878f8 100644 --- a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug-internal.log +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug-internal.log @@ -8,3 +8,12 @@ {"time":"2025-05-09T16:51:02.122798906Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/jaesunghan-postech/lerobot/el8imly4/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} {"time":"2025-05-09T19:47:55.073959875Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/jaesunghan-postech/lerobot/el8imly4/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} {"time":"2025-05-10T23:50:18.71286255Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} +{"time":"2025-05-12T02:21:51.227152305Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/jaesunghan-postech/lerobot/el8imly4/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-05-12T04:22:16.076513163Z","level":"INFO","msg":"stream: closing","id":"el8imly4"} +{"time":"2025-05-12T04:22:16.076609613Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-05-12T04:22:16.078445309Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-05-12T04:22:35.315504548Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-05-12T04:22:35.582918725Z","level":"INFO","msg":"writer: Close: closed","stream_id":"el8imly4"} +{"time":"2025-05-12T04:22:35.583471795Z","level":"INFO","msg":"handler: closed","stream_id":"el8imly4"} +{"time":"2025-05-12T04:22:35.585413928Z","level":"INFO","msg":"sender: closed","stream_id":"el8imly4"} +{"time":"2025-05-12T04:22:35.585504302Z","level":"INFO","msg":"stream: closed","id":"el8imly4"} diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug.log b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug.log index 7f63278ff5c72d19cdd80cac91af806f529d6fc2..7655023cce3cae4d2c1b21971ad2cb7225bc6f3d 100644 --- a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug.log +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug.log @@ -20,3 +20,4 @@ config: {'dataset': {'repo_id': 'shylee/so100_pengripF', 'root': '/SSD/LSY/lerob 2025-05-09 11:59:31,470 INFO MainThread:5678 [wandb_run.py:_redirect():2371] Wrapping output streams. 2025-05-09 11:59:31,470 INFO MainThread:5678 [wandb_run.py:_redirect():2394] Redirects installed. 2025-05-09 11:59:31,472 INFO MainThread:5678 [wandb_init.py:init():1056] run started, returning control to user process +2025-05-12 04:22:16,072 INFO MsgRouterThr:5678 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles. diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/config.yaml b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de5c0d9fa4e2ae4a41557dd5f38eb49307bd84c0 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/config.yaml @@ -0,0 +1,191 @@ +_wandb: + value: + cli_version: 0.19.9 + m: [] + python_version: 3.10.17 + t: + "1": + - 1 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 41 + - 49 + - 51 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.10.17 + "5": 0.19.9 + "8": + - 5 + "12": 0.19.9 + "13": linux-x86_64 +batch_size: + value: 16 +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: shylee/so100_pengripF + revision: null + root: /SSD/LSY/lerobot_dataset/pengripF + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.95 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 0.0001 + type: adam + weight_decay: 1e-06 +output_dir: + value: /SSD/LSY/lerobot_model/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain +policy: + value: + ac_threshold: 0.98 + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + clip_sample_range: 1 + crop_is_random: false + crop_shape: + - 224 + - 224 + device: cuda + diffusion_step_embed_dim: 128 + do_mask_loss_for_padding: false + down_dims: + - 256 + - 512 + - 1024 + drop_n_last_frames: 7 + freeze_vision_backbone: false + horizon: 16 + kernel_size: 5 + n_action_steps: 8 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler_type: DDIM + normalization_mapping: + ACTION: MIN_MAX + STATE: MIN_MAX + VISUAL: MEAN_STD + num_inference_steps: 10 + num_train_timesteps: 100 + optimizer_betas: + - 0.95 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-06 + pcg_weight: 0.1 + pre_resize_shape: + - 224 + - 224 + prediction_type: epsilon + pretrained_backbone_weights: ResNet50_Weights.IMAGENET1K_V1 + scheduler_name: cosine + scheduler_warmup_steps: 500 + spatial_softmax_num_keypoints: 32 + type: diffusion + use_ac: false + use_amp: false + use_film_scale_modulation: true + use_group_norm: false + use_pcg: false + use_separate_rgb_encoder_per_camera: false + vision_backbone: resnet50 +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 30000 +scheduler: + value: + name: cosine + num_warmup_steps: 500 + type: diffuser +seed: + value: 1000 +steps: + value: 900000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/output.log b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/output.log index f27fce413c4162f6cf539104167bc4401d44ac82..8b650dc007484377f109f6ac4ecfc7333ab78ee1 100644 --- a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/output.log +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/output.log @@ -4786,3 +4786,969 @@ INFO 2025-05-11 14:59:36 ts/train.py:232 step:708K smpl:11M ep:57K epch:190.12 l INFO 2025-05-11 15:00:27 ts/train.py:232 step:708K smpl:11M ep:57K epch:190.17 loss:0.002 grdn:0.160 lr:1.1e-05 updt_s:0.227 data_s:0.025 INFO 2025-05-11 15:01:17 ts/train.py:232 step:708K smpl:11M ep:57K epch:190.22 loss:0.002 grdn:0.147 lr:1.1e-05 updt_s:0.227 data_s:0.025 INFO 2025-05-11 15:02:08 ts/train.py:232 step:708K smpl:11M ep:57K epch:190.28 loss:0.002 grdn:0.159 lr:1.1e-05 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 15:02:58 ts/train.py:232 step:709K smpl:11M ep:57K epch:190.33 loss:0.002 grdn:0.140 lr:1.1e-05 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 15:03:50 ts/train.py:232 step:709K smpl:11M ep:57K epch:190.38 loss:0.002 grdn:0.138 lr:1.1e-05 updt_s:0.228 data_s:0.029 +INFO 2025-05-11 15:04:42 ts/train.py:232 step:709K smpl:11M ep:57K epch:190.44 loss:0.002 grdn:0.135 lr:1.1e-05 updt_s:0.227 data_s:0.031 +INFO 2025-05-11 15:05:32 ts/train.py:232 step:709K smpl:11M ep:57K epch:190.49 loss:0.002 grdn:0.133 lr:1.1e-05 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 15:06:24 ts/train.py:232 step:709K smpl:11M ep:57K epch:190.55 loss:0.002 grdn:0.141 lr:1.1e-05 updt_s:0.227 data_s:0.031 +INFO 2025-05-11 15:07:16 ts/train.py:232 step:710K smpl:11M ep:57K epch:190.60 loss:0.002 grdn:0.143 lr:1.1e-05 updt_s:0.227 data_s:0.031 +INFO 2025-05-11 15:08:07 ts/train.py:232 step:710K smpl:11M ep:57K epch:190.65 loss:0.002 grdn:0.153 lr:1.1e-05 updt_s:0.227 data_s:0.032 +INFO 2025-05-11 15:08:59 ts/train.py:232 step:710K smpl:11M ep:57K epch:190.71 loss:0.002 grdn:0.135 lr:1.1e-05 updt_s:0.227 data_s:0.029 +INFO 2025-05-11 15:09:49 ts/train.py:232 step:710K smpl:11M ep:57K epch:190.76 loss:0.002 grdn:0.139 lr:1.1e-05 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 15:10:40 ts/train.py:232 step:710K smpl:11M ep:57K epch:190.81 loss:0.002 grdn:0.144 lr:1.1e-05 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 15:11:30 ts/train.py:232 step:711K smpl:11M ep:57K epch:190.87 loss:0.002 grdn:0.132 lr:1.1e-05 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 15:12:20 ts/train.py:232 step:711K smpl:11M ep:57K epch:190.92 loss:0.002 grdn:0.138 lr:1.1e-05 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 15:13:10 ts/train.py:232 step:711K smpl:11M ep:57K epch:190.98 loss:0.002 grdn:0.148 lr:1.1e-05 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 15:14:00 ts/train.py:232 step:711K smpl:11M ep:57K epch:191.03 loss:0.002 grdn:0.158 lr:1.0e-05 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 15:14:53 ts/train.py:232 step:711K smpl:11M ep:57K epch:191.08 loss:0.002 grdn:0.131 lr:1.0e-05 updt_s:0.225 data_s:0.035 +INFO 2025-05-11 15:15:42 ts/train.py:232 step:712K smpl:11M ep:57K epch:191.14 loss:0.002 grdn:0.146 lr:1.0e-05 updt_s:0.226 data_s:0.017 +INFO 2025-05-11 15:16:30 ts/train.py:232 step:712K smpl:11M ep:57K epch:191.19 loss:0.002 grdn:0.150 lr:1.0e-05 updt_s:0.226 data_s:0.017 +INFO 2025-05-11 15:17:19 ts/train.py:232 step:712K smpl:11M ep:57K epch:191.24 loss:0.002 grdn:0.160 lr:1.0e-05 updt_s:0.226 data_s:0.019 +INFO 2025-05-11 15:18:09 ts/train.py:232 step:712K smpl:11M ep:57K epch:191.30 loss:0.002 grdn:0.152 lr:1.0e-05 updt_s:0.226 data_s:0.020 +INFO 2025-05-11 15:18:58 ts/train.py:232 step:712K smpl:11M ep:57K epch:191.35 loss:0.002 grdn:0.139 lr:1.0e-05 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 15:19:46 ts/train.py:232 step:713K smpl:11M ep:57K epch:191.40 loss:0.002 grdn:0.135 lr:1.0e-05 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 15:20:34 ts/train.py:232 step:713K smpl:11M ep:57K epch:191.46 loss:0.002 grdn:0.128 lr:1.0e-05 updt_s:0.227 data_s:0.013 +INFO 2025-05-11 15:21:23 ts/train.py:232 step:713K smpl:11M ep:57K epch:191.51 loss:0.002 grdn:0.143 lr:1.0e-05 updt_s:0.227 data_s:0.013 +INFO 2025-05-11 15:22:12 ts/train.py:232 step:713K smpl:11M ep:57K epch:191.57 loss:0.002 grdn:0.140 lr:1.0e-05 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 15:22:59 ts/train.py:232 step:713K smpl:11M ep:57K epch:191.62 loss:0.002 grdn:0.163 lr:1.0e-05 updt_s:0.227 data_s:0.012 +INFO 2025-05-11 15:23:47 ts/train.py:232 step:714K smpl:11M ep:58K epch:191.67 loss:0.002 grdn:0.145 lr:1.0e-05 updt_s:0.227 data_s:0.011 +INFO 2025-05-11 15:24:35 ts/train.py:232 step:714K smpl:11M ep:58K epch:191.73 loss:0.002 grdn:0.149 lr:1.0e-05 updt_s:0.227 data_s:0.011 +INFO 2025-05-11 15:25:23 ts/train.py:232 step:714K smpl:11M ep:58K epch:191.78 loss:0.002 grdn:0.154 lr:1.0e-05 updt_s:0.227 data_s:0.012 +INFO 2025-05-11 15:26:10 ts/train.py:232 step:714K smpl:11M ep:58K epch:191.83 loss:0.002 grdn:0.140 lr:1.0e-05 updt_s:0.227 data_s:0.009 +INFO 2025-05-11 15:26:58 ts/train.py:232 step:714K smpl:11M ep:58K epch:191.89 loss:0.002 grdn:0.147 lr:1.0e-05 updt_s:0.227 data_s:0.010 +INFO 2025-05-11 15:27:45 ts/train.py:232 step:715K smpl:11M ep:58K epch:191.94 loss:0.002 grdn:0.146 lr:1.0e-05 updt_s:0.227 data_s:0.009 +INFO 2025-05-11 15:28:32 ts/train.py:232 step:715K smpl:11M ep:58K epch:192.00 loss:0.002 grdn:0.142 lr:1.0e-05 updt_s:0.227 data_s:0.009 +INFO 2025-05-11 15:29:24 ts/train.py:232 step:715K smpl:11M ep:58K epch:192.05 loss:0.002 grdn:0.139 lr:1.0e-05 updt_s:0.227 data_s:0.030 +INFO 2025-05-11 15:30:13 ts/train.py:232 step:715K smpl:11M ep:58K epch:192.10 loss:0.002 grdn:0.124 lr:1.0e-05 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 15:31:02 ts/train.py:232 step:715K smpl:11M ep:58K epch:192.16 loss:0.002 grdn:0.142 lr:1.0e-05 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 15:31:51 ts/train.py:232 step:716K smpl:11M ep:58K epch:192.21 loss:0.002 grdn:0.146 lr:1.0e-05 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 15:32:40 ts/train.py:232 step:716K smpl:11M ep:58K epch:192.26 loss:0.002 grdn:0.144 lr:1.0e-05 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 15:33:29 ts/train.py:232 step:716K smpl:11M ep:58K epch:192.32 loss:0.002 grdn:0.133 lr:1.0e-05 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 15:34:17 ts/train.py:232 step:716K smpl:11M ep:58K epch:192.37 loss:0.002 grdn:0.141 lr:1.0e-05 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 15:35:07 ts/train.py:232 step:716K smpl:11M ep:58K epch:192.43 loss:0.002 grdn:0.150 lr:9.9e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 15:35:56 ts/train.py:232 step:717K smpl:11M ep:58K epch:192.48 loss:0.002 grdn:0.158 lr:9.9e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 15:36:45 ts/train.py:232 step:717K smpl:11M ep:58K epch:192.53 loss:0.002 grdn:0.152 lr:9.9e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 15:37:33 ts/train.py:232 step:717K smpl:11M ep:58K epch:192.59 loss:0.002 grdn:0.151 lr:9.9e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 15:38:22 ts/train.py:232 step:717K smpl:11M ep:58K epch:192.64 loss:0.002 grdn:0.148 lr:9.9e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 15:39:11 ts/train.py:232 step:717K smpl:11M ep:58K epch:192.69 loss:0.002 grdn:0.152 lr:9.8e-06 updt_s:0.228 data_s:0.018 +INFO 2025-05-11 15:40:00 ts/train.py:232 step:718K smpl:11M ep:58K epch:192.75 loss:0.002 grdn:0.150 lr:9.8e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 15:40:50 ts/train.py:232 step:718K smpl:11M ep:58K epch:192.80 loss:0.002 grdn:0.126 lr:9.8e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 15:41:39 ts/train.py:232 step:718K smpl:11M ep:58K epch:192.86 loss:0.002 grdn:0.141 lr:9.8e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 15:42:27 ts/train.py:232 step:718K smpl:11M ep:58K epch:192.91 loss:0.002 grdn:0.145 lr:9.8e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 15:43:16 ts/train.py:232 step:718K smpl:11M ep:58K epch:192.96 loss:0.002 grdn:0.142 lr:9.7e-06 updt_s:0.227 data_s:0.014 +INFO 2025-05-11 15:44:07 ts/train.py:232 step:719K smpl:11M ep:58K epch:193.02 loss:0.002 grdn:0.157 lr:9.7e-06 updt_s:0.226 data_s:0.031 +INFO 2025-05-11 15:44:57 ts/train.py:232 step:719K smpl:12M ep:58K epch:193.07 loss:0.002 grdn:0.148 lr:9.7e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 15:45:46 ts/train.py:232 step:719K smpl:12M ep:58K epch:193.12 loss:0.002 grdn:0.131 lr:9.7e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 15:46:34 ts/train.py:232 step:719K smpl:12M ep:58K epch:193.18 loss:0.002 grdn:0.152 lr:9.7e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 15:47:24 ts/train.py:232 step:719K smpl:12M ep:58K epch:193.23 loss:0.002 grdn:0.147 lr:9.6e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 15:48:12 ts/train.py:232 step:720K smpl:12M ep:58K epch:193.28 loss:0.002 grdn:0.168 lr:9.6e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 15:49:02 ts/train.py:232 step:720K smpl:12M ep:58K epch:193.34 loss:0.002 grdn:0.138 lr:9.6e-06 updt_s:0.228 data_s:0.020 +INFO 2025-05-11 15:49:51 ts/train.py:232 step:720K smpl:12M ep:58K epch:193.39 loss:0.002 grdn:0.149 lr:9.6e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 15:49:51 ts/train.py:241 Checkpoint policy after step 720000 +INFO 2025-05-11 15:50:45 ts/train.py:232 step:720K smpl:12M ep:58K epch:193.45 loss:0.002 grdn:0.147 lr:9.5e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 15:51:34 ts/train.py:232 step:720K smpl:12M ep:58K epch:193.50 loss:0.002 grdn:0.132 lr:9.5e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 15:52:24 ts/train.py:232 step:721K smpl:12M ep:58K epch:193.55 loss:0.002 grdn:0.132 lr:9.5e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 15:53:14 ts/train.py:232 step:721K smpl:12M ep:58K epch:193.61 loss:0.002 grdn:0.128 lr:9.5e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 15:54:04 ts/train.py:232 step:721K smpl:12M ep:58K epch:193.66 loss:0.002 grdn:0.149 lr:9.5e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 15:54:53 ts/train.py:232 step:721K smpl:12M ep:58K epch:193.71 loss:0.002 grdn:0.131 lr:9.4e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 15:55:43 ts/train.py:232 step:721K smpl:12M ep:58K epch:193.77 loss:0.002 grdn:0.132 lr:9.4e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 15:56:33 ts/train.py:232 step:722K smpl:12M ep:58K epch:193.82 loss:0.002 grdn:0.145 lr:9.4e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 15:57:22 ts/train.py:232 step:722K smpl:12M ep:58K epch:193.88 loss:0.002 grdn:0.136 lr:9.4e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 15:58:14 ts/train.py:232 step:722K smpl:12M ep:58K epch:193.93 loss:0.002 grdn:0.137 lr:9.4e-06 updt_s:0.227 data_s:0.032 +INFO 2025-05-11 15:59:05 ts/train.py:232 step:722K smpl:12M ep:58K epch:193.98 loss:0.002 grdn:0.137 lr:9.3e-06 updt_s:0.228 data_s:0.025 +INFO 2025-05-11 15:59:54 ts/train.py:232 step:722K smpl:12M ep:58K epch:194.04 loss:0.002 grdn:0.167 lr:9.3e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 16:00:44 ts/train.py:232 step:723K smpl:12M ep:58K epch:194.09 loss:0.002 grdn:0.154 lr:9.3e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 16:01:34 ts/train.py:232 step:723K smpl:12M ep:58K epch:194.14 loss:0.002 grdn:0.151 lr:9.3e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:02:24 ts/train.py:232 step:723K smpl:12M ep:58K epch:194.20 loss:0.002 grdn:0.148 lr:9.3e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:03:14 ts/train.py:232 step:723K smpl:12M ep:58K epch:194.25 loss:0.002 grdn:0.126 lr:9.2e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 16:04:04 ts/train.py:232 step:723K smpl:12M ep:58K epch:194.31 loss:0.002 grdn:0.141 lr:9.2e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 16:04:55 ts/train.py:232 step:724K smpl:12M ep:58K epch:194.36 loss:0.002 grdn:0.138 lr:9.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:05:45 ts/train.py:232 step:724K smpl:12M ep:58K epch:194.41 loss:0.002 grdn:0.129 lr:9.2e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 16:06:35 ts/train.py:232 step:724K smpl:12M ep:58K epch:194.47 loss:0.002 grdn:0.145 lr:9.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:07:25 ts/train.py:232 step:724K smpl:12M ep:58K epch:194.52 loss:0.002 grdn:0.141 lr:9.1e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:08:15 ts/train.py:232 step:724K smpl:12M ep:58K epch:194.57 loss:0.002 grdn:0.154 lr:9.1e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:09:06 ts/train.py:232 step:725K smpl:12M ep:58K epch:194.63 loss:0.002 grdn:0.133 lr:9.1e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 16:09:56 ts/train.py:232 step:725K smpl:12M ep:58K epch:194.68 loss:0.002 grdn:0.151 lr:9.1e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:10:47 ts/train.py:232 step:725K smpl:12M ep:58K epch:194.74 loss:0.002 grdn:0.144 lr:9.1e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 16:11:36 ts/train.py:232 step:725K smpl:12M ep:58K epch:194.79 loss:0.002 grdn:0.164 lr:9.0e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 16:12:28 ts/train.py:232 step:725K smpl:12M ep:58K epch:194.84 loss:0.002 grdn:0.133 lr:9.0e-06 updt_s:0.227 data_s:0.029 +INFO 2025-05-11 16:13:20 ts/train.py:232 step:726K smpl:12M ep:58K epch:194.90 loss:0.002 grdn:0.139 lr:9.0e-06 updt_s:0.227 data_s:0.034 +INFO 2025-05-11 16:14:10 ts/train.py:232 step:726K smpl:12M ep:58K epch:194.95 loss:0.002 grdn:0.139 lr:9.0e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 16:15:01 ts/train.py:232 step:726K smpl:12M ep:59K epch:195.00 loss:0.002 grdn:0.154 lr:9.0e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 16:15:51 ts/train.py:232 step:726K smpl:12M ep:59K epch:195.06 loss:0.002 grdn:0.127 lr:8.9e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:16:42 ts/train.py:232 step:726K smpl:12M ep:59K epch:195.11 loss:0.002 grdn:0.127 lr:8.9e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 16:17:32 ts/train.py:232 step:727K smpl:12M ep:59K epch:195.17 loss:0.002 grdn:0.134 lr:8.9e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 16:18:23 ts/train.py:232 step:727K smpl:12M ep:59K epch:195.22 loss:0.002 grdn:0.150 lr:8.9e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 16:19:13 ts/train.py:232 step:727K smpl:12M ep:59K epch:195.27 loss:0.002 grdn:0.137 lr:8.9e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:20:04 ts/train.py:232 step:727K smpl:12M ep:59K epch:195.33 loss:0.002 grdn:0.125 lr:8.8e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 16:20:55 ts/train.py:232 step:727K smpl:12M ep:59K epch:195.38 loss:0.002 grdn:0.137 lr:8.8e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 16:21:45 ts/train.py:232 step:728K smpl:12M ep:59K epch:195.43 loss:0.002 grdn:0.148 lr:8.8e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 16:22:36 ts/train.py:232 step:728K smpl:12M ep:59K epch:195.49 loss:0.002 grdn:0.148 lr:8.8e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 16:23:26 ts/train.py:232 step:728K smpl:12M ep:59K epch:195.54 loss:0.002 grdn:0.147 lr:8.8e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 16:24:17 ts/train.py:232 step:728K smpl:12M ep:59K epch:195.59 loss:0.002 grdn:0.141 lr:8.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:25:07 ts/train.py:232 step:728K smpl:12M ep:59K epch:195.65 loss:0.002 grdn:0.142 lr:8.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:25:58 ts/train.py:232 step:729K smpl:12M ep:59K epch:195.70 loss:0.002 grdn:0.132 lr:8.7e-06 updt_s:0.228 data_s:0.027 +INFO 2025-05-11 16:26:49 ts/train.py:232 step:729K smpl:12M ep:59K epch:195.76 loss:0.002 grdn:0.150 lr:8.7e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 16:27:40 ts/train.py:232 step:729K smpl:12M ep:59K epch:195.81 loss:0.002 grdn:0.131 lr:8.7e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 16:28:32 ts/train.py:232 step:729K smpl:12M ep:59K epch:195.86 loss:0.002 grdn:0.133 lr:8.6e-06 updt_s:0.227 data_s:0.036 +INFO 2025-05-11 16:29:22 ts/train.py:232 step:729K smpl:12M ep:59K epch:195.92 loss:0.002 grdn:0.138 lr:8.6e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 16:30:12 ts/train.py:232 step:730K smpl:12M ep:59K epch:195.97 loss:0.002 grdn:0.127 lr:8.6e-06 updt_s:0.226 data_s:0.020 +INFO 2025-05-11 16:31:00 ts/train.py:232 step:730K smpl:12M ep:59K epch:196.02 loss:0.002 grdn:0.138 lr:8.6e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 16:31:50 ts/train.py:232 step:730K smpl:12M ep:59K epch:196.08 loss:0.002 grdn:0.152 lr:8.6e-06 updt_s:0.226 data_s:0.020 +INFO 2025-05-11 16:32:40 ts/train.py:232 step:730K smpl:12M ep:59K epch:196.13 loss:0.002 grdn:0.154 lr:8.5e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:33:31 ts/train.py:232 step:730K smpl:12M ep:59K epch:196.19 loss:0.002 grdn:0.147 lr:8.5e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 16:34:20 ts/train.py:232 step:731K smpl:12M ep:59K epch:196.24 loss:0.002 grdn:0.149 lr:8.5e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 16:35:11 ts/train.py:232 step:731K smpl:12M ep:59K epch:196.29 loss:0.002 grdn:0.132 lr:8.5e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:36:01 ts/train.py:232 step:731K smpl:12M ep:59K epch:196.35 loss:0.002 grdn:0.135 lr:8.5e-06 updt_s:0.229 data_s:0.022 +INFO 2025-05-11 16:36:52 ts/train.py:232 step:731K smpl:12M ep:59K epch:196.40 loss:0.002 grdn:0.139 lr:8.5e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:37:43 ts/train.py:232 step:731K smpl:12M ep:59K epch:196.45 loss:0.002 grdn:0.140 lr:8.4e-06 updt_s:0.227 data_s:0.029 +INFO 2025-05-11 16:38:32 ts/train.py:232 step:732K smpl:12M ep:59K epch:196.51 loss:0.002 grdn:0.145 lr:8.4e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 16:39:22 ts/train.py:232 step:732K smpl:12M ep:59K epch:196.56 loss:0.002 grdn:0.144 lr:8.4e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 16:40:12 ts/train.py:232 step:732K smpl:12M ep:59K epch:196.62 loss:0.002 grdn:0.139 lr:8.4e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 16:41:01 ts/train.py:232 step:732K smpl:12M ep:59K epch:196.67 loss:0.002 grdn:0.144 lr:8.4e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 16:41:51 ts/train.py:232 step:732K smpl:12M ep:59K epch:196.72 loss:0.002 grdn:0.140 lr:8.3e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:42:41 ts/train.py:232 step:733K smpl:12M ep:59K epch:196.78 loss:0.002 grdn:0.140 lr:8.3e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 16:43:34 ts/train.py:232 step:733K smpl:12M ep:59K epch:196.83 loss:0.002 grdn:0.139 lr:8.3e-06 updt_s:0.227 data_s:0.034 +INFO 2025-05-11 16:44:23 ts/train.py:232 step:733K smpl:12M ep:59K epch:196.88 loss:0.002 grdn:0.141 lr:8.3e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 16:45:12 ts/train.py:232 step:733K smpl:12M ep:59K epch:196.94 loss:0.002 grdn:0.148 lr:8.3e-06 updt_s:0.228 data_s:0.017 +INFO 2025-05-11 16:46:00 ts/train.py:232 step:733K smpl:12M ep:59K epch:196.99 loss:0.002 grdn:0.137 lr:8.2e-06 updt_s:0.227 data_s:0.014 +INFO 2025-05-11 16:46:49 ts/train.py:232 step:734K smpl:12M ep:59K epch:197.05 loss:0.002 grdn:0.135 lr:8.2e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 16:47:39 ts/train.py:232 step:734K smpl:12M ep:59K epch:197.10 loss:0.002 grdn:0.164 lr:8.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:48:30 ts/train.py:232 step:734K smpl:12M ep:59K epch:197.15 loss:0.002 grdn:0.145 lr:8.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:49:20 ts/train.py:232 step:734K smpl:12M ep:59K epch:197.21 loss:0.002 grdn:0.145 lr:8.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:50:11 ts/train.py:232 step:734K smpl:12M ep:59K epch:197.26 loss:0.002 grdn:0.147 lr:8.1e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 16:51:01 ts/train.py:232 step:735K smpl:12M ep:59K epch:197.31 loss:0.002 grdn:0.139 lr:8.1e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:51:52 ts/train.py:232 step:735K smpl:12M ep:59K epch:197.37 loss:0.002 grdn:0.142 lr:8.1e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 16:52:42 ts/train.py:232 step:735K smpl:12M ep:59K epch:197.42 loss:0.002 grdn:0.129 lr:8.1e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 16:53:32 ts/train.py:232 step:735K smpl:12M ep:59K epch:197.48 loss:0.002 grdn:0.142 lr:8.1e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:54:22 ts/train.py:232 step:735K smpl:12M ep:59K epch:197.53 loss:0.002 grdn:0.142 lr:8.0e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:55:13 ts/train.py:232 step:736K smpl:12M ep:59K epch:197.58 loss:0.002 grdn:0.134 lr:8.0e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:56:03 ts/train.py:232 step:736K smpl:12M ep:59K epch:197.64 loss:0.002 grdn:0.130 lr:8.0e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 16:56:53 ts/train.py:232 step:736K smpl:12M ep:59K epch:197.69 loss:0.002 grdn:0.130 lr:8.0e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 16:57:44 ts/train.py:232 step:736K smpl:12M ep:59K epch:197.74 loss:0.002 grdn:0.149 lr:8.0e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 16:58:36 ts/train.py:232 step:736K smpl:12M ep:59K epch:197.80 loss:0.002 grdn:0.159 lr:8.0e-06 updt_s:0.227 data_s:0.035 +INFO 2025-05-11 16:59:28 ts/train.py:232 step:737K smpl:12M ep:59K epch:197.85 loss:0.002 grdn:0.136 lr:7.9e-06 updt_s:0.226 data_s:0.031 +INFO 2025-05-11 17:00:18 ts/train.py:232 step:737K smpl:12M ep:59K epch:197.90 loss:0.002 grdn:0.145 lr:7.9e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 17:01:08 ts/train.py:232 step:737K smpl:12M ep:59K epch:197.96 loss:0.002 grdn:0.152 lr:7.9e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 17:01:58 ts/train.py:232 step:737K smpl:12M ep:59K epch:198.01 loss:0.002 grdn:0.126 lr:7.9e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 17:02:49 ts/train.py:232 step:737K smpl:12M ep:59K epch:198.07 loss:0.002 grdn:0.126 lr:7.9e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 17:03:41 ts/train.py:232 step:738K smpl:12M ep:59K epch:198.12 loss:0.002 grdn:0.137 lr:7.8e-06 updt_s:0.227 data_s:0.030 +INFO 2025-05-11 17:04:31 ts/train.py:232 step:738K smpl:12M ep:59K epch:198.17 loss:0.002 grdn:0.126 lr:7.8e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 17:05:22 ts/train.py:232 step:738K smpl:12M ep:59K epch:198.23 loss:0.002 grdn:0.131 lr:7.8e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 17:06:13 ts/train.py:232 step:738K smpl:12M ep:59K epch:198.28 loss:0.002 grdn:0.151 lr:7.8e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 17:07:04 ts/train.py:232 step:738K smpl:12M ep:60K epch:198.33 loss:0.002 grdn:0.146 lr:7.8e-06 updt_s:0.226 data_s:0.029 +INFO 2025-05-11 17:07:56 ts/train.py:232 step:739K smpl:12M ep:60K epch:198.39 loss:0.002 grdn:0.139 lr:7.7e-06 updt_s:0.227 data_s:0.030 +INFO 2025-05-11 17:08:47 ts/train.py:232 step:739K smpl:12M ep:60K epch:198.44 loss:0.002 grdn:0.154 lr:7.7e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 17:09:38 ts/train.py:232 step:739K smpl:12M ep:60K epch:198.50 loss:0.002 grdn:0.154 lr:7.7e-06 updt_s:0.227 data_s:0.029 +INFO 2025-05-11 17:10:29 ts/train.py:232 step:739K smpl:12M ep:60K epch:198.55 loss:0.002 grdn:0.144 lr:7.7e-06 updt_s:0.227 data_s:0.029 +INFO 2025-05-11 17:11:20 ts/train.py:232 step:739K smpl:12M ep:60K epch:198.60 loss:0.002 grdn:0.137 lr:7.7e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 17:12:11 ts/train.py:232 step:740K smpl:12M ep:60K epch:198.66 loss:0.002 grdn:0.146 lr:7.7e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 17:13:02 ts/train.py:232 step:740K smpl:12M ep:60K epch:198.71 loss:0.002 grdn:0.168 lr:7.6e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 17:13:57 ts/train.py:232 step:740K smpl:12M ep:60K epch:198.76 loss:0.002 grdn:0.137 lr:7.6e-06 updt_s:0.228 data_s:0.048 +INFO 2025-05-11 17:14:47 ts/train.py:232 step:740K smpl:12M ep:60K epch:198.82 loss:0.002 grdn:0.148 lr:7.6e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 17:15:36 ts/train.py:232 step:740K smpl:12M ep:60K epch:198.87 loss:0.002 grdn:0.152 lr:7.6e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 17:16:25 ts/train.py:232 step:741K smpl:12M ep:60K epch:198.93 loss:0.002 grdn:0.137 lr:7.6e-06 updt_s:0.228 data_s:0.016 +INFO 2025-05-11 17:17:15 ts/train.py:232 step:741K smpl:12M ep:60K epch:198.98 loss:0.002 grdn:0.141 lr:7.5e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 17:18:06 ts/train.py:232 step:741K smpl:12M ep:60K epch:199.03 loss:0.002 grdn:0.129 lr:7.5e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 17:18:56 ts/train.py:232 step:741K smpl:12M ep:60K epch:199.09 loss:0.002 grdn:0.136 lr:7.5e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 17:19:46 ts/train.py:232 step:741K smpl:12M ep:60K epch:199.14 loss:0.002 grdn:0.133 lr:7.5e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 17:20:37 ts/train.py:232 step:742K smpl:12M ep:60K epch:199.19 loss:0.002 grdn:0.143 lr:7.5e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 17:21:27 ts/train.py:232 step:742K smpl:12M ep:60K epch:199.25 loss:0.002 grdn:0.129 lr:7.4e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 17:22:17 ts/train.py:232 step:742K smpl:12M ep:60K epch:199.30 loss:0.002 grdn:0.123 lr:7.4e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 17:23:08 ts/train.py:232 step:742K smpl:12M ep:60K epch:199.36 loss:0.002 grdn:0.128 lr:7.4e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 17:23:58 ts/train.py:232 step:742K smpl:12M ep:60K epch:199.41 loss:0.002 grdn:0.153 lr:7.4e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 17:24:49 ts/train.py:232 step:743K smpl:12M ep:60K epch:199.46 loss:0.002 grdn:0.133 lr:7.4e-06 updt_s:0.228 data_s:0.025 +INFO 2025-05-11 17:25:39 ts/train.py:232 step:743K smpl:12M ep:60K epch:199.52 loss:0.002 grdn:0.142 lr:7.4e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 17:26:30 ts/train.py:232 step:743K smpl:12M ep:60K epch:199.57 loss:0.002 grdn:0.147 lr:7.3e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 17:27:21 ts/train.py:232 step:743K smpl:12M ep:60K epch:199.62 loss:0.002 grdn:0.142 lr:7.3e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 17:28:11 ts/train.py:232 step:743K smpl:12M ep:60K epch:199.68 loss:0.002 grdn:0.149 lr:7.3e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 17:29:03 ts/train.py:232 step:744K smpl:12M ep:60K epch:199.73 loss:0.002 grdn:0.135 lr:7.3e-06 updt_s:0.227 data_s:0.031 +INFO 2025-05-11 17:29:53 ts/train.py:232 step:744K smpl:12M ep:60K epch:199.79 loss:0.002 grdn:0.143 lr:7.3e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 17:30:42 ts/train.py:232 step:744K smpl:12M ep:60K epch:199.84 loss:0.002 grdn:0.140 lr:7.2e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 17:31:31 ts/train.py:232 step:744K smpl:12M ep:60K epch:199.89 loss:0.002 grdn:0.132 lr:7.2e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 17:32:21 ts/train.py:232 step:744K smpl:12M ep:60K epch:199.95 loss:0.002 grdn:0.139 lr:7.2e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 17:33:11 ts/train.py:232 step:745K smpl:12M ep:60K epch:200.00 loss:0.002 grdn:0.153 lr:7.2e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 17:34:01 ts/train.py:232 step:745K smpl:12M ep:60K epch:200.05 loss:0.002 grdn:0.149 lr:7.2e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 17:34:50 ts/train.py:232 step:745K smpl:12M ep:60K epch:200.11 loss:0.002 grdn:0.148 lr:7.2e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 17:35:40 ts/train.py:232 step:745K smpl:12M ep:60K epch:200.16 loss:0.002 grdn:0.126 lr:7.1e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 17:36:30 ts/train.py:232 step:745K smpl:12M ep:60K epch:200.21 loss:0.002 grdn:0.137 lr:7.1e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 17:37:19 ts/train.py:232 step:746K smpl:12M ep:60K epch:200.27 loss:0.002 grdn:0.146 lr:7.1e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 17:38:09 ts/train.py:232 step:746K smpl:12M ep:60K epch:200.32 loss:0.002 grdn:0.138 lr:7.1e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 17:38:59 ts/train.py:232 step:746K smpl:12M ep:60K epch:200.38 loss:0.002 grdn:0.142 lr:7.1e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 17:39:49 ts/train.py:232 step:746K smpl:12M ep:60K epch:200.43 loss:0.002 grdn:0.152 lr:7.1e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 17:40:39 ts/train.py:232 step:746K smpl:12M ep:60K epch:200.48 loss:0.002 grdn:0.122 lr:7.0e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 17:41:28 ts/train.py:232 step:747K smpl:12M ep:60K epch:200.54 loss:0.002 grdn:0.116 lr:7.0e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 17:42:18 ts/train.py:232 step:747K smpl:12M ep:60K epch:200.59 loss:0.002 grdn:0.138 lr:7.0e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 17:43:08 ts/train.py:232 step:747K smpl:12M ep:60K epch:200.64 loss:0.002 grdn:0.127 lr:7.0e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 17:44:00 ts/train.py:232 step:747K smpl:12M ep:60K epch:200.70 loss:0.002 grdn:0.123 lr:7.0e-06 updt_s:0.227 data_s:0.032 +INFO 2025-05-11 17:44:49 ts/train.py:232 step:747K smpl:12M ep:60K epch:200.75 loss:0.002 grdn:0.145 lr:6.9e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 17:45:39 ts/train.py:232 step:748K smpl:12M ep:60K epch:200.81 loss:0.002 grdn:0.132 lr:6.9e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 17:46:28 ts/train.py:232 step:748K smpl:12M ep:60K epch:200.86 loss:0.001 grdn:0.121 lr:6.9e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 17:47:18 ts/train.py:232 step:748K smpl:12M ep:60K epch:200.91 loss:0.002 grdn:0.131 lr:6.9e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 17:48:08 ts/train.py:232 step:748K smpl:12M ep:60K epch:200.97 loss:0.002 grdn:0.145 lr:6.9e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 17:48:58 ts/train.py:232 step:748K smpl:12M ep:60K epch:201.02 loss:0.002 grdn:0.147 lr:6.9e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 17:49:48 ts/train.py:232 step:749K smpl:12M ep:60K epch:201.07 loss:0.002 grdn:0.136 lr:6.8e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 17:50:38 ts/train.py:232 step:749K smpl:12M ep:60K epch:201.13 loss:0.002 grdn:0.135 lr:6.8e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 17:51:29 ts/train.py:232 step:749K smpl:12M ep:60K epch:201.18 loss:0.002 grdn:0.136 lr:6.8e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 17:52:19 ts/train.py:232 step:749K smpl:12M ep:60K epch:201.24 loss:0.001 grdn:0.111 lr:6.8e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 17:53:08 ts/train.py:232 step:749K smpl:12M ep:60K epch:201.29 loss:0.002 grdn:0.130 lr:6.8e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 17:53:59 ts/train.py:232 step:750K smpl:12M ep:60K epch:201.34 loss:0.002 grdn:0.144 lr:6.7e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 17:54:49 ts/train.py:232 step:750K smpl:12M ep:60K epch:201.40 loss:0.002 grdn:0.130 lr:6.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 17:55:40 ts/train.py:232 step:750K smpl:12M ep:60K epch:201.45 loss:0.002 grdn:0.133 lr:6.7e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 17:55:40 ts/train.py:241 Checkpoint policy after step 750000 +INFO 2025-05-11 17:56:35 ts/train.py:232 step:750K smpl:12M ep:60K epch:201.50 loss:0.002 grdn:0.152 lr:6.7e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 17:57:25 ts/train.py:232 step:750K smpl:12M ep:60K epch:201.56 loss:0.002 grdn:0.144 lr:6.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 17:58:15 ts/train.py:232 step:751K smpl:12M ep:60K epch:201.61 loss:0.002 grdn:0.131 lr:6.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 17:59:07 ts/train.py:232 step:751K smpl:12M ep:60K epch:201.67 loss:0.002 grdn:0.148 lr:6.6e-06 updt_s:0.227 data_s:0.031 +INFO 2025-05-11 17:59:57 ts/train.py:232 step:751K smpl:12M ep:61K epch:201.72 loss:0.001 grdn:0.122 lr:6.6e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 18:00:46 ts/train.py:232 step:751K smpl:12M ep:61K epch:201.77 loss:0.002 grdn:0.126 lr:6.6e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 18:01:35 ts/train.py:232 step:751K smpl:12M ep:61K epch:201.83 loss:0.001 grdn:0.126 lr:6.6e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 18:02:24 ts/train.py:232 step:752K smpl:12M ep:61K epch:201.88 loss:0.001 grdn:0.124 lr:6.6e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 18:03:13 ts/train.py:232 step:752K smpl:12M ep:61K epch:201.93 loss:0.002 grdn:0.133 lr:6.6e-06 updt_s:0.228 data_s:0.018 +INFO 2025-05-11 18:04:03 ts/train.py:232 step:752K smpl:12M ep:61K epch:201.99 loss:0.002 grdn:0.133 lr:6.5e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 18:04:53 ts/train.py:232 step:752K smpl:12M ep:61K epch:202.04 loss:0.002 grdn:0.136 lr:6.5e-06 updt_s:0.228 data_s:0.020 +INFO 2025-05-11 18:05:42 ts/train.py:232 step:752K smpl:12M ep:61K epch:202.10 loss:0.002 grdn:0.139 lr:6.5e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 18:06:32 ts/train.py:232 step:753K smpl:12M ep:61K epch:202.15 loss:0.001 grdn:0.123 lr:6.5e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 18:07:21 ts/train.py:232 step:753K smpl:12M ep:61K epch:202.20 loss:0.002 grdn:0.144 lr:6.5e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 18:08:11 ts/train.py:232 step:753K smpl:12M ep:61K epch:202.26 loss:0.002 grdn:0.143 lr:6.5e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 18:09:00 ts/train.py:232 step:753K smpl:12M ep:61K epch:202.31 loss:0.002 grdn:0.153 lr:6.4e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 18:09:50 ts/train.py:232 step:753K smpl:12M ep:61K epch:202.36 loss:0.002 grdn:0.132 lr:6.4e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:10:40 ts/train.py:232 step:754K smpl:12M ep:61K epch:202.42 loss:0.002 grdn:0.137 lr:6.4e-06 updt_s:0.228 data_s:0.018 +INFO 2025-05-11 18:11:29 ts/train.py:232 step:754K smpl:12M ep:61K epch:202.47 loss:0.002 grdn:0.128 lr:6.4e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 18:12:18 ts/train.py:232 step:754K smpl:12M ep:61K epch:202.52 loss:0.002 grdn:0.138 lr:6.4e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 18:13:08 ts/train.py:232 step:754K smpl:12M ep:61K epch:202.58 loss:0.002 grdn:0.135 lr:6.4e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 18:14:00 ts/train.py:232 step:754K smpl:12M ep:61K epch:202.63 loss:0.002 grdn:0.133 lr:6.3e-06 updt_s:0.227 data_s:0.031 +INFO 2025-05-11 18:14:50 ts/train.py:232 step:755K smpl:12M ep:61K epch:202.69 loss:0.002 grdn:0.146 lr:6.3e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 18:15:40 ts/train.py:232 step:755K smpl:12M ep:61K epch:202.74 loss:0.001 grdn:0.119 lr:6.3e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 18:16:30 ts/train.py:232 step:755K smpl:12M ep:61K epch:202.79 loss:0.002 grdn:0.136 lr:6.3e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 18:17:20 ts/train.py:232 step:755K smpl:12M ep:61K epch:202.85 loss:0.002 grdn:0.140 lr:6.3e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 18:18:10 ts/train.py:232 step:755K smpl:12M ep:61K epch:202.90 loss:0.002 grdn:0.134 lr:6.3e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 18:19:00 ts/train.py:232 step:756K smpl:12M ep:61K epch:202.95 loss:0.002 grdn:0.143 lr:6.2e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:19:51 ts/train.py:232 step:756K smpl:12M ep:61K epch:203.01 loss:0.002 grdn:0.129 lr:6.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:20:40 ts/train.py:232 step:756K smpl:12M ep:61K epch:203.06 loss:0.002 grdn:0.132 lr:6.2e-06 updt_s:0.228 data_s:0.020 +INFO 2025-05-11 18:21:31 ts/train.py:232 step:756K smpl:12M ep:61K epch:203.12 loss:0.002 grdn:0.131 lr:6.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:22:21 ts/train.py:232 step:756K smpl:12M ep:61K epch:203.17 loss:0.002 grdn:0.130 lr:6.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:23:11 ts/train.py:232 step:757K smpl:12M ep:61K epch:203.22 loss:0.002 grdn:0.135 lr:6.1e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:24:01 ts/train.py:232 step:757K smpl:12M ep:61K epch:203.28 loss:0.002 grdn:0.154 lr:6.1e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:24:51 ts/train.py:232 step:757K smpl:12M ep:61K epch:203.33 loss:0.002 grdn:0.129 lr:6.1e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:25:42 ts/train.py:232 step:757K smpl:12M ep:61K epch:203.38 loss:0.002 grdn:0.137 lr:6.1e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 18:26:33 ts/train.py:232 step:757K smpl:12M ep:61K epch:203.44 loss:0.002 grdn:0.134 lr:6.1e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:27:22 ts/train.py:232 step:758K smpl:12M ep:61K epch:203.49 loss:0.001 grdn:0.117 lr:6.1e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 18:28:13 ts/train.py:232 step:758K smpl:12M ep:61K epch:203.55 loss:0.001 grdn:0.113 lr:6.0e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 18:29:06 ts/train.py:232 step:758K smpl:12M ep:61K epch:203.60 loss:0.002 grdn:0.136 lr:6.0e-06 updt_s:0.227 data_s:0.038 +INFO 2025-05-11 18:29:57 ts/train.py:232 step:758K smpl:12M ep:61K epch:203.65 loss:0.002 grdn:0.132 lr:6.0e-06 updt_s:0.226 data_s:0.025 +INFO 2025-05-11 18:30:48 ts/train.py:232 step:758K smpl:12M ep:61K epch:203.71 loss:0.002 grdn:0.132 lr:6.0e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 18:31:38 ts/train.py:232 step:759K smpl:12M ep:61K epch:203.76 loss:0.002 grdn:0.135 lr:6.0e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 18:32:29 ts/train.py:232 step:759K smpl:12M ep:61K epch:203.81 loss:0.002 grdn:0.138 lr:6.0e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 18:33:21 ts/train.py:232 step:759K smpl:12M ep:61K epch:203.87 loss:0.002 grdn:0.143 lr:5.9e-06 updt_s:0.226 data_s:0.031 +INFO 2025-05-11 18:34:11 ts/train.py:232 step:759K smpl:12M ep:61K epch:203.92 loss:0.002 grdn:0.137 lr:5.9e-06 updt_s:0.226 data_s:0.028 +INFO 2025-05-11 18:35:04 ts/train.py:232 step:759K smpl:12M ep:61K epch:203.98 loss:0.002 grdn:0.143 lr:5.9e-06 updt_s:0.226 data_s:0.034 +INFO 2025-05-11 18:35:55 ts/train.py:232 step:760K smpl:12M ep:61K epch:204.03 loss:0.002 grdn:0.127 lr:5.9e-06 updt_s:0.226 data_s:0.028 +INFO 2025-05-11 18:36:46 ts/train.py:232 step:760K smpl:12M ep:61K epch:204.08 loss:0.002 grdn:0.134 lr:5.9e-06 updt_s:0.226 data_s:0.030 +INFO 2025-05-11 18:37:37 ts/train.py:232 step:760K smpl:12M ep:61K epch:204.14 loss:0.002 grdn:0.140 lr:5.9e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 18:38:28 ts/train.py:232 step:760K smpl:12M ep:61K epch:204.19 loss:0.002 grdn:0.145 lr:5.9e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 18:39:20 ts/train.py:232 step:760K smpl:12M ep:61K epch:204.24 loss:0.001 grdn:0.123 lr:5.8e-06 updt_s:0.227 data_s:0.030 +INFO 2025-05-11 18:40:10 ts/train.py:232 step:761K smpl:12M ep:61K epch:204.30 loss:0.002 grdn:0.135 lr:5.8e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:41:01 ts/train.py:232 step:761K smpl:12M ep:61K epch:204.35 loss:0.002 grdn:0.136 lr:5.8e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 18:41:52 ts/train.py:232 step:761K smpl:12M ep:61K epch:204.41 loss:0.002 grdn:0.137 lr:5.8e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 18:42:42 ts/train.py:232 step:761K smpl:12M ep:61K epch:204.46 loss:0.002 grdn:0.129 lr:5.8e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:43:33 ts/train.py:232 step:761K smpl:12M ep:61K epch:204.51 loss:0.002 grdn:0.147 lr:5.8e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 18:44:25 ts/train.py:232 step:762K smpl:12M ep:61K epch:204.57 loss:0.002 grdn:0.130 lr:5.7e-06 updt_s:0.227 data_s:0.037 +INFO 2025-05-11 18:45:16 ts/train.py:232 step:762K smpl:12M ep:61K epch:204.62 loss:0.001 grdn:0.117 lr:5.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:46:06 ts/train.py:232 step:762K smpl:12M ep:61K epch:204.67 loss:0.002 grdn:0.136 lr:5.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:46:56 ts/train.py:232 step:762K smpl:12M ep:61K epch:204.73 loss:0.001 grdn:0.125 lr:5.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:47:47 ts/train.py:232 step:762K smpl:12M ep:61K epch:204.78 loss:0.002 grdn:0.147 lr:5.7e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 18:48:37 ts/train.py:232 step:763K smpl:12M ep:61K epch:204.83 loss:0.002 grdn:0.130 lr:5.7e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 18:49:27 ts/train.py:232 step:763K smpl:12M ep:61K epch:204.89 loss:0.002 grdn:0.136 lr:5.6e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:50:17 ts/train.py:232 step:763K smpl:12M ep:61K epch:204.94 loss:0.002 grdn:0.130 lr:5.6e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 18:51:08 ts/train.py:232 step:763K smpl:12M ep:61K epch:205.00 loss:0.002 grdn:0.129 lr:5.6e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 18:51:58 ts/train.py:232 step:763K smpl:12M ep:62K epch:205.05 loss:0.002 grdn:0.150 lr:5.6e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:52:48 ts/train.py:232 step:764K smpl:12M ep:62K epch:205.10 loss:0.002 grdn:0.132 lr:5.6e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:53:39 ts/train.py:232 step:764K smpl:12M ep:62K epch:205.16 loss:0.002 grdn:0.133 lr:5.6e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:54:29 ts/train.py:232 step:764K smpl:12M ep:62K epch:205.21 loss:0.001 grdn:0.125 lr:5.5e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:55:19 ts/train.py:232 step:764K smpl:12M ep:62K epch:205.26 loss:0.002 grdn:0.139 lr:5.5e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:56:09 ts/train.py:232 step:764K smpl:12M ep:62K epch:205.32 loss:0.002 grdn:0.139 lr:5.5e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 18:57:00 ts/train.py:232 step:765K smpl:12M ep:62K epch:205.37 loss:0.002 grdn:0.151 lr:5.5e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 18:57:50 ts/train.py:232 step:765K smpl:12M ep:62K epch:205.43 loss:0.002 grdn:0.131 lr:5.5e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 18:58:41 ts/train.py:232 step:765K smpl:12M ep:62K epch:205.48 loss:0.001 grdn:0.127 lr:5.5e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 18:59:35 ts/train.py:232 step:765K smpl:12M ep:62K epch:205.53 loss:0.001 grdn:0.113 lr:5.4e-06 updt_s:0.226 data_s:0.041 +INFO 2025-05-11 19:00:25 ts/train.py:232 step:765K smpl:12M ep:62K epch:205.59 loss:0.001 grdn:0.123 lr:5.4e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 19:01:16 ts/train.py:232 step:766K smpl:12M ep:62K epch:205.64 loss:0.002 grdn:0.145 lr:5.4e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 19:02:06 ts/train.py:232 step:766K smpl:12M ep:62K epch:205.69 loss:0.002 grdn:0.129 lr:5.4e-06 updt_s:0.226 data_s:0.024 +INFO 2025-05-11 19:02:58 ts/train.py:232 step:766K smpl:12M ep:62K epch:205.75 loss:0.002 grdn:0.131 lr:5.4e-06 updt_s:0.227 data_s:0.033 +INFO 2025-05-11 19:03:49 ts/train.py:232 step:766K smpl:12M ep:62K epch:205.80 loss:0.002 grdn:0.129 lr:5.4e-06 updt_s:0.228 data_s:0.026 +INFO 2025-05-11 19:04:40 ts/train.py:232 step:766K smpl:12M ep:62K epch:205.86 loss:0.001 grdn:0.127 lr:5.4e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 19:05:30 ts/train.py:232 step:767K smpl:12M ep:62K epch:205.91 loss:0.002 grdn:0.147 lr:5.3e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 19:06:20 ts/train.py:232 step:767K smpl:12M ep:62K epch:205.96 loss:0.002 grdn:0.133 lr:5.3e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 19:07:10 ts/train.py:232 step:767K smpl:12M ep:62K epch:206.02 loss:0.002 grdn:0.132 lr:5.3e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 19:08:00 ts/train.py:232 step:767K smpl:12M ep:62K epch:206.07 loss:0.002 grdn:0.130 lr:5.3e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 19:08:50 ts/train.py:232 step:767K smpl:12M ep:62K epch:206.12 loss:0.002 grdn:0.144 lr:5.3e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 19:09:41 ts/train.py:232 step:768K smpl:12M ep:62K epch:206.18 loss:0.001 grdn:0.128 lr:5.3e-06 updt_s:0.229 data_s:0.022 +INFO 2025-05-11 19:10:31 ts/train.py:232 step:768K smpl:12M ep:62K epch:206.23 loss:0.001 grdn:0.117 lr:5.2e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 19:11:21 ts/train.py:232 step:768K smpl:12M ep:62K epch:206.29 loss:0.002 grdn:0.140 lr:5.2e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 19:12:11 ts/train.py:232 step:768K smpl:12M ep:62K epch:206.34 loss:0.002 grdn:0.140 lr:5.2e-06 updt_s:0.228 data_s:0.021 +INFO 2025-05-11 19:13:01 ts/train.py:232 step:768K smpl:12M ep:62K epch:206.39 loss:0.002 grdn:0.133 lr:5.2e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 19:13:52 ts/train.py:232 step:769K smpl:12M ep:62K epch:206.45 loss:0.002 grdn:0.133 lr:5.2e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 19:14:44 ts/train.py:232 step:769K smpl:12M ep:62K epch:206.50 loss:0.001 grdn:0.124 lr:5.2e-06 updt_s:0.227 data_s:0.036 +INFO 2025-05-11 19:15:34 ts/train.py:232 step:769K smpl:12M ep:62K epch:206.55 loss:0.002 grdn:0.138 lr:5.2e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 19:16:22 ts/train.py:232 step:769K smpl:12M ep:62K epch:206.61 loss:0.002 grdn:0.153 lr:5.1e-06 updt_s:0.228 data_s:0.015 +INFO 2025-05-11 19:17:12 ts/train.py:232 step:769K smpl:12M ep:62K epch:206.66 loss:0.001 grdn:0.125 lr:5.1e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 19:18:02 ts/train.py:232 step:770K smpl:12M ep:62K epch:206.72 loss:0.002 grdn:0.140 lr:5.1e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 19:18:51 ts/train.py:232 step:770K smpl:12M ep:62K epch:206.77 loss:0.002 grdn:0.146 lr:5.1e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 19:19:42 ts/train.py:232 step:770K smpl:12M ep:62K epch:206.82 loss:0.002 grdn:0.150 lr:5.1e-06 updt_s:0.229 data_s:0.022 +INFO 2025-05-11 19:20:32 ts/train.py:232 step:770K smpl:12M ep:62K epch:206.88 loss:0.001 grdn:0.125 lr:5.1e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 19:21:22 ts/train.py:232 step:770K smpl:12M ep:62K epch:206.93 loss:0.002 grdn:0.134 lr:5.0e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 19:22:12 ts/train.py:232 step:771K smpl:12M ep:62K epch:206.98 loss:0.002 grdn:0.132 lr:5.0e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 19:23:02 ts/train.py:232 step:771K smpl:12M ep:62K epch:207.04 loss:0.002 grdn:0.136 lr:5.0e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 19:23:52 ts/train.py:232 step:771K smpl:12M ep:62K epch:207.09 loss:0.002 grdn:0.132 lr:5.0e-06 updt_s:0.228 data_s:0.019 +INFO 2025-05-11 19:24:42 ts/train.py:232 step:771K smpl:12M ep:62K epch:207.14 loss:0.001 grdn:0.117 lr:5.0e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 19:25:31 ts/train.py:232 step:771K smpl:12M ep:62K epch:207.20 loss:0.002 grdn:0.132 lr:5.0e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 19:26:22 ts/train.py:232 step:772K smpl:12M ep:62K epch:207.25 loss:0.002 grdn:0.138 lr:5.0e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 19:27:12 ts/train.py:232 step:772K smpl:12M ep:62K epch:207.31 loss:0.001 grdn:0.123 lr:4.9e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 19:28:03 ts/train.py:232 step:772K smpl:12M ep:62K epch:207.36 loss:0.001 grdn:0.119 lr:4.9e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 19:28:53 ts/train.py:232 step:772K smpl:12M ep:62K epch:207.41 loss:0.001 grdn:0.131 lr:4.9e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 19:29:46 ts/train.py:232 step:772K smpl:12M ep:62K epch:207.47 loss:0.002 grdn:0.127 lr:4.9e-06 updt_s:0.228 data_s:0.036 +INFO 2025-05-11 19:30:37 ts/train.py:232 step:773K smpl:12M ep:62K epch:207.52 loss:0.001 grdn:0.119 lr:4.9e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 19:31:28 ts/train.py:232 step:773K smpl:12M ep:62K epch:207.57 loss:0.002 grdn:0.160 lr:4.9e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 19:32:18 ts/train.py:232 step:773K smpl:12M ep:62K epch:207.63 loss:0.001 grdn:0.125 lr:4.8e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 19:33:10 ts/train.py:232 step:773K smpl:12M ep:62K epch:207.68 loss:0.002 grdn:0.137 lr:4.8e-06 updt_s:0.227 data_s:0.029 +INFO 2025-05-11 19:34:00 ts/train.py:232 step:773K smpl:12M ep:62K epch:207.74 loss:0.002 grdn:0.135 lr:4.8e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 19:34:51 ts/train.py:232 step:774K smpl:12M ep:62K epch:207.79 loss:0.002 grdn:0.126 lr:4.8e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 19:35:41 ts/train.py:232 step:774K smpl:12M ep:62K epch:207.84 loss:0.002 grdn:0.129 lr:4.8e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 19:36:32 ts/train.py:232 step:774K smpl:12M ep:62K epch:207.90 loss:0.001 grdn:0.132 lr:4.8e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 19:37:22 ts/train.py:232 step:774K smpl:12M ep:62K epch:207.95 loss:0.001 grdn:0.121 lr:4.8e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 19:38:13 ts/train.py:232 step:774K smpl:12M ep:62K epch:208.00 loss:0.002 grdn:0.130 lr:4.7e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 19:39:04 ts/train.py:232 step:775K smpl:12M ep:62K epch:208.06 loss:0.002 grdn:0.136 lr:4.7e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 19:39:55 ts/train.py:232 step:775K smpl:12M ep:62K epch:208.11 loss:0.002 grdn:0.131 lr:4.7e-06 updt_s:0.228 data_s:0.026 +INFO 2025-05-11 19:40:45 ts/train.py:232 step:775K smpl:12M ep:62K epch:208.17 loss:0.001 grdn:0.124 lr:4.7e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 19:41:36 ts/train.py:232 step:775K smpl:12M ep:62K epch:208.22 loss:0.002 grdn:0.145 lr:4.7e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 19:42:26 ts/train.py:232 step:775K smpl:12M ep:62K epch:208.27 loss:0.001 grdn:0.128 lr:4.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 19:43:16 ts/train.py:232 step:776K smpl:12M ep:62K epch:208.33 loss:0.001 grdn:0.122 lr:4.7e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 19:44:06 ts/train.py:232 step:776K smpl:12M ep:63K epch:208.38 loss:0.001 grdn:0.126 lr:4.6e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 19:44:59 ts/train.py:232 step:776K smpl:12M ep:63K epch:208.43 loss:0.002 grdn:0.135 lr:4.6e-06 updt_s:0.227 data_s:0.039 +INFO 2025-05-11 19:45:49 ts/train.py:232 step:776K smpl:12M ep:63K epch:208.49 loss:0.002 grdn:0.132 lr:4.6e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 19:46:40 ts/train.py:232 step:776K smpl:12M ep:63K epch:208.54 loss:0.001 grdn:0.130 lr:4.6e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 19:47:30 ts/train.py:232 step:777K smpl:12M ep:63K epch:208.60 loss:0.001 grdn:0.118 lr:4.6e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 19:48:21 ts/train.py:232 step:777K smpl:12M ep:63K epch:208.65 loss:0.001 grdn:0.121 lr:4.6e-06 updt_s:0.227 data_s:0.029 +INFO 2025-05-11 19:49:12 ts/train.py:232 step:777K smpl:12M ep:63K epch:208.70 loss:0.001 grdn:0.123 lr:4.6e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 19:50:05 ts/train.py:232 step:777K smpl:12M ep:63K epch:208.76 loss:0.001 grdn:0.121 lr:4.5e-06 updt_s:0.227 data_s:0.037 +INFO 2025-05-11 19:50:56 ts/train.py:232 step:777K smpl:12M ep:63K epch:208.81 loss:0.002 grdn:0.134 lr:4.5e-06 updt_s:0.228 data_s:0.028 +INFO 2025-05-11 19:51:47 ts/train.py:232 step:778K smpl:12M ep:63K epch:208.86 loss:0.002 grdn:0.136 lr:4.5e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 19:52:38 ts/train.py:232 step:778K smpl:12M ep:63K epch:208.92 loss:0.001 grdn:0.128 lr:4.5e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 19:53:29 ts/train.py:232 step:778K smpl:12M ep:63K epch:208.97 loss:0.001 grdn:0.128 lr:4.5e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 19:54:20 ts/train.py:232 step:778K smpl:12M ep:63K epch:209.02 loss:0.001 grdn:0.129 lr:4.5e-06 updt_s:0.227 data_s:0.029 +INFO 2025-05-11 19:55:11 ts/train.py:232 step:778K smpl:12M ep:63K epch:209.08 loss:0.001 grdn:0.128 lr:4.4e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 19:56:02 ts/train.py:232 step:779K smpl:12M ep:63K epch:209.13 loss:0.002 grdn:0.148 lr:4.4e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 19:56:53 ts/train.py:232 step:779K smpl:12M ep:63K epch:209.19 loss:0.001 grdn:0.118 lr:4.4e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 19:57:44 ts/train.py:232 step:779K smpl:12M ep:63K epch:209.24 loss:0.002 grdn:0.141 lr:4.4e-06 updt_s:0.228 data_s:0.027 +INFO 2025-05-11 19:58:36 ts/train.py:232 step:779K smpl:12M ep:63K epch:209.29 loss:0.002 grdn:0.143 lr:4.4e-06 updt_s:0.227 data_s:0.032 +INFO 2025-05-11 19:59:27 ts/train.py:232 step:779K smpl:12M ep:63K epch:209.35 loss:0.002 grdn:0.148 lr:4.4e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:00:21 ts/train.py:232 step:780K smpl:12M ep:63K epch:209.40 loss:0.001 grdn:0.123 lr:4.4e-06 updt_s:0.226 data_s:0.040 +INFO 2025-05-11 20:01:12 ts/train.py:232 step:780K smpl:12M ep:63K epch:209.45 loss:0.001 grdn:0.132 lr:4.3e-06 updt_s:0.226 data_s:0.027 +INFO 2025-05-11 20:02:02 ts/train.py:232 step:780K smpl:12M ep:63K epch:209.51 loss:0.001 grdn:0.126 lr:4.3e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 20:02:02 ts/train.py:241 Checkpoint policy after step 780000 +INFO 2025-05-11 20:02:57 ts/train.py:232 step:780K smpl:12M ep:63K epch:209.56 loss:0.001 grdn:0.121 lr:4.3e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:03:47 ts/train.py:232 step:780K smpl:12M ep:63K epch:209.62 loss:0.001 grdn:0.123 lr:4.3e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 20:04:38 ts/train.py:232 step:781K smpl:12M ep:63K epch:209.67 loss:0.002 grdn:0.141 lr:4.3e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 20:05:30 ts/train.py:232 step:781K smpl:12M ep:63K epch:209.72 loss:0.001 grdn:0.123 lr:4.3e-06 updt_s:0.227 data_s:0.030 +INFO 2025-05-11 20:06:21 ts/train.py:232 step:781K smpl:12M ep:63K epch:209.78 loss:0.002 grdn:0.147 lr:4.3e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 20:07:11 ts/train.py:232 step:781K smpl:12M ep:63K epch:209.83 loss:0.002 grdn:0.132 lr:4.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 20:08:01 ts/train.py:232 step:781K smpl:13M ep:63K epch:209.88 loss:0.001 grdn:0.114 lr:4.2e-06 updt_s:0.229 data_s:0.023 +INFO 2025-05-11 20:08:52 ts/train.py:232 step:782K smpl:13M ep:63K epch:209.94 loss:0.002 grdn:0.152 lr:4.2e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:09:43 ts/train.py:232 step:782K smpl:13M ep:63K epch:209.99 loss:0.001 grdn:0.128 lr:4.2e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:10:34 ts/train.py:232 step:782K smpl:13M ep:63K epch:210.05 loss:0.002 grdn:0.139 lr:4.2e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:11:25 ts/train.py:232 step:782K smpl:13M ep:63K epch:210.10 loss:0.001 grdn:0.134 lr:4.2e-06 updt_s:0.227 data_s:0.029 +INFO 2025-05-11 20:12:16 ts/train.py:232 step:782K smpl:13M ep:63K epch:210.15 loss:0.002 grdn:0.139 lr:4.2e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:13:06 ts/train.py:232 step:783K smpl:13M ep:63K epch:210.21 loss:0.002 grdn:0.133 lr:4.2e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:13:57 ts/train.py:232 step:783K smpl:13M ep:63K epch:210.26 loss:0.002 grdn:0.142 lr:4.1e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 20:14:48 ts/train.py:232 step:783K smpl:13M ep:63K epch:210.31 loss:0.001 grdn:0.130 lr:4.1e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:15:41 ts/train.py:232 step:783K smpl:13M ep:63K epch:210.37 loss:0.002 grdn:0.132 lr:4.1e-06 updt_s:0.227 data_s:0.039 +INFO 2025-05-11 20:16:31 ts/train.py:232 step:783K smpl:13M ep:63K epch:210.42 loss:0.001 grdn:0.122 lr:4.1e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 20:17:21 ts/train.py:232 step:784K smpl:13M ep:63K epch:210.48 loss:0.002 grdn:0.140 lr:4.1e-06 updt_s:0.228 data_s:0.017 +INFO 2025-05-11 20:18:10 ts/train.py:232 step:784K smpl:13M ep:63K epch:210.53 loss:0.001 grdn:0.128 lr:4.1e-06 updt_s:0.229 data_s:0.018 +INFO 2025-05-11 20:19:00 ts/train.py:232 step:784K smpl:13M ep:63K epch:210.58 loss:0.002 grdn:0.136 lr:4.1e-06 updt_s:0.228 data_s:0.021 +INFO 2025-05-11 20:19:50 ts/train.py:232 step:784K smpl:13M ep:63K epch:210.64 loss:0.001 grdn:0.115 lr:4.0e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 20:20:40 ts/train.py:232 step:784K smpl:13M ep:63K epch:210.69 loss:0.002 grdn:0.151 lr:4.0e-06 updt_s:0.228 data_s:0.021 +INFO 2025-05-11 20:21:31 ts/train.py:232 step:785K smpl:13M ep:63K epch:210.74 loss:0.002 grdn:0.133 lr:4.0e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 20:22:21 ts/train.py:232 step:785K smpl:13M ep:63K epch:210.80 loss:0.002 grdn:0.143 lr:4.0e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 20:23:12 ts/train.py:232 step:785K smpl:13M ep:63K epch:210.85 loss:0.001 grdn:0.134 lr:4.0e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 20:24:02 ts/train.py:232 step:785K smpl:13M ep:63K epch:210.91 loss:0.002 grdn:0.144 lr:4.0e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 20:24:52 ts/train.py:232 step:785K smpl:13M ep:63K epch:210.96 loss:0.002 grdn:0.142 lr:4.0e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 20:25:43 ts/train.py:232 step:786K smpl:13M ep:63K epch:211.01 loss:0.001 grdn:0.122 lr:3.9e-06 updt_s:0.228 data_s:0.023 +INFO 2025-05-11 20:26:33 ts/train.py:232 step:786K smpl:13M ep:63K epch:211.07 loss:0.001 grdn:0.122 lr:3.9e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:27:24 ts/train.py:232 step:786K smpl:13M ep:63K epch:211.12 loss:0.002 grdn:0.147 lr:3.9e-06 updt_s:0.228 data_s:0.026 +INFO 2025-05-11 20:28:15 ts/train.py:232 step:786K smpl:13M ep:63K epch:211.17 loss:0.001 grdn:0.127 lr:3.9e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 20:29:06 ts/train.py:232 step:786K smpl:13M ep:63K epch:211.23 loss:0.001 grdn:0.130 lr:3.9e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 20:29:57 ts/train.py:232 step:787K smpl:13M ep:63K epch:211.28 loss:0.002 grdn:0.139 lr:3.9e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 20:30:50 ts/train.py:232 step:787K smpl:13M ep:63K epch:211.33 loss:0.002 grdn:0.140 lr:3.9e-06 updt_s:0.226 data_s:0.041 +INFO 2025-05-11 20:31:42 ts/train.py:232 step:787K smpl:13M ep:63K epch:211.39 loss:0.002 grdn:0.132 lr:3.9e-06 updt_s:0.226 data_s:0.030 +INFO 2025-05-11 20:32:32 ts/train.py:232 step:787K smpl:13M ep:63K epch:211.44 loss:0.002 grdn:0.137 lr:3.8e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:33:23 ts/train.py:232 step:787K smpl:13M ep:63K epch:211.50 loss:0.001 grdn:0.112 lr:3.8e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 20:34:13 ts/train.py:232 step:788K smpl:13M ep:63K epch:211.55 loss:0.001 grdn:0.133 lr:3.8e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 20:35:03 ts/train.py:232 step:788K smpl:13M ep:63K epch:211.60 loss:0.001 grdn:0.127 lr:3.8e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 20:35:54 ts/train.py:232 step:788K smpl:13M ep:63K epch:211.66 loss:0.001 grdn:0.124 lr:3.8e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 20:36:44 ts/train.py:232 step:788K smpl:13M ep:64K epch:211.71 loss:0.001 grdn:0.118 lr:3.8e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 20:37:34 ts/train.py:232 step:788K smpl:13M ep:64K epch:211.76 loss:0.002 grdn:0.139 lr:3.8e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 20:38:25 ts/train.py:232 step:789K smpl:13M ep:64K epch:211.82 loss:0.002 grdn:0.153 lr:3.7e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:39:15 ts/train.py:232 step:789K smpl:13M ep:64K epch:211.87 loss:0.001 grdn:0.116 lr:3.7e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 20:40:05 ts/train.py:232 step:789K smpl:13M ep:64K epch:211.93 loss:0.001 grdn:0.124 lr:3.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 20:40:55 ts/train.py:232 step:789K smpl:13M ep:64K epch:211.98 loss:0.002 grdn:0.130 lr:3.7e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 20:41:45 ts/train.py:232 step:789K smpl:13M ep:64K epch:212.03 loss:0.001 grdn:0.112 lr:3.7e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 20:42:36 ts/train.py:232 step:790K smpl:13M ep:64K epch:212.09 loss:0.001 grdn:0.125 lr:3.7e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 20:43:27 ts/train.py:232 step:790K smpl:13M ep:64K epch:212.14 loss:0.002 grdn:0.148 lr:3.7e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 20:44:17 ts/train.py:232 step:790K smpl:13M ep:64K epch:212.19 loss:0.001 grdn:0.128 lr:3.7e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 20:45:07 ts/train.py:232 step:790K smpl:13M ep:64K epch:212.25 loss:0.001 grdn:0.131 lr:3.6e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 20:46:01 ts/train.py:232 step:790K smpl:13M ep:64K epch:212.30 loss:0.002 grdn:0.133 lr:3.6e-06 updt_s:0.227 data_s:0.040 +INFO 2025-05-11 20:46:50 ts/train.py:232 step:791K smpl:13M ep:64K epch:212.36 loss:0.001 grdn:0.115 lr:3.6e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 20:47:40 ts/train.py:232 step:791K smpl:13M ep:64K epch:212.41 loss:0.001 grdn:0.115 lr:3.6e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 20:48:30 ts/train.py:232 step:791K smpl:13M ep:64K epch:212.46 loss:0.002 grdn:0.140 lr:3.6e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 20:49:19 ts/train.py:232 step:791K smpl:13M ep:64K epch:212.52 loss:0.001 grdn:0.124 lr:3.6e-06 updt_s:0.228 data_s:0.015 +INFO 2025-05-11 20:50:08 ts/train.py:232 step:791K smpl:13M ep:64K epch:212.57 loss:0.001 grdn:0.132 lr:3.6e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 20:50:57 ts/train.py:232 step:792K smpl:13M ep:64K epch:212.62 loss:0.002 grdn:0.137 lr:3.5e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 20:51:46 ts/train.py:232 step:792K smpl:13M ep:64K epch:212.68 loss:0.001 grdn:0.125 lr:3.5e-06 updt_s:0.228 data_s:0.018 +INFO 2025-05-11 20:52:36 ts/train.py:232 step:792K smpl:13M ep:64K epch:212.73 loss:0.002 grdn:0.139 lr:3.5e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 20:53:25 ts/train.py:232 step:792K smpl:13M ep:64K epch:212.79 loss:0.001 grdn:0.122 lr:3.5e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 20:54:14 ts/train.py:232 step:792K smpl:13M ep:64K epch:212.84 loss:0.001 grdn:0.125 lr:3.5e-06 updt_s:0.228 data_s:0.016 +INFO 2025-05-11 20:55:03 ts/train.py:232 step:793K smpl:13M ep:64K epch:212.89 loss:0.001 grdn:0.118 lr:3.5e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 20:55:53 ts/train.py:232 step:793K smpl:13M ep:64K epch:212.95 loss:0.002 grdn:0.146 lr:3.5e-06 updt_s:0.228 data_s:0.019 +INFO 2025-05-11 20:56:43 ts/train.py:232 step:793K smpl:13M ep:64K epch:213.00 loss:0.002 grdn:0.138 lr:3.5e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 20:57:32 ts/train.py:232 step:793K smpl:13M ep:64K epch:213.05 loss:0.001 grdn:0.130 lr:3.4e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 20:58:22 ts/train.py:232 step:793K smpl:13M ep:64K epch:213.11 loss:0.002 grdn:0.132 lr:3.4e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 20:59:11 ts/train.py:232 step:794K smpl:13M ep:64K epch:213.16 loss:0.001 grdn:0.120 lr:3.4e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 21:00:00 ts/train.py:232 step:794K smpl:13M ep:64K epch:213.22 loss:0.001 grdn:0.121 lr:3.4e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 21:00:53 ts/train.py:232 step:794K smpl:13M ep:64K epch:213.27 loss:0.002 grdn:0.132 lr:3.4e-06 updt_s:0.226 data_s:0.035 +INFO 2025-05-11 21:01:43 ts/train.py:232 step:794K smpl:13M ep:64K epch:213.32 loss:0.002 grdn:0.138 lr:3.4e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 21:02:33 ts/train.py:232 step:794K smpl:13M ep:64K epch:213.38 loss:0.002 grdn:0.135 lr:3.4e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 21:03:23 ts/train.py:232 step:795K smpl:13M ep:64K epch:213.43 loss:0.001 grdn:0.126 lr:3.4e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 21:04:13 ts/train.py:232 step:795K smpl:13M ep:64K epch:213.48 loss:0.001 grdn:0.117 lr:3.3e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 21:05:03 ts/train.py:232 step:795K smpl:13M ep:64K epch:213.54 loss:0.001 grdn:0.125 lr:3.3e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 21:05:53 ts/train.py:232 step:795K smpl:13M ep:64K epch:213.59 loss:0.001 grdn:0.124 lr:3.3e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 21:06:43 ts/train.py:232 step:795K smpl:13M ep:64K epch:213.64 loss:0.001 grdn:0.120 lr:3.3e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:07:33 ts/train.py:232 step:796K smpl:13M ep:64K epch:213.70 loss:0.001 grdn:0.114 lr:3.3e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:08:23 ts/train.py:232 step:796K smpl:13M ep:64K epch:213.75 loss:0.001 grdn:0.126 lr:3.3e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:09:13 ts/train.py:232 step:796K smpl:13M ep:64K epch:213.81 loss:0.001 grdn:0.123 lr:3.3e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 21:10:02 ts/train.py:232 step:796K smpl:13M ep:64K epch:213.86 loss:0.001 grdn:0.114 lr:3.3e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:10:53 ts/train.py:232 step:796K smpl:13M ep:64K epch:213.91 loss:0.001 grdn:0.133 lr:3.2e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 21:11:43 ts/train.py:232 step:797K smpl:13M ep:64K epch:213.97 loss:0.001 grdn:0.112 lr:3.2e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 21:12:33 ts/train.py:232 step:797K smpl:13M ep:64K epch:214.02 loss:0.001 grdn:0.117 lr:3.2e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 21:13:22 ts/train.py:232 step:797K smpl:13M ep:64K epch:214.07 loss:0.001 grdn:0.130 lr:3.2e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:14:12 ts/train.py:232 step:797K smpl:13M ep:64K epch:214.13 loss:0.001 grdn:0.129 lr:3.2e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:15:02 ts/train.py:232 step:797K smpl:13M ep:64K epch:214.18 loss:0.001 grdn:0.132 lr:3.2e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 21:15:54 ts/train.py:232 step:798K smpl:13M ep:64K epch:214.24 loss:0.001 grdn:0.128 lr:3.2e-06 updt_s:0.227 data_s:0.034 +INFO 2025-05-11 21:16:43 ts/train.py:232 step:798K smpl:13M ep:64K epch:214.29 loss:0.001 grdn:0.113 lr:3.2e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 21:17:32 ts/train.py:232 step:798K smpl:13M ep:64K epch:214.34 loss:0.001 grdn:0.127 lr:3.1e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 21:18:22 ts/train.py:232 step:798K smpl:13M ep:64K epch:214.40 loss:0.001 grdn:0.118 lr:3.1e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 21:19:10 ts/train.py:232 step:798K smpl:13M ep:64K epch:214.45 loss:0.001 grdn:0.123 lr:3.1e-06 updt_s:0.227 data_s:0.013 +INFO 2025-05-11 21:19:58 ts/train.py:232 step:799K smpl:13M ep:64K epch:214.50 loss:0.001 grdn:0.127 lr:3.1e-06 updt_s:0.228 data_s:0.014 +INFO 2025-05-11 21:20:47 ts/train.py:232 step:799K smpl:13M ep:64K epch:214.56 loss:0.002 grdn:0.126 lr:3.1e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 21:21:35 ts/train.py:232 step:799K smpl:13M ep:64K epch:214.61 loss:0.001 grdn:0.131 lr:3.1e-06 updt_s:0.228 data_s:0.014 +INFO 2025-05-11 21:22:24 ts/train.py:232 step:799K smpl:13M ep:64K epch:214.67 loss:0.001 grdn:0.123 lr:3.1e-06 updt_s:0.228 data_s:0.015 +INFO 2025-05-11 21:23:13 ts/train.py:232 step:799K smpl:13M ep:64K epch:214.72 loss:0.001 grdn:0.127 lr:3.1e-06 updt_s:0.228 data_s:0.015 +INFO 2025-05-11 21:24:02 ts/train.py:232 step:800K smpl:13M ep:64K epch:214.77 loss:0.002 grdn:0.136 lr:3.0e-06 updt_s:0.228 data_s:0.017 +INFO 2025-05-11 21:24:50 ts/train.py:232 step:800K smpl:13M ep:64K epch:214.83 loss:0.001 grdn:0.124 lr:3.0e-06 updt_s:0.228 data_s:0.015 +INFO 2025-05-11 21:25:39 ts/train.py:232 step:800K smpl:13M ep:64K epch:214.88 loss:0.001 grdn:0.125 lr:3.0e-06 updt_s:0.228 data_s:0.013 +INFO 2025-05-11 21:26:28 ts/train.py:232 step:800K smpl:13M ep:64K epch:214.93 loss:0.001 grdn:0.123 lr:3.0e-06 updt_s:0.228 data_s:0.016 +INFO 2025-05-11 21:27:17 ts/train.py:232 step:800K smpl:13M ep:64K epch:214.99 loss:0.002 grdn:0.135 lr:3.0e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 21:28:05 ts/train.py:232 step:801K smpl:13M ep:65K epch:215.04 loss:0.001 grdn:0.106 lr:3.0e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 21:28:54 ts/train.py:232 step:801K smpl:13M ep:65K epch:215.10 loss:0.001 grdn:0.114 lr:3.0e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 21:29:42 ts/train.py:232 step:801K smpl:13M ep:65K epch:215.15 loss:0.001 grdn:0.130 lr:3.0e-06 updt_s:0.228 data_s:0.015 +INFO 2025-05-11 21:30:35 ts/train.py:232 step:801K smpl:13M ep:65K epch:215.20 loss:0.002 grdn:0.130 lr:3.0e-06 updt_s:0.226 data_s:0.035 +INFO 2025-05-11 21:31:25 ts/train.py:232 step:801K smpl:13M ep:65K epch:215.26 loss:0.001 grdn:0.115 lr:2.9e-06 updt_s:0.226 data_s:0.022 +INFO 2025-05-11 21:32:15 ts/train.py:232 step:802K smpl:13M ep:65K epch:215.31 loss:0.001 grdn:0.118 lr:2.9e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 21:33:05 ts/train.py:232 step:802K smpl:13M ep:65K epch:215.36 loss:0.001 grdn:0.113 lr:2.9e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:33:55 ts/train.py:232 step:802K smpl:13M ep:65K epch:215.42 loss:0.001 grdn:0.122 lr:2.9e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 21:34:45 ts/train.py:232 step:802K smpl:13M ep:65K epch:215.47 loss:0.001 grdn:0.123 lr:2.9e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:35:34 ts/train.py:232 step:802K smpl:13M ep:65K epch:215.53 loss:0.001 grdn:0.133 lr:2.9e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:36:24 ts/train.py:232 step:803K smpl:13M ep:65K epch:215.58 loss:0.002 grdn:0.129 lr:2.9e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 21:37:15 ts/train.py:232 step:803K smpl:13M ep:65K epch:215.63 loss:0.001 grdn:0.119 lr:2.9e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 21:38:05 ts/train.py:232 step:803K smpl:13M ep:65K epch:215.69 loss:0.001 grdn:0.104 lr:2.8e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 21:38:55 ts/train.py:232 step:803K smpl:13M ep:65K epch:215.74 loss:0.001 grdn:0.126 lr:2.8e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 21:39:45 ts/train.py:232 step:803K smpl:13M ep:65K epch:215.79 loss:0.002 grdn:0.137 lr:2.8e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 21:40:36 ts/train.py:232 step:804K smpl:13M ep:65K epch:215.85 loss:0.001 grdn:0.109 lr:2.8e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 21:41:26 ts/train.py:232 step:804K smpl:13M ep:65K epch:215.90 loss:0.001 grdn:0.122 lr:2.8e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 21:42:17 ts/train.py:232 step:804K smpl:13M ep:65K epch:215.95 loss:0.001 grdn:0.115 lr:2.8e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 21:43:07 ts/train.py:232 step:804K smpl:13M ep:65K epch:216.01 loss:0.001 grdn:0.123 lr:2.8e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 21:43:58 ts/train.py:232 step:804K smpl:13M ep:65K epch:216.06 loss:0.002 grdn:0.139 lr:2.8e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 21:44:48 ts/train.py:232 step:805K smpl:13M ep:65K epch:216.12 loss:0.001 grdn:0.124 lr:2.8e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 21:45:39 ts/train.py:232 step:805K smpl:13M ep:65K epch:216.17 loss:0.001 grdn:0.122 lr:2.7e-06 updt_s:0.227 data_s:0.030 +INFO 2025-05-11 21:46:29 ts/train.py:232 step:805K smpl:13M ep:65K epch:216.22 loss:0.002 grdn:0.136 lr:2.7e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 21:47:18 ts/train.py:232 step:805K smpl:13M ep:65K epch:216.28 loss:0.001 grdn:0.126 lr:2.7e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 21:48:08 ts/train.py:232 step:805K smpl:13M ep:65K epch:216.33 loss:0.001 grdn:0.121 lr:2.7e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 21:48:57 ts/train.py:232 step:806K smpl:13M ep:65K epch:216.38 loss:0.001 grdn:0.118 lr:2.7e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 21:49:47 ts/train.py:232 step:806K smpl:13M ep:65K epch:216.44 loss:0.001 grdn:0.128 lr:2.7e-06 updt_s:0.228 data_s:0.019 +INFO 2025-05-11 21:50:37 ts/train.py:232 step:806K smpl:13M ep:65K epch:216.49 loss:0.002 grdn:0.142 lr:2.7e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 21:51:27 ts/train.py:232 step:806K smpl:13M ep:65K epch:216.55 loss:0.001 grdn:0.125 lr:2.7e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 21:52:17 ts/train.py:232 step:806K smpl:13M ep:65K epch:216.60 loss:0.002 grdn:0.136 lr:2.7e-06 updt_s:0.229 data_s:0.019 +INFO 2025-05-11 21:53:06 ts/train.py:232 step:807K smpl:13M ep:65K epch:216.65 loss:0.001 grdn:0.139 lr:2.6e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 21:53:56 ts/train.py:232 step:807K smpl:13M ep:65K epch:216.71 loss:0.001 grdn:0.122 lr:2.6e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 21:54:45 ts/train.py:232 step:807K smpl:13M ep:65K epch:216.76 loss:0.001 grdn:0.129 lr:2.6e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 21:55:35 ts/train.py:232 step:807K smpl:13M ep:65K epch:216.81 loss:0.002 grdn:0.134 lr:2.6e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 21:56:25 ts/train.py:232 step:807K smpl:13M ep:65K epch:216.87 loss:0.002 grdn:0.139 lr:2.6e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 21:57:17 ts/train.py:232 step:808K smpl:13M ep:65K epch:216.92 loss:0.002 grdn:0.135 lr:2.6e-06 updt_s:0.227 data_s:0.033 +INFO 2025-05-11 21:58:07 ts/train.py:232 step:808K smpl:13M ep:65K epch:216.98 loss:0.001 grdn:0.126 lr:2.6e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 21:58:57 ts/train.py:232 step:808K smpl:13M ep:65K epch:217.03 loss:0.002 grdn:0.146 lr:2.6e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 21:59:47 ts/train.py:232 step:808K smpl:13M ep:65K epch:217.08 loss:0.001 grdn:0.135 lr:2.6e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 22:00:40 ts/train.py:232 step:808K smpl:13M ep:65K epch:217.14 loss:0.001 grdn:0.123 lr:2.5e-06 updt_s:0.227 data_s:0.037 +INFO 2025-05-11 22:01:30 ts/train.py:232 step:809K smpl:13M ep:65K epch:217.19 loss:0.001 grdn:0.116 lr:2.5e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 22:02:20 ts/train.py:232 step:809K smpl:13M ep:65K epch:217.24 loss:0.001 grdn:0.124 lr:2.5e-06 updt_s:0.228 data_s:0.021 +INFO 2025-05-11 22:03:10 ts/train.py:232 step:809K smpl:13M ep:65K epch:217.30 loss:0.001 grdn:0.132 lr:2.5e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 22:04:00 ts/train.py:232 step:809K smpl:13M ep:65K epch:217.35 loss:0.001 grdn:0.113 lr:2.5e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 22:04:51 ts/train.py:232 step:809K smpl:13M ep:65K epch:217.41 loss:0.001 grdn:0.119 lr:2.5e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 22:05:42 ts/train.py:232 step:810K smpl:13M ep:65K epch:217.46 loss:0.001 grdn:0.130 lr:2.5e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 22:06:33 ts/train.py:232 step:810K smpl:13M ep:65K epch:217.51 loss:0.001 grdn:0.120 lr:2.5e-06 updt_s:0.227 data_s:0.027 +INFO 2025-05-11 22:07:24 ts/train.py:232 step:810K smpl:13M ep:65K epch:217.57 loss:0.001 grdn:0.116 lr:2.5e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 22:07:24 ts/train.py:241 Checkpoint policy after step 810000 +INFO 2025-05-11 22:08:19 ts/train.py:232 step:810K smpl:13M ep:65K epch:217.62 loss:0.001 grdn:0.117 lr:2.4e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 22:09:09 ts/train.py:232 step:810K smpl:13M ep:65K epch:217.67 loss:0.001 grdn:0.117 lr:2.4e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 22:10:00 ts/train.py:232 step:811K smpl:13M ep:65K epch:217.73 loss:0.002 grdn:0.136 lr:2.4e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 22:10:50 ts/train.py:232 step:811K smpl:13M ep:65K epch:217.78 loss:0.001 grdn:0.121 lr:2.4e-06 updt_s:0.228 data_s:0.024 +INFO 2025-05-11 22:11:41 ts/train.py:232 step:811K smpl:13M ep:65K epch:217.84 loss:0.001 grdn:0.130 lr:2.4e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 22:12:31 ts/train.py:232 step:811K smpl:13M ep:65K epch:217.89 loss:0.002 grdn:0.139 lr:2.4e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 22:13:22 ts/train.py:232 step:811K smpl:13M ep:65K epch:217.94 loss:0.001 grdn:0.126 lr:2.4e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 22:14:12 ts/train.py:232 step:812K smpl:13M ep:65K epch:218.00 loss:0.001 grdn:0.129 lr:2.4e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 22:15:06 ts/train.py:232 step:812K smpl:13M ep:65K epch:218.05 loss:0.001 grdn:0.127 lr:2.4e-06 updt_s:0.227 data_s:0.039 +INFO 2025-05-11 22:15:56 ts/train.py:232 step:812K smpl:13M ep:65K epch:218.10 loss:0.002 grdn:0.131 lr:2.3e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 22:16:46 ts/train.py:232 step:812K smpl:13M ep:65K epch:218.16 loss:0.001 grdn:0.126 lr:2.3e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 22:17:37 ts/train.py:232 step:812K smpl:13M ep:65K epch:218.21 loss:0.001 grdn:0.107 lr:2.3e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 22:18:26 ts/train.py:232 step:813K smpl:13M ep:65K epch:218.26 loss:0.001 grdn:0.127 lr:2.3e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 22:19:16 ts/train.py:232 step:813K smpl:13M ep:65K epch:218.32 loss:0.002 grdn:0.139 lr:2.3e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 22:20:05 ts/train.py:232 step:813K smpl:13M ep:66K epch:218.37 loss:0.001 grdn:0.127 lr:2.3e-06 updt_s:0.227 data_s:0.014 +INFO 2025-05-11 22:20:54 ts/train.py:232 step:813K smpl:13M ep:66K epch:218.43 loss:0.001 grdn:0.134 lr:2.3e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 22:21:43 ts/train.py:232 step:813K smpl:13M ep:66K epch:218.48 loss:0.001 grdn:0.132 lr:2.3e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 22:22:32 ts/train.py:232 step:814K smpl:13M ep:66K epch:218.53 loss:0.001 grdn:0.122 lr:2.3e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 22:23:21 ts/train.py:232 step:814K smpl:13M ep:66K epch:218.59 loss:0.001 grdn:0.113 lr:2.3e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 22:24:10 ts/train.py:232 step:814K smpl:13M ep:66K epch:218.64 loss:0.001 grdn:0.121 lr:2.2e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 22:24:59 ts/train.py:232 step:814K smpl:13M ep:66K epch:218.69 loss:0.001 grdn:0.143 lr:2.2e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 22:25:49 ts/train.py:232 step:814K smpl:13M ep:66K epch:218.75 loss:0.002 grdn:0.144 lr:2.2e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 22:26:39 ts/train.py:232 step:815K smpl:13M ep:66K epch:218.80 loss:0.001 grdn:0.128 lr:2.2e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 22:27:28 ts/train.py:232 step:815K smpl:13M ep:66K epch:218.86 loss:0.001 grdn:0.133 lr:2.2e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 22:28:17 ts/train.py:232 step:815K smpl:13M ep:66K epch:218.91 loss:0.001 grdn:0.119 lr:2.2e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 22:29:06 ts/train.py:232 step:815K smpl:13M ep:66K epch:218.96 loss:0.001 grdn:0.120 lr:2.2e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 22:29:57 ts/train.py:232 step:815K smpl:13M ep:66K epch:219.02 loss:0.001 grdn:0.124 lr:2.2e-06 updt_s:0.227 data_s:0.030 +INFO 2025-05-11 22:30:47 ts/train.py:232 step:816K smpl:13M ep:66K epch:219.07 loss:0.001 grdn:0.131 lr:2.2e-06 updt_s:0.226 data_s:0.022 +INFO 2025-05-11 22:31:37 ts/train.py:232 step:816K smpl:13M ep:66K epch:219.12 loss:0.001 grdn:0.126 lr:2.2e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 22:32:27 ts/train.py:232 step:816K smpl:13M ep:66K epch:219.18 loss:0.001 grdn:0.131 lr:2.1e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 22:33:16 ts/train.py:232 step:816K smpl:13M ep:66K epch:219.23 loss:0.001 grdn:0.126 lr:2.1e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 22:34:05 ts/train.py:232 step:816K smpl:13M ep:66K epch:219.29 loss:0.001 grdn:0.110 lr:2.1e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 22:34:54 ts/train.py:232 step:817K smpl:13M ep:66K epch:219.34 loss:0.001 grdn:0.128 lr:2.1e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 22:35:42 ts/train.py:232 step:817K smpl:13M ep:66K epch:219.39 loss:0.001 grdn:0.124 lr:2.1e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 22:36:31 ts/train.py:232 step:817K smpl:13M ep:66K epch:219.45 loss:0.001 grdn:0.120 lr:2.1e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 22:37:20 ts/train.py:232 step:817K smpl:13M ep:66K epch:219.50 loss:0.002 grdn:0.139 lr:2.1e-06 updt_s:0.227 data_s:0.014 +INFO 2025-05-11 22:38:08 ts/train.py:232 step:817K smpl:13M ep:66K epch:219.55 loss:0.001 grdn:0.117 lr:2.1e-06 updt_s:0.228 data_s:0.015 +INFO 2025-05-11 22:38:57 ts/train.py:232 step:818K smpl:13M ep:66K epch:219.61 loss:0.001 grdn:0.123 lr:2.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-11 22:39:46 ts/train.py:232 step:818K smpl:13M ep:66K epch:219.66 loss:0.001 grdn:0.122 lr:2.1e-06 updt_s:0.228 data_s:0.014 +INFO 2025-05-11 22:40:34 ts/train.py:232 step:818K smpl:13M ep:66K epch:219.72 loss:0.001 grdn:0.119 lr:2.0e-06 updt_s:0.227 data_s:0.014 +INFO 2025-05-11 22:41:23 ts/train.py:232 step:818K smpl:13M ep:66K epch:219.77 loss:0.001 grdn:0.129 lr:2.0e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 22:42:11 ts/train.py:232 step:818K smpl:13M ep:66K epch:219.82 loss:0.001 grdn:0.115 lr:2.0e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 22:43:00 ts/train.py:232 step:819K smpl:13M ep:66K epch:219.88 loss:0.002 grdn:0.149 lr:2.0e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 22:43:48 ts/train.py:232 step:819K smpl:13M ep:66K epch:219.93 loss:0.001 grdn:0.123 lr:2.0e-06 updt_s:0.227 data_s:0.012 +INFO 2025-05-11 22:44:39 ts/train.py:232 step:819K smpl:13M ep:66K epch:219.98 loss:0.001 grdn:0.132 lr:2.0e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 22:45:29 ts/train.py:232 step:819K smpl:13M ep:66K epch:220.04 loss:0.001 grdn:0.131 lr:2.0e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 22:46:18 ts/train.py:232 step:819K smpl:13M ep:66K epch:220.09 loss:0.002 grdn:0.136 lr:2.0e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 22:47:08 ts/train.py:232 step:820K smpl:13M ep:66K epch:220.15 loss:0.001 grdn:0.125 lr:2.0e-06 updt_s:0.229 data_s:0.019 +INFO 2025-05-11 22:47:57 ts/train.py:232 step:820K smpl:13M ep:66K epch:220.20 loss:0.001 grdn:0.112 lr:2.0e-06 updt_s:0.228 data_s:0.015 +INFO 2025-05-11 22:48:46 ts/train.py:232 step:820K smpl:13M ep:66K epch:220.25 loss:0.001 grdn:0.119 lr:1.9e-06 updt_s:0.228 data_s:0.019 +INFO 2025-05-11 22:49:35 ts/train.py:232 step:820K smpl:13M ep:66K epch:220.31 loss:0.001 grdn:0.108 lr:1.9e-06 updt_s:0.228 data_s:0.016 +INFO 2025-05-11 22:50:24 ts/train.py:232 step:820K smpl:13M ep:66K epch:220.36 loss:0.001 grdn:0.127 lr:1.9e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 22:51:13 ts/train.py:232 step:821K smpl:13M ep:66K epch:220.41 loss:0.001 grdn:0.125 lr:1.9e-06 updt_s:0.228 data_s:0.014 +INFO 2025-05-11 22:52:02 ts/train.py:232 step:821K smpl:13M ep:66K epch:220.47 loss:0.001 grdn:0.134 lr:1.9e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 22:52:51 ts/train.py:232 step:821K smpl:13M ep:66K epch:220.52 loss:0.001 grdn:0.129 lr:1.9e-06 updt_s:0.228 data_s:0.017 +INFO 2025-05-11 22:53:39 ts/train.py:232 step:821K smpl:13M ep:66K epch:220.57 loss:0.001 grdn:0.123 lr:1.9e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 22:54:29 ts/train.py:232 step:821K smpl:13M ep:66K epch:220.63 loss:0.001 grdn:0.117 lr:1.9e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 22:55:18 ts/train.py:232 step:822K smpl:13M ep:66K epch:220.68 loss:0.002 grdn:0.135 lr:1.9e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 22:56:08 ts/train.py:232 step:822K smpl:13M ep:66K epch:220.74 loss:0.002 grdn:0.140 lr:1.9e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 22:56:58 ts/train.py:232 step:822K smpl:13M ep:66K epch:220.79 loss:0.001 grdn:0.116 lr:1.8e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 22:57:47 ts/train.py:232 step:822K smpl:13M ep:66K epch:220.84 loss:0.001 grdn:0.115 lr:1.8e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 22:58:37 ts/train.py:232 step:822K smpl:13M ep:66K epch:220.90 loss:0.001 grdn:0.128 lr:1.8e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 22:59:29 ts/train.py:232 step:823K smpl:13M ep:66K epch:220.95 loss:0.001 grdn:0.134 lr:1.8e-06 updt_s:0.227 data_s:0.033 +INFO 2025-05-11 23:00:17 ts/train.py:232 step:823K smpl:13M ep:66K epch:221.00 loss:0.001 grdn:0.123 lr:1.8e-06 updt_s:0.227 data_s:0.014 +INFO 2025-05-11 23:01:05 ts/train.py:232 step:823K smpl:13M ep:66K epch:221.06 loss:0.001 grdn:0.123 lr:1.8e-06 updt_s:0.227 data_s:0.009 +INFO 2025-05-11 23:01:53 ts/train.py:232 step:823K smpl:13M ep:66K epch:221.11 loss:0.001 grdn:0.124 lr:1.8e-06 updt_s:0.227 data_s:0.012 +INFO 2025-05-11 23:02:40 ts/train.py:232 step:823K smpl:13M ep:66K epch:221.17 loss:0.001 grdn:0.127 lr:1.8e-06 updt_s:0.227 data_s:0.012 +INFO 2025-05-11 23:03:28 ts/train.py:232 step:824K smpl:13M ep:66K epch:221.22 loss:0.001 grdn:0.131 lr:1.8e-06 updt_s:0.227 data_s:0.011 +INFO 2025-05-11 23:04:17 ts/train.py:232 step:824K smpl:13M ep:66K epch:221.27 loss:0.001 grdn:0.125 lr:1.8e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-11 23:05:05 ts/train.py:232 step:824K smpl:13M ep:66K epch:221.33 loss:0.001 grdn:0.133 lr:1.8e-06 updt_s:0.227 data_s:0.012 +INFO 2025-05-11 23:05:52 ts/train.py:232 step:824K smpl:13M ep:66K epch:221.38 loss:0.002 grdn:0.142 lr:1.7e-06 updt_s:0.227 data_s:0.010 +INFO 2025-05-11 23:06:40 ts/train.py:232 step:824K smpl:13M ep:66K epch:221.43 loss:0.001 grdn:0.123 lr:1.7e-06 updt_s:0.227 data_s:0.012 +INFO 2025-05-11 23:07:27 ts/train.py:232 step:825K smpl:13M ep:66K epch:221.49 loss:0.002 grdn:0.146 lr:1.7e-06 updt_s:0.227 data_s:0.009 +INFO 2025-05-11 23:08:16 ts/train.py:232 step:825K smpl:13M ep:66K epch:221.54 loss:0.001 grdn:0.125 lr:1.7e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-11 23:09:03 ts/train.py:232 step:825K smpl:13M ep:66K epch:221.60 loss:0.001 grdn:0.123 lr:1.7e-06 updt_s:0.227 data_s:0.010 +INFO 2025-05-11 23:09:51 ts/train.py:232 step:825K smpl:13M ep:66K epch:221.65 loss:0.001 grdn:0.113 lr:1.7e-06 updt_s:0.227 data_s:0.011 +INFO 2025-05-11 23:10:38 ts/train.py:232 step:825K smpl:13M ep:67K epch:221.70 loss:0.001 grdn:0.109 lr:1.7e-06 updt_s:0.227 data_s:0.008 +INFO 2025-05-11 23:11:26 ts/train.py:232 step:826K smpl:13M ep:67K epch:221.76 loss:0.001 grdn:0.120 lr:1.7e-06 updt_s:0.227 data_s:0.012 +INFO 2025-05-11 23:12:13 ts/train.py:232 step:826K smpl:13M ep:67K epch:221.81 loss:0.001 grdn:0.119 lr:1.7e-06 updt_s:0.227 data_s:0.009 +INFO 2025-05-11 23:13:01 ts/train.py:232 step:826K smpl:13M ep:67K epch:221.86 loss:0.001 grdn:0.121 lr:1.7e-06 updt_s:0.227 data_s:0.011 +INFO 2025-05-11 23:13:52 ts/train.py:232 step:826K smpl:13M ep:67K epch:221.92 loss:0.001 grdn:0.119 lr:1.7e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-11 23:14:41 ts/train.py:232 step:826K smpl:13M ep:67K epch:221.97 loss:0.001 grdn:0.117 lr:1.6e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 23:15:31 ts/train.py:232 step:827K smpl:13M ep:67K epch:222.03 loss:0.001 grdn:0.131 lr:1.6e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 23:16:21 ts/train.py:232 step:827K smpl:13M ep:67K epch:222.08 loss:0.001 grdn:0.111 lr:1.6e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 23:17:10 ts/train.py:232 step:827K smpl:13M ep:67K epch:222.13 loss:0.001 grdn:0.125 lr:1.6e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 23:17:59 ts/train.py:232 step:827K smpl:13M ep:67K epch:222.19 loss:0.001 grdn:0.114 lr:1.6e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 23:18:49 ts/train.py:232 step:827K smpl:13M ep:67K epch:222.24 loss:0.001 grdn:0.116 lr:1.6e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 23:19:38 ts/train.py:232 step:828K smpl:13M ep:67K epch:222.29 loss:0.001 grdn:0.118 lr:1.6e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 23:20:28 ts/train.py:232 step:828K smpl:13M ep:67K epch:222.35 loss:0.001 grdn:0.143 lr:1.6e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 23:21:18 ts/train.py:232 step:828K smpl:13M ep:67K epch:222.40 loss:0.002 grdn:0.157 lr:1.6e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 23:22:07 ts/train.py:232 step:828K smpl:13M ep:67K epch:222.46 loss:0.001 grdn:0.123 lr:1.6e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 23:22:56 ts/train.py:232 step:828K smpl:13M ep:67K epch:222.51 loss:0.001 grdn:0.121 lr:1.6e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 23:23:46 ts/train.py:232 step:829K smpl:13M ep:67K epch:222.56 loss:0.002 grdn:0.149 lr:1.6e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 23:24:36 ts/train.py:232 step:829K smpl:13M ep:67K epch:222.62 loss:0.001 grdn:0.118 lr:1.5e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 23:25:26 ts/train.py:232 step:829K smpl:13M ep:67K epch:222.67 loss:0.001 grdn:0.125 lr:1.5e-06 updt_s:0.228 data_s:0.020 +INFO 2025-05-11 23:26:15 ts/train.py:232 step:829K smpl:13M ep:67K epch:222.72 loss:0.001 grdn:0.112 lr:1.5e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 23:27:06 ts/train.py:232 step:829K smpl:13M ep:67K epch:222.78 loss:0.001 grdn:0.127 lr:1.5e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 23:27:56 ts/train.py:232 step:830K smpl:13M ep:67K epch:222.83 loss:0.001 grdn:0.123 lr:1.5e-06 updt_s:0.227 data_s:0.026 +INFO 2025-05-11 23:28:49 ts/train.py:232 step:830K smpl:13M ep:67K epch:222.88 loss:0.001 grdn:0.130 lr:1.5e-06 updt_s:0.227 data_s:0.037 +INFO 2025-05-11 23:29:40 ts/train.py:232 step:830K smpl:13M ep:67K epch:222.94 loss:0.001 grdn:0.130 lr:1.5e-06 updt_s:0.226 data_s:0.027 +INFO 2025-05-11 23:30:31 ts/train.py:232 step:830K smpl:13M ep:67K epch:222.99 loss:0.001 grdn:0.129 lr:1.5e-06 updt_s:0.226 data_s:0.026 +INFO 2025-05-11 23:31:21 ts/train.py:232 step:830K smpl:13M ep:67K epch:223.05 loss:0.001 grdn:0.119 lr:1.5e-06 updt_s:0.226 data_s:0.024 +INFO 2025-05-11 23:32:12 ts/train.py:232 step:831K smpl:13M ep:67K epch:223.10 loss:0.001 grdn:0.132 lr:1.5e-06 updt_s:0.226 data_s:0.027 +INFO 2025-05-11 23:33:03 ts/train.py:232 step:831K smpl:13M ep:67K epch:223.15 loss:0.001 grdn:0.117 lr:1.5e-06 updt_s:0.227 data_s:0.028 +INFO 2025-05-11 23:33:53 ts/train.py:232 step:831K smpl:13M ep:67K epch:223.21 loss:0.001 grdn:0.126 lr:1.4e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 23:34:43 ts/train.py:232 step:831K smpl:13M ep:67K epch:223.26 loss:0.001 grdn:0.116 lr:1.4e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 23:35:33 ts/train.py:232 step:831K smpl:13M ep:67K epch:223.31 loss:0.001 grdn:0.112 lr:1.4e-06 updt_s:0.228 data_s:0.021 +INFO 2025-05-11 23:36:23 ts/train.py:232 step:832K smpl:13M ep:67K epch:223.37 loss:0.001 grdn:0.091 lr:1.4e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 23:37:13 ts/train.py:232 step:832K smpl:13M ep:67K epch:223.42 loss:0.001 grdn:0.115 lr:1.4e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 23:38:03 ts/train.py:232 step:832K smpl:13M ep:67K epch:223.48 loss:0.001 grdn:0.119 lr:1.4e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 23:38:53 ts/train.py:232 step:832K smpl:13M ep:67K epch:223.53 loss:0.001 grdn:0.121 lr:1.4e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 23:39:43 ts/train.py:232 step:832K smpl:13M ep:67K epch:223.58 loss:0.001 grdn:0.124 lr:1.4e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 23:40:33 ts/train.py:232 step:833K smpl:13M ep:67K epch:223.64 loss:0.001 grdn:0.110 lr:1.4e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 23:41:23 ts/train.py:232 step:833K smpl:13M ep:67K epch:223.69 loss:0.001 grdn:0.110 lr:1.4e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-11 23:42:13 ts/train.py:232 step:833K smpl:13M ep:67K epch:223.74 loss:0.001 grdn:0.114 lr:1.4e-06 updt_s:0.227 data_s:0.024 +INFO 2025-05-11 23:43:03 ts/train.py:232 step:833K smpl:13M ep:67K epch:223.80 loss:0.001 grdn:0.110 lr:1.4e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 23:43:57 ts/train.py:232 step:833K smpl:13M ep:67K epch:223.85 loss:0.001 grdn:0.115 lr:1.4e-06 updt_s:0.226 data_s:0.039 +INFO 2025-05-11 23:44:47 ts/train.py:232 step:834K smpl:13M ep:67K epch:223.91 loss:0.001 grdn:0.121 lr:1.3e-06 updt_s:0.226 data_s:0.024 +INFO 2025-05-11 23:45:37 ts/train.py:232 step:834K smpl:13M ep:67K epch:223.96 loss:0.001 grdn:0.109 lr:1.3e-06 updt_s:0.228 data_s:0.022 +INFO 2025-05-11 23:46:27 ts/train.py:232 step:834K smpl:13M ep:67K epch:224.01 loss:0.001 grdn:0.130 lr:1.3e-06 updt_s:0.226 data_s:0.021 +INFO 2025-05-11 23:47:17 ts/train.py:232 step:834K smpl:13M ep:67K epch:224.07 loss:0.001 grdn:0.119 lr:1.3e-06 updt_s:0.226 data_s:0.025 +INFO 2025-05-11 23:48:07 ts/train.py:232 step:834K smpl:13M ep:67K epch:224.12 loss:0.001 grdn:0.110 lr:1.3e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 23:48:56 ts/train.py:232 step:835K smpl:13M ep:67K epch:224.17 loss:0.001 grdn:0.127 lr:1.3e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 23:49:46 ts/train.py:232 step:835K smpl:13M ep:67K epch:224.23 loss:0.001 grdn:0.112 lr:1.3e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 23:50:35 ts/train.py:232 step:835K smpl:13M ep:67K epch:224.28 loss:0.001 grdn:0.122 lr:1.3e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 23:51:25 ts/train.py:232 step:835K smpl:13M ep:67K epch:224.34 loss:0.001 grdn:0.110 lr:1.3e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-11 23:52:15 ts/train.py:232 step:835K smpl:13M ep:67K epch:224.39 loss:0.002 grdn:0.133 lr:1.3e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 23:53:04 ts/train.py:232 step:836K smpl:13M ep:67K epch:224.44 loss:0.001 grdn:0.128 lr:1.3e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-11 23:53:53 ts/train.py:232 step:836K smpl:13M ep:67K epch:224.50 loss:0.001 grdn:0.125 lr:1.3e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 23:54:43 ts/train.py:232 step:836K smpl:13M ep:67K epch:224.55 loss:0.001 grdn:0.125 lr:1.2e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 23:55:33 ts/train.py:232 step:836K smpl:13M ep:67K epch:224.60 loss:0.001 grdn:0.120 lr:1.2e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 23:56:23 ts/train.py:232 step:836K smpl:13M ep:67K epch:224.66 loss:0.001 grdn:0.120 lr:1.2e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-11 23:57:13 ts/train.py:232 step:837K smpl:13M ep:67K epch:224.71 loss:0.001 grdn:0.127 lr:1.2e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-11 23:58:02 ts/train.py:232 step:837K smpl:13M ep:67K epch:224.76 loss:0.001 grdn:0.132 lr:1.2e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-11 23:58:55 ts/train.py:232 step:837K smpl:13M ep:67K epch:224.82 loss:0.001 grdn:0.105 lr:1.2e-06 updt_s:0.227 data_s:0.037 +INFO 2025-05-11 23:59:45 ts/train.py:232 step:837K smpl:13M ep:67K epch:224.87 loss:0.001 grdn:0.107 lr:1.2e-06 updt_s:0.226 data_s:0.021 +INFO 2025-05-12 00:00:35 ts/train.py:232 step:837K smpl:13M ep:67K epch:224.93 loss:0.001 grdn:0.117 lr:1.2e-06 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 00:01:23 ts/train.py:232 step:838K smpl:13M ep:67K epch:224.98 loss:0.001 grdn:0.126 lr:1.2e-06 updt_s:0.227 data_s:0.015 +INFO 2025-05-12 00:02:12 ts/train.py:232 step:838K smpl:13M ep:68K epch:225.03 loss:0.001 grdn:0.112 lr:1.2e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:03:01 ts/train.py:232 step:838K smpl:13M ep:68K epch:225.09 loss:0.001 grdn:0.110 lr:1.2e-06 updt_s:0.227 data_s:0.016 +INFO 2025-05-12 00:03:50 ts/train.py:232 step:838K smpl:13M ep:68K epch:225.14 loss:0.001 grdn:0.109 lr:1.2e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 00:04:39 ts/train.py:232 step:838K smpl:13M ep:68K epch:225.19 loss:0.001 grdn:0.128 lr:1.2e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 00:05:28 ts/train.py:232 step:839K smpl:13M ep:68K epch:225.25 loss:0.001 grdn:0.111 lr:1.1e-06 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 00:06:17 ts/train.py:232 step:839K smpl:13M ep:68K epch:225.30 loss:0.001 grdn:0.119 lr:1.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:07:06 ts/train.py:232 step:839K smpl:13M ep:68K epch:225.36 loss:0.001 grdn:0.142 lr:1.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:07:55 ts/train.py:232 step:839K smpl:13M ep:68K epch:225.41 loss:0.001 grdn:0.131 lr:1.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:08:44 ts/train.py:232 step:839K smpl:13M ep:68K epch:225.46 loss:0.001 grdn:0.120 lr:1.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:09:33 ts/train.py:232 step:840K smpl:13M ep:68K epch:225.52 loss:0.001 grdn:0.116 lr:1.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:10:22 ts/train.py:232 step:840K smpl:13M ep:68K epch:225.57 loss:0.001 grdn:0.113 lr:1.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:11:11 ts/train.py:232 step:840K smpl:13M ep:68K epch:225.62 loss:0.001 grdn:0.128 lr:1.1e-06 updt_s:0.228 data_s:0.018 +INFO 2025-05-12 00:11:11 ts/train.py:241 Checkpoint policy after step 840000 +INFO 2025-05-12 00:12:05 ts/train.py:232 step:840K smpl:13M ep:68K epch:225.68 loss:0.001 grdn:0.132 lr:1.1e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 00:12:54 ts/train.py:232 step:840K smpl:13M ep:68K epch:225.73 loss:0.001 grdn:0.123 lr:1.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:13:45 ts/train.py:232 step:841K smpl:13M ep:68K epch:225.79 loss:0.001 grdn:0.129 lr:1.1e-06 updt_s:0.227 data_s:0.029 +INFO 2025-05-12 00:14:36 ts/train.py:232 step:841K smpl:13M ep:68K epch:225.84 loss:0.001 grdn:0.113 lr:1.1e-06 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 00:15:25 ts/train.py:232 step:841K smpl:13M ep:68K epch:225.89 loss:0.001 grdn:0.128 lr:1.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:16:14 ts/train.py:232 step:841K smpl:13M ep:68K epch:225.95 loss:0.001 grdn:0.115 lr:1.1e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:17:03 ts/train.py:232 step:841K smpl:13M ep:68K epch:226.00 loss:0.001 grdn:0.116 lr:1.0e-06 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:17:53 ts/train.py:232 step:842K smpl:13M ep:68K epch:226.05 loss:0.001 grdn:0.108 lr:1.0e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 00:18:43 ts/train.py:232 step:842K smpl:13M ep:68K epch:226.11 loss:0.001 grdn:0.124 lr:1.0e-06 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 00:19:33 ts/train.py:232 step:842K smpl:13M ep:68K epch:226.16 loss:0.001 grdn:0.124 lr:1.0e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 00:20:23 ts/train.py:232 step:842K smpl:13M ep:68K epch:226.22 loss:0.002 grdn:0.138 lr:1.0e-06 updt_s:0.229 data_s:0.022 +INFO 2025-05-12 00:21:13 ts/train.py:232 step:842K smpl:13M ep:68K epch:226.27 loss:0.001 grdn:0.119 lr:1.0e-06 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 00:22:03 ts/train.py:232 step:843K smpl:13M ep:68K epch:226.32 loss:0.001 grdn:0.120 lr:1.0e-06 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 00:22:53 ts/train.py:232 step:843K smpl:13M ep:68K epch:226.38 loss:0.001 grdn:0.131 lr:1.0e-06 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 00:23:42 ts/train.py:232 step:843K smpl:13M ep:68K epch:226.43 loss:0.001 grdn:0.125 lr:9.9e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 00:24:32 ts/train.py:232 step:843K smpl:13M ep:68K epch:226.48 loss:0.001 grdn:0.128 lr:9.8e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 00:25:22 ts/train.py:232 step:843K smpl:13M ep:68K epch:226.54 loss:0.001 grdn:0.116 lr:9.8e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 00:26:12 ts/train.py:232 step:844K smpl:13M ep:68K epch:226.59 loss:0.001 grdn:0.131 lr:9.7e-07 updt_s:0.228 data_s:0.018 +INFO 2025-05-12 00:27:02 ts/train.py:232 step:844K smpl:14M ep:68K epch:226.65 loss:0.001 grdn:0.133 lr:9.6e-07 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 00:27:51 ts/train.py:232 step:844K smpl:14M ep:68K epch:226.70 loss:0.001 grdn:0.127 lr:9.6e-07 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 00:28:44 ts/train.py:232 step:844K smpl:14M ep:68K epch:226.75 loss:0.002 grdn:0.140 lr:9.5e-07 updt_s:0.227 data_s:0.036 +INFO 2025-05-12 00:29:34 ts/train.py:232 step:844K smpl:14M ep:68K epch:226.81 loss:0.001 grdn:0.112 lr:9.4e-07 updt_s:0.228 data_s:0.023 +INFO 2025-05-12 00:30:24 ts/train.py:232 step:845K smpl:14M ep:68K epch:226.86 loss:0.001 grdn:0.126 lr:9.4e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 00:31:14 ts/train.py:232 step:845K smpl:14M ep:68K epch:226.91 loss:0.001 grdn:0.102 lr:9.3e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 00:32:03 ts/train.py:232 step:845K smpl:14M ep:68K epch:226.97 loss:0.001 grdn:0.115 lr:9.2e-07 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 00:32:53 ts/train.py:232 step:845K smpl:14M ep:68K epch:227.02 loss:0.001 grdn:0.113 lr:9.2e-07 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 00:33:41 ts/train.py:232 step:845K smpl:14M ep:68K epch:227.07 loss:0.001 grdn:0.136 lr:9.1e-07 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:34:31 ts/train.py:232 step:846K smpl:14M ep:68K epch:227.13 loss:0.001 grdn:0.127 lr:9.0e-07 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 00:35:21 ts/train.py:232 step:846K smpl:14M ep:68K epch:227.18 loss:0.001 grdn:0.120 lr:9.0e-07 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 00:36:10 ts/train.py:232 step:846K smpl:14M ep:68K epch:227.24 loss:0.001 grdn:0.108 lr:8.9e-07 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:36:59 ts/train.py:232 step:846K smpl:14M ep:68K epch:227.29 loss:0.001 grdn:0.116 lr:8.8e-07 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 00:37:48 ts/train.py:232 step:846K smpl:14M ep:68K epch:227.34 loss:0.001 grdn:0.131 lr:8.8e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 00:38:38 ts/train.py:232 step:847K smpl:14M ep:68K epch:227.40 loss:0.001 grdn:0.131 lr:8.7e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 00:39:27 ts/train.py:232 step:847K smpl:14M ep:68K epch:227.45 loss:0.001 grdn:0.110 lr:8.6e-07 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 00:40:16 ts/train.py:232 step:847K smpl:14M ep:68K epch:227.50 loss:0.001 grdn:0.115 lr:8.6e-07 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 00:41:06 ts/train.py:232 step:847K smpl:14M ep:68K epch:227.56 loss:0.001 grdn:0.126 lr:8.5e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 00:41:55 ts/train.py:232 step:847K smpl:14M ep:68K epch:227.61 loss:0.001 grdn:0.117 lr:8.4e-07 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 00:42:45 ts/train.py:232 step:848K smpl:14M ep:68K epch:227.67 loss:0.001 grdn:0.110 lr:8.4e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 00:43:39 ts/train.py:232 step:848K smpl:14M ep:68K epch:227.72 loss:0.001 grdn:0.134 lr:8.3e-07 updt_s:0.226 data_s:0.041 +INFO 2025-05-12 00:44:29 ts/train.py:232 step:848K smpl:14M ep:68K epch:227.77 loss:0.001 grdn:0.129 lr:8.3e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 00:45:18 ts/train.py:232 step:848K smpl:14M ep:68K epch:227.83 loss:0.001 grdn:0.110 lr:8.2e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 00:46:07 ts/train.py:232 step:848K smpl:14M ep:68K epch:227.88 loss:0.001 grdn:0.126 lr:8.1e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 00:46:58 ts/train.py:232 step:849K smpl:14M ep:68K epch:227.93 loss:0.001 grdn:0.107 lr:8.1e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 00:47:48 ts/train.py:232 step:849K smpl:14M ep:68K epch:227.99 loss:0.001 grdn:0.116 lr:8.0e-07 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 00:48:38 ts/train.py:232 step:849K smpl:14M ep:68K epch:228.04 loss:0.001 grdn:0.124 lr:7.9e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 00:49:28 ts/train.py:232 step:849K smpl:14M ep:68K epch:228.10 loss:0.001 grdn:0.108 lr:7.9e-07 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 00:50:19 ts/train.py:232 step:849K smpl:14M ep:68K epch:228.15 loss:0.001 grdn:0.129 lr:7.8e-07 updt_s:0.228 data_s:0.024 +INFO 2025-05-12 00:51:09 ts/train.py:232 step:850K smpl:14M ep:68K epch:228.20 loss:0.001 grdn:0.132 lr:7.8e-07 updt_s:0.228 data_s:0.023 +INFO 2025-05-12 00:51:59 ts/train.py:232 step:850K smpl:14M ep:68K epch:228.26 loss:0.001 grdn:0.125 lr:7.7e-07 updt_s:0.228 data_s:0.023 +INFO 2025-05-12 00:52:50 ts/train.py:232 step:850K smpl:14M ep:68K epch:228.31 loss:0.001 grdn:0.111 lr:7.6e-07 updt_s:0.228 data_s:0.022 +INFO 2025-05-12 00:53:40 ts/train.py:232 step:850K smpl:14M ep:69K epch:228.36 loss:0.001 grdn:0.120 lr:7.6e-07 updt_s:0.228 data_s:0.025 +INFO 2025-05-12 00:54:30 ts/train.py:232 step:850K smpl:14M ep:69K epch:228.42 loss:0.001 grdn:0.127 lr:7.5e-07 updt_s:0.228 data_s:0.024 +INFO 2025-05-12 00:55:21 ts/train.py:232 step:851K smpl:14M ep:69K epch:228.47 loss:0.001 grdn:0.137 lr:7.5e-07 updt_s:0.228 data_s:0.024 +INFO 2025-05-12 00:56:11 ts/train.py:232 step:851K smpl:14M ep:69K epch:228.53 loss:0.001 grdn:0.116 lr:7.4e-07 updt_s:0.228 data_s:0.022 +INFO 2025-05-12 00:57:01 ts/train.py:232 step:851K smpl:14M ep:69K epch:228.58 loss:0.002 grdn:0.140 lr:7.3e-07 updt_s:0.228 data_s:0.024 +INFO 2025-05-12 00:57:52 ts/train.py:232 step:851K smpl:14M ep:69K epch:228.63 loss:0.001 grdn:0.112 lr:7.3e-07 updt_s:0.228 data_s:0.025 +INFO 2025-05-12 00:58:44 ts/train.py:232 step:851K smpl:14M ep:69K epch:228.69 loss:0.001 grdn:0.122 lr:7.2e-07 updt_s:0.227 data_s:0.033 +INFO 2025-05-12 00:59:34 ts/train.py:232 step:852K smpl:14M ep:69K epch:228.74 loss:0.001 grdn:0.125 lr:7.2e-07 updt_s:0.226 data_s:0.023 +INFO 2025-05-12 01:00:24 ts/train.py:232 step:852K smpl:14M ep:69K epch:228.79 loss:0.001 grdn:0.115 lr:7.1e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 01:01:14 ts/train.py:232 step:852K smpl:14M ep:69K epch:228.85 loss:0.001 grdn:0.121 lr:7.0e-07 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 01:02:04 ts/train.py:232 step:852K smpl:14M ep:69K epch:228.90 loss:0.001 grdn:0.120 lr:7.0e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 01:02:55 ts/train.py:232 step:852K smpl:14M ep:69K epch:228.96 loss:0.001 grdn:0.115 lr:6.9e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 01:03:46 ts/train.py:232 step:853K smpl:14M ep:69K epch:229.01 loss:0.001 grdn:0.125 lr:6.9e-07 updt_s:0.227 data_s:0.027 +INFO 2025-05-12 01:04:37 ts/train.py:232 step:853K smpl:14M ep:69K epch:229.06 loss:0.001 grdn:0.121 lr:6.8e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 01:05:27 ts/train.py:232 step:853K smpl:14M ep:69K epch:229.12 loss:0.001 grdn:0.120 lr:6.7e-07 updt_s:0.227 data_s:0.027 +INFO 2025-05-12 01:06:19 ts/train.py:232 step:853K smpl:14M ep:69K epch:229.17 loss:0.001 grdn:0.127 lr:6.7e-07 updt_s:0.228 data_s:0.027 +INFO 2025-05-12 01:07:10 ts/train.py:232 step:853K smpl:14M ep:69K epch:229.22 loss:0.001 grdn:0.106 lr:6.6e-07 updt_s:0.228 data_s:0.027 +INFO 2025-05-12 01:08:01 ts/train.py:232 step:854K smpl:14M ep:69K epch:229.28 loss:0.001 grdn:0.110 lr:6.6e-07 updt_s:0.229 data_s:0.025 +INFO 2025-05-12 01:08:52 ts/train.py:232 step:854K smpl:14M ep:69K epch:229.33 loss:0.001 grdn:0.127 lr:6.5e-07 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 01:09:42 ts/train.py:232 step:854K smpl:14M ep:69K epch:229.38 loss:0.001 grdn:0.114 lr:6.5e-07 updt_s:0.228 data_s:0.025 +INFO 2025-05-12 01:10:33 ts/train.py:232 step:854K smpl:14M ep:69K epch:229.44 loss:0.001 grdn:0.115 lr:6.4e-07 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 01:11:24 ts/train.py:232 step:854K smpl:14M ep:69K epch:229.49 loss:0.001 grdn:0.118 lr:6.4e-07 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 01:12:15 ts/train.py:232 step:855K smpl:14M ep:69K epch:229.55 loss:0.001 grdn:0.113 lr:6.3e-07 updt_s:0.228 data_s:0.027 +INFO 2025-05-12 01:13:06 ts/train.py:232 step:855K smpl:14M ep:69K epch:229.60 loss:0.001 grdn:0.129 lr:6.2e-07 updt_s:0.228 data_s:0.024 +INFO 2025-05-12 01:13:58 ts/train.py:232 step:855K smpl:14M ep:69K epch:229.65 loss:0.001 grdn:0.129 lr:6.2e-07 updt_s:0.227 data_s:0.036 +INFO 2025-05-12 01:14:49 ts/train.py:232 step:855K smpl:14M ep:69K epch:229.71 loss:0.001 grdn:0.109 lr:6.1e-07 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 01:15:39 ts/train.py:232 step:855K smpl:14M ep:69K epch:229.76 loss:0.001 grdn:0.119 lr:6.1e-07 updt_s:0.228 data_s:0.020 +INFO 2025-05-12 01:16:29 ts/train.py:232 step:856K smpl:14M ep:69K epch:229.81 loss:0.001 grdn:0.107 lr:6.0e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 01:17:20 ts/train.py:232 step:856K smpl:14M ep:69K epch:229.87 loss:0.001 grdn:0.118 lr:6.0e-07 updt_s:0.229 data_s:0.026 +INFO 2025-05-12 01:18:11 ts/train.py:232 step:856K smpl:14M ep:69K epch:229.92 loss:0.001 grdn:0.120 lr:5.9e-07 updt_s:0.228 data_s:0.025 +INFO 2025-05-12 01:19:01 ts/train.py:232 step:856K smpl:14M ep:69K epch:229.98 loss:0.001 grdn:0.110 lr:5.9e-07 updt_s:0.228 data_s:0.024 +INFO 2025-05-12 01:19:52 ts/train.py:232 step:856K smpl:14M ep:69K epch:230.03 loss:0.001 grdn:0.114 lr:5.8e-07 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 01:20:43 ts/train.py:232 step:857K smpl:14M ep:69K epch:230.08 loss:0.001 grdn:0.117 lr:5.8e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 01:21:33 ts/train.py:232 step:857K smpl:14M ep:69K epch:230.14 loss:0.001 grdn:0.127 lr:5.7e-07 updt_s:0.228 data_s:0.023 +INFO 2025-05-12 01:22:24 ts/train.py:232 step:857K smpl:14M ep:69K epch:230.19 loss:0.001 grdn:0.107 lr:5.7e-07 updt_s:0.228 data_s:0.025 +INFO 2025-05-12 01:23:15 ts/train.py:232 step:857K smpl:14M ep:69K epch:230.24 loss:0.001 grdn:0.128 lr:5.6e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 01:24:06 ts/train.py:232 step:857K smpl:14M ep:69K epch:230.30 loss:0.001 grdn:0.109 lr:5.5e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 01:24:56 ts/train.py:232 step:858K smpl:14M ep:69K epch:230.35 loss:0.001 grdn:0.107 lr:5.5e-07 updt_s:0.228 data_s:0.023 +INFO 2025-05-12 01:25:47 ts/train.py:232 step:858K smpl:14M ep:69K epch:230.41 loss:0.001 grdn:0.120 lr:5.4e-07 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 01:26:38 ts/train.py:232 step:858K smpl:14M ep:69K epch:230.46 loss:0.001 grdn:0.112 lr:5.4e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 01:27:29 ts/train.py:232 step:858K smpl:14M ep:69K epch:230.51 loss:0.001 grdn:0.131 lr:5.3e-07 updt_s:0.228 data_s:0.027 +INFO 2025-05-12 01:28:20 ts/train.py:232 step:858K smpl:14M ep:69K epch:230.57 loss:0.001 grdn:0.122 lr:5.3e-07 updt_s:0.227 data_s:0.027 +INFO 2025-05-12 01:29:13 ts/train.py:232 step:859K smpl:14M ep:69K epch:230.62 loss:0.001 grdn:0.129 lr:5.2e-07 updt_s:0.227 data_s:0.037 +INFO 2025-05-12 01:30:02 ts/train.py:232 step:859K smpl:14M ep:69K epch:230.67 loss:0.001 grdn:0.119 lr:5.2e-07 updt_s:0.226 data_s:0.022 +INFO 2025-05-12 01:30:51 ts/train.py:232 step:859K smpl:14M ep:69K epch:230.73 loss:0.001 grdn:0.110 lr:5.1e-07 updt_s:0.226 data_s:0.018 +INFO 2025-05-12 01:31:41 ts/train.py:232 step:859K smpl:14M ep:69K epch:230.78 loss:0.001 grdn:0.127 lr:5.1e-07 updt_s:0.226 data_s:0.020 +INFO 2025-05-12 01:32:31 ts/train.py:232 step:859K smpl:14M ep:69K epch:230.84 loss:0.001 grdn:0.114 lr:5.0e-07 updt_s:0.226 data_s:0.023 +INFO 2025-05-12 01:33:19 ts/train.py:232 step:860K smpl:14M ep:69K epch:230.89 loss:0.001 grdn:0.116 lr:5.0e-07 updt_s:0.227 data_s:0.016 +INFO 2025-05-12 01:34:10 ts/train.py:232 step:860K smpl:14M ep:69K epch:230.94 loss:0.001 grdn:0.096 lr:4.9e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 01:34:59 ts/train.py:232 step:860K smpl:14M ep:69K epch:231.00 loss:0.001 grdn:0.126 lr:4.9e-07 updt_s:0.227 data_s:0.016 +INFO 2025-05-12 01:35:48 ts/train.py:232 step:860K smpl:14M ep:69K epch:231.05 loss:0.001 grdn:0.124 lr:4.8e-07 updt_s:0.227 data_s:0.016 +INFO 2025-05-12 01:36:37 ts/train.py:232 step:860K smpl:14M ep:69K epch:231.10 loss:0.001 grdn:0.113 lr:4.8e-07 updt_s:0.228 data_s:0.018 +INFO 2025-05-12 01:37:26 ts/train.py:232 step:861K smpl:14M ep:69K epch:231.16 loss:0.001 grdn:0.129 lr:4.8e-07 updt_s:0.228 data_s:0.017 +INFO 2025-05-12 01:38:15 ts/train.py:232 step:861K smpl:14M ep:69K epch:231.21 loss:0.001 grdn:0.116 lr:4.7e-07 updt_s:0.228 data_s:0.017 +INFO 2025-05-12 01:39:04 ts/train.py:232 step:861K smpl:14M ep:69K epch:231.27 loss:0.001 grdn:0.129 lr:4.7e-07 updt_s:0.228 data_s:0.016 +INFO 2025-05-12 01:39:54 ts/train.py:232 step:861K smpl:14M ep:69K epch:231.32 loss:0.001 grdn:0.125 lr:4.6e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 01:40:43 ts/train.py:232 step:861K smpl:14M ep:69K epch:231.37 loss:0.001 grdn:0.104 lr:4.6e-07 updt_s:0.227 data_s:0.016 +INFO 2025-05-12 01:41:32 ts/train.py:232 step:862K smpl:14M ep:69K epch:231.43 loss:0.001 grdn:0.111 lr:4.5e-07 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 01:42:21 ts/train.py:232 step:862K smpl:14M ep:69K epch:231.48 loss:0.001 grdn:0.109 lr:4.5e-07 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 01:43:10 ts/train.py:232 step:862K smpl:14M ep:69K epch:231.53 loss:0.001 grdn:0.112 lr:4.4e-07 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 01:44:03 ts/train.py:232 step:862K smpl:14M ep:69K epch:231.59 loss:0.001 grdn:0.128 lr:4.4e-07 updt_s:0.227 data_s:0.037 +INFO 2025-05-12 01:44:52 ts/train.py:232 step:862K smpl:14M ep:69K epch:231.64 loss:0.001 grdn:0.113 lr:4.3e-07 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 01:45:42 ts/train.py:232 step:863K smpl:14M ep:70K epch:231.69 loss:0.001 grdn:0.114 lr:4.3e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 01:46:31 ts/train.py:232 step:863K smpl:14M ep:70K epch:231.75 loss:0.001 grdn:0.109 lr:4.2e-07 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 01:47:21 ts/train.py:232 step:863K smpl:14M ep:70K epch:231.80 loss:0.001 grdn:0.116 lr:4.2e-07 updt_s:0.228 data_s:0.019 +INFO 2025-05-12 01:48:11 ts/train.py:232 step:863K smpl:14M ep:70K epch:231.86 loss:0.001 grdn:0.112 lr:4.1e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 01:49:00 ts/train.py:232 step:863K smpl:14M ep:70K epch:231.91 loss:0.001 grdn:0.099 lr:4.1e-07 updt_s:0.228 data_s:0.020 +INFO 2025-05-12 01:49:50 ts/train.py:232 step:864K smpl:14M ep:70K epch:231.96 loss:0.001 grdn:0.120 lr:4.1e-07 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 01:50:39 ts/train.py:232 step:864K smpl:14M ep:70K epch:232.02 loss:0.001 grdn:0.107 lr:4.0e-07 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 01:51:29 ts/train.py:232 step:864K smpl:14M ep:70K epch:232.07 loss:0.001 grdn:0.119 lr:4.0e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 01:52:18 ts/train.py:232 step:864K smpl:14M ep:70K epch:232.12 loss:0.001 grdn:0.112 lr:3.9e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 01:53:08 ts/train.py:232 step:864K smpl:14M ep:70K epch:232.18 loss:0.001 grdn:0.118 lr:3.9e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 01:53:58 ts/train.py:232 step:865K smpl:14M ep:70K epch:232.23 loss:0.001 grdn:0.111 lr:3.8e-07 updt_s:0.228 data_s:0.022 +INFO 2025-05-12 01:54:48 ts/train.py:232 step:865K smpl:14M ep:70K epch:232.29 loss:0.001 grdn:0.117 lr:3.8e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 01:55:38 ts/train.py:232 step:865K smpl:14M ep:70K epch:232.34 loss:0.001 grdn:0.116 lr:3.8e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 01:56:28 ts/train.py:232 step:865K smpl:14M ep:70K epch:232.39 loss:0.001 grdn:0.107 lr:3.7e-07 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 01:57:18 ts/train.py:232 step:865K smpl:14M ep:70K epch:232.45 loss:0.001 grdn:0.109 lr:3.7e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 01:58:08 ts/train.py:232 step:866K smpl:14M ep:70K epch:232.50 loss:0.001 grdn:0.111 lr:3.6e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 01:59:00 ts/train.py:232 step:866K smpl:14M ep:70K epch:232.55 loss:0.001 grdn:0.115 lr:3.6e-07 updt_s:0.227 data_s:0.030 +INFO 2025-05-12 01:59:49 ts/train.py:232 step:866K smpl:14M ep:70K epch:232.61 loss:0.001 grdn:0.120 lr:3.5e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 02:00:38 ts/train.py:232 step:866K smpl:14M ep:70K epch:232.66 loss:0.001 grdn:0.124 lr:3.5e-07 updt_s:0.226 data_s:0.020 +INFO 2025-05-12 02:01:28 ts/train.py:232 step:866K smpl:14M ep:70K epch:232.72 loss:0.001 grdn:0.120 lr:3.5e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 02:02:18 ts/train.py:232 step:867K smpl:14M ep:70K epch:232.77 loss:0.001 grdn:0.127 lr:3.4e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 02:03:07 ts/train.py:232 step:867K smpl:14M ep:70K epch:232.82 loss:0.001 grdn:0.102 lr:3.4e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 02:03:56 ts/train.py:232 step:867K smpl:14M ep:70K epch:232.88 loss:0.001 grdn:0.110 lr:3.3e-07 updt_s:0.227 data_s:0.016 +INFO 2025-05-12 02:04:46 ts/train.py:232 step:867K smpl:14M ep:70K epch:232.93 loss:0.001 grdn:0.124 lr:3.3e-07 updt_s:0.229 data_s:0.021 +INFO 2025-05-12 02:05:36 ts/train.py:232 step:867K smpl:14M ep:70K epch:232.98 loss:0.001 grdn:0.116 lr:3.3e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 02:06:25 ts/train.py:232 step:868K smpl:14M ep:70K epch:233.04 loss:0.001 grdn:0.118 lr:3.2e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:07:15 ts/train.py:232 step:868K smpl:14M ep:70K epch:233.09 loss:0.001 grdn:0.104 lr:3.2e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 02:08:05 ts/train.py:232 step:868K smpl:14M ep:70K epch:233.15 loss:0.001 grdn:0.127 lr:3.1e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 02:08:54 ts/train.py:232 step:868K smpl:14M ep:70K epch:233.20 loss:0.001 grdn:0.123 lr:3.1e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 02:09:44 ts/train.py:232 step:868K smpl:14M ep:70K epch:233.25 loss:0.001 grdn:0.103 lr:3.1e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:10:34 ts/train.py:232 step:869K smpl:14M ep:70K epch:233.31 loss:0.001 grdn:0.110 lr:3.0e-07 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 02:11:23 ts/train.py:232 step:869K smpl:14M ep:70K epch:233.36 loss:0.001 grdn:0.131 lr:3.0e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 02:12:13 ts/train.py:232 step:869K smpl:14M ep:70K epch:233.41 loss:0.001 grdn:0.132 lr:2.9e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:13:03 ts/train.py:232 step:869K smpl:14M ep:70K epch:233.47 loss:0.001 grdn:0.112 lr:2.9e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 02:13:55 ts/train.py:232 step:869K smpl:14M ep:70K epch:233.52 loss:0.001 grdn:0.133 lr:2.9e-07 updt_s:0.226 data_s:0.035 +INFO 2025-05-12 02:14:46 ts/train.py:232 step:870K smpl:14M ep:70K epch:233.58 loss:0.001 grdn:0.103 lr:2.8e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 02:15:36 ts/train.py:232 step:870K smpl:14M ep:70K epch:233.63 loss:0.001 grdn:0.109 lr:2.8e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 02:16:28 ts/train.py:232 step:870K smpl:14M ep:70K epch:233.68 loss:0.001 grdn:0.128 lr:2.8e-07 updt_s:0.227 data_s:0.029 +INFO 2025-05-12 02:16:28 ts/train.py:241 Checkpoint policy after step 870000 +INFO 2025-05-12 02:17:23 ts/train.py:232 step:870K smpl:14M ep:70K epch:233.74 loss:0.001 grdn:0.100 lr:2.7e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 02:18:13 ts/train.py:232 step:870K smpl:14M ep:70K epch:233.79 loss:0.002 grdn:0.138 lr:2.7e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 02:19:04 ts/train.py:232 step:871K smpl:14M ep:70K epch:233.84 loss:0.001 grdn:0.114 lr:2.7e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 02:19:54 ts/train.py:232 step:871K smpl:14M ep:70K epch:233.90 loss:0.001 grdn:0.118 lr:2.6e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 02:20:45 ts/train.py:232 step:871K smpl:14M ep:70K epch:233.95 loss:0.001 grdn:0.098 lr:2.6e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 02:21:35 ts/train.py:232 step:871K smpl:14M ep:70K epch:234.00 loss:0.001 grdn:0.114 lr:2.5e-07 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 02:22:25 ts/train.py:232 step:871K smpl:14M ep:70K epch:234.06 loss:0.001 grdn:0.109 lr:2.5e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 02:23:16 ts/train.py:232 step:872K smpl:14M ep:70K epch:234.11 loss:0.001 grdn:0.103 lr:2.5e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 02:24:07 ts/train.py:232 step:872K smpl:14M ep:70K epch:234.17 loss:0.001 grdn:0.125 lr:2.4e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 02:24:58 ts/train.py:232 step:872K smpl:14M ep:70K epch:234.22 loss:0.001 grdn:0.105 lr:2.4e-07 updt_s:0.227 data_s:0.027 +INFO 2025-05-12 02:25:48 ts/train.py:232 step:872K smpl:14M ep:70K epch:234.27 loss:0.001 grdn:0.108 lr:2.4e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 02:26:38 ts/train.py:232 step:872K smpl:14M ep:70K epch:234.33 loss:0.001 grdn:0.126 lr:2.3e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 02:27:29 ts/train.py:232 step:873K smpl:14M ep:70K epch:234.38 loss:0.001 grdn:0.124 lr:2.3e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 02:28:19 ts/train.py:232 step:873K smpl:14M ep:70K epch:234.43 loss:0.001 grdn:0.118 lr:2.3e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 02:29:13 ts/train.py:232 step:873K smpl:14M ep:70K epch:234.49 loss:0.001 grdn:0.120 lr:2.2e-07 updt_s:0.227 data_s:0.040 +INFO 2025-05-12 02:30:02 ts/train.py:232 step:873K smpl:14M ep:70K epch:234.54 loss:0.001 grdn:0.119 lr:2.2e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 02:30:52 ts/train.py:232 step:873K smpl:14M ep:70K epch:234.60 loss:0.001 grdn:0.127 lr:2.2e-07 updt_s:0.228 data_s:0.019 +INFO 2025-05-12 02:31:41 ts/train.py:232 step:874K smpl:14M ep:70K epch:234.65 loss:0.001 grdn:0.124 lr:2.1e-07 updt_s:0.228 data_s:0.018 +INFO 2025-05-12 02:32:31 ts/train.py:232 step:874K smpl:14M ep:70K epch:234.70 loss:0.001 grdn:0.104 lr:2.1e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 02:33:21 ts/train.py:232 step:874K smpl:14M ep:70K epch:234.76 loss:0.001 grdn:0.116 lr:2.1e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 02:34:11 ts/train.py:232 step:874K smpl:14M ep:70K epch:234.81 loss:0.001 grdn:0.123 lr:2.0e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:35:01 ts/train.py:232 step:874K smpl:14M ep:70K epch:234.86 loss:0.001 grdn:0.110 lr:2.0e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 02:35:51 ts/train.py:232 step:875K smpl:14M ep:70K epch:234.92 loss:0.001 grdn:0.113 lr:2.0e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 02:36:41 ts/train.py:232 step:875K smpl:14M ep:70K epch:234.97 loss:0.001 grdn:0.106 lr:2.0e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:37:30 ts/train.py:232 step:875K smpl:14M ep:71K epch:235.03 loss:0.001 grdn:0.109 lr:1.9e-07 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 02:38:20 ts/train.py:232 step:875K smpl:14M ep:71K epch:235.08 loss:0.001 grdn:0.116 lr:1.9e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:39:10 ts/train.py:232 step:875K smpl:14M ep:71K epch:235.13 loss:0.001 grdn:0.107 lr:1.9e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 02:40:00 ts/train.py:232 step:876K smpl:14M ep:71K epch:235.19 loss:0.001 grdn:0.125 lr:1.8e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:40:50 ts/train.py:232 step:876K smpl:14M ep:71K epch:235.24 loss:0.001 grdn:0.114 lr:1.8e-07 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 02:41:40 ts/train.py:232 step:876K smpl:14M ep:71K epch:235.29 loss:0.001 grdn:0.110 lr:1.8e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:42:30 ts/train.py:232 step:876K smpl:14M ep:71K epch:235.35 loss:0.001 grdn:0.106 lr:1.7e-07 updt_s:0.228 data_s:0.023 +INFO 2025-05-12 02:43:20 ts/train.py:232 step:876K smpl:14M ep:71K epch:235.40 loss:0.001 grdn:0.106 lr:1.7e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:44:12 ts/train.py:232 step:877K smpl:14M ep:71K epch:235.46 loss:0.001 grdn:0.117 lr:1.7e-07 updt_s:0.226 data_s:0.034 +INFO 2025-05-12 02:45:03 ts/train.py:232 step:877K smpl:14M ep:71K epch:235.51 loss:0.001 grdn:0.116 lr:1.7e-07 updt_s:0.226 data_s:0.027 +INFO 2025-05-12 02:45:54 ts/train.py:232 step:877K smpl:14M ep:71K epch:235.56 loss:0.002 grdn:0.131 lr:1.6e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 02:46:44 ts/train.py:232 step:877K smpl:14M ep:71K epch:235.62 loss:0.001 grdn:0.125 lr:1.6e-07 updt_s:0.226 data_s:0.025 +INFO 2025-05-12 02:47:35 ts/train.py:232 step:877K smpl:14M ep:71K epch:235.67 loss:0.001 grdn:0.117 lr:1.6e-07 updt_s:0.226 data_s:0.030 +INFO 2025-05-12 02:48:27 ts/train.py:232 step:878K smpl:14M ep:71K epch:235.72 loss:0.001 grdn:0.113 lr:1.5e-07 updt_s:0.227 data_s:0.029 +INFO 2025-05-12 02:49:17 ts/train.py:232 step:878K smpl:14M ep:71K epch:235.78 loss:0.001 grdn:0.113 lr:1.5e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 02:50:09 ts/train.py:232 step:878K smpl:14M ep:71K epch:235.83 loss:0.001 grdn:0.114 lr:1.5e-07 updt_s:0.227 data_s:0.030 +INFO 2025-05-12 02:50:59 ts/train.py:232 step:878K smpl:14M ep:71K epch:235.89 loss:0.001 grdn:0.107 lr:1.5e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 02:51:50 ts/train.py:232 step:878K smpl:14M ep:71K epch:235.94 loss:0.001 grdn:0.112 lr:1.4e-07 updt_s:0.227 data_s:0.029 +INFO 2025-05-12 02:52:41 ts/train.py:232 step:879K smpl:14M ep:71K epch:235.99 loss:0.001 grdn:0.114 lr:1.4e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 02:53:32 ts/train.py:232 step:879K smpl:14M ep:71K epch:236.05 loss:0.001 grdn:0.117 lr:1.4e-07 updt_s:0.228 data_s:0.024 +INFO 2025-05-12 02:54:23 ts/train.py:232 step:879K smpl:14M ep:71K epch:236.10 loss:0.001 grdn:0.104 lr:1.4e-07 updt_s:0.227 data_s:0.027 +INFO 2025-05-12 02:55:13 ts/train.py:232 step:879K smpl:14M ep:71K epch:236.15 loss:0.001 grdn:0.123 lr:1.3e-07 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 02:56:03 ts/train.py:232 step:879K smpl:14M ep:71K epch:236.21 loss:0.001 grdn:0.121 lr:1.3e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 02:56:54 ts/train.py:232 step:880K smpl:14M ep:71K epch:236.26 loss:0.001 grdn:0.133 lr:1.3e-07 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 02:57:44 ts/train.py:232 step:880K smpl:14M ep:71K epch:236.31 loss:0.001 grdn:0.115 lr:1.3e-07 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 02:58:35 ts/train.py:232 step:880K smpl:14M ep:71K epch:236.37 loss:0.001 grdn:0.113 lr:1.2e-07 updt_s:0.227 data_s:0.026 +INFO 2025-05-12 02:59:27 ts/train.py:232 step:880K smpl:14M ep:71K epch:236.42 loss:0.001 grdn:0.107 lr:1.2e-07 updt_s:0.227 data_s:0.033 +INFO 2025-05-12 03:00:17 ts/train.py:232 step:880K smpl:14M ep:71K epch:236.48 loss:0.001 grdn:0.128 lr:1.2e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:01:06 ts/train.py:232 step:881K smpl:14M ep:71K epch:236.53 loss:0.001 grdn:0.127 lr:1.2e-07 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 03:01:55 ts/train.py:232 step:881K smpl:14M ep:71K epch:236.58 loss:0.001 grdn:0.108 lr:1.1e-07 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:02:46 ts/train.py:232 step:881K smpl:14M ep:71K epch:236.64 loss:0.001 grdn:0.112 lr:1.1e-07 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 03:03:37 ts/train.py:232 step:881K smpl:14M ep:71K epch:236.69 loss:0.001 grdn:0.121 lr:1.1e-07 updt_s:0.229 data_s:0.029 +INFO 2025-05-12 03:04:29 ts/train.py:232 step:881K smpl:14M ep:71K epch:236.74 loss:0.001 grdn:0.107 lr:1.1e-07 updt_s:0.227 data_s:0.027 +INFO 2025-05-12 03:05:20 ts/train.py:232 step:882K smpl:14M ep:71K epch:236.80 loss:0.001 grdn:0.120 lr:1.0e-07 updt_s:0.228 data_s:0.028 +INFO 2025-05-12 03:06:11 ts/train.py:232 step:882K smpl:14M ep:71K epch:236.85 loss:0.001 grdn:0.112 lr:1.0e-07 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 03:07:02 ts/train.py:232 step:882K smpl:14M ep:71K epch:236.91 loss:0.001 grdn:0.110 lr:1.0e-07 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 03:07:53 ts/train.py:232 step:882K smpl:14M ep:71K epch:236.96 loss:0.001 grdn:0.118 lr:9.8e-08 updt_s:0.228 data_s:0.028 +INFO 2025-05-12 03:08:45 ts/train.py:232 step:882K smpl:14M ep:71K epch:237.01 loss:0.001 grdn:0.115 lr:9.6e-08 updt_s:0.228 data_s:0.029 +INFO 2025-05-12 03:09:36 ts/train.py:232 step:883K smpl:14M ep:71K epch:237.07 loss:0.001 grdn:0.122 lr:9.3e-08 updt_s:0.227 data_s:0.029 +INFO 2025-05-12 03:10:27 ts/train.py:232 step:883K smpl:14M ep:71K epch:237.12 loss:0.001 grdn:0.129 lr:9.1e-08 updt_s:0.228 data_s:0.029 +INFO 2025-05-12 03:11:18 ts/train.py:232 step:883K smpl:14M ep:71K epch:237.17 loss:0.001 grdn:0.123 lr:8.9e-08 updt_s:0.227 data_s:0.028 +INFO 2025-05-12 03:12:09 ts/train.py:232 step:883K smpl:14M ep:71K epch:237.23 loss:0.001 grdn:0.114 lr:8.7e-08 updt_s:0.228 data_s:0.025 +INFO 2025-05-12 03:13:00 ts/train.py:232 step:883K smpl:14M ep:71K epch:237.28 loss:0.001 grdn:0.118 lr:8.5e-08 updt_s:0.227 data_s:0.029 +INFO 2025-05-12 03:13:51 ts/train.py:232 step:884K smpl:14M ep:71K epch:237.34 loss:0.001 grdn:0.123 lr:8.3e-08 updt_s:0.228 data_s:0.025 +INFO 2025-05-12 03:14:44 ts/train.py:232 step:884K smpl:14M ep:71K epch:237.39 loss:0.001 grdn:0.117 lr:8.1e-08 updt_s:0.227 data_s:0.037 +INFO 2025-05-12 03:15:34 ts/train.py:232 step:884K smpl:14M ep:71K epch:237.44 loss:0.001 grdn:0.122 lr:7.9e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:16:24 ts/train.py:232 step:884K smpl:14M ep:71K epch:237.50 loss:0.001 grdn:0.111 lr:7.7e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:17:14 ts/train.py:232 step:884K smpl:14M ep:71K epch:237.55 loss:0.001 grdn:0.116 lr:7.5e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:18:03 ts/train.py:232 step:885K smpl:14M ep:71K epch:237.60 loss:0.001 grdn:0.123 lr:7.3e-08 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 03:18:52 ts/train.py:232 step:885K smpl:14M ep:71K epch:237.66 loss:0.001 grdn:0.114 lr:7.1e-08 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 03:19:41 ts/train.py:232 step:885K smpl:14M ep:71K epch:237.71 loss:0.001 grdn:0.122 lr:7.0e-08 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 03:20:29 ts/train.py:232 step:885K smpl:14M ep:71K epch:237.77 loss:0.001 grdn:0.112 lr:6.8e-08 updt_s:0.227 data_s:0.016 +INFO 2025-05-12 03:21:19 ts/train.py:232 step:885K smpl:14M ep:71K epch:237.82 loss:0.001 grdn:0.101 lr:6.6e-08 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 03:22:08 ts/train.py:232 step:886K smpl:14M ep:71K epch:237.87 loss:0.001 grdn:0.117 lr:6.4e-08 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 03:22:57 ts/train.py:232 step:886K smpl:14M ep:71K epch:237.93 loss:0.001 grdn:0.116 lr:6.2e-08 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 03:23:47 ts/train.py:232 step:886K smpl:14M ep:71K epch:237.98 loss:0.001 grdn:0.107 lr:6.1e-08 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 03:24:36 ts/train.py:232 step:886K smpl:14M ep:71K epch:238.03 loss:0.001 grdn:0.129 lr:5.9e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:25:26 ts/train.py:232 step:886K smpl:14M ep:71K epch:238.09 loss:0.001 grdn:0.122 lr:5.7e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:26:15 ts/train.py:232 step:887K smpl:14M ep:71K epch:238.14 loss:0.001 grdn:0.112 lr:5.6e-08 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 03:27:05 ts/train.py:232 step:887K smpl:14M ep:71K epch:238.20 loss:0.001 grdn:0.119 lr:5.4e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:27:54 ts/train.py:232 step:887K smpl:14M ep:71K epch:238.25 loss:0.001 grdn:0.100 lr:5.2e-08 updt_s:0.227 data_s:0.017 +INFO 2025-05-12 03:28:42 ts/train.py:232 step:887K smpl:14M ep:71K epch:238.30 loss:0.001 grdn:0.108 lr:5.1e-08 updt_s:0.227 data_s:0.015 +INFO 2025-05-12 03:29:36 ts/train.py:232 step:887K smpl:14M ep:72K epch:238.36 loss:0.001 grdn:0.117 lr:4.9e-08 updt_s:0.226 data_s:0.042 +INFO 2025-05-12 03:30:26 ts/train.py:232 step:888K smpl:14M ep:72K epch:238.41 loss:0.001 grdn:0.122 lr:4.8e-08 updt_s:0.227 data_s:0.025 +INFO 2025-05-12 03:31:16 ts/train.py:232 step:888K smpl:14M ep:72K epch:238.46 loss:0.001 grdn:0.100 lr:4.6e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:32:06 ts/train.py:232 step:888K smpl:14M ep:72K epch:238.52 loss:0.001 grdn:0.118 lr:4.5e-08 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 03:32:56 ts/train.py:232 step:888K smpl:14M ep:72K epch:238.57 loss:0.001 grdn:0.106 lr:4.3e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:33:46 ts/train.py:232 step:888K smpl:14M ep:72K epch:238.62 loss:0.001 grdn:0.108 lr:4.2e-08 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 03:34:36 ts/train.py:232 step:889K smpl:14M ep:72K epch:238.68 loss:0.001 grdn:0.113 lr:4.0e-08 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 03:35:26 ts/train.py:232 step:889K smpl:14M ep:72K epch:238.73 loss:0.001 grdn:0.112 lr:3.9e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:36:16 ts/train.py:232 step:889K smpl:14M ep:72K epch:238.79 loss:0.001 grdn:0.116 lr:3.8e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:37:07 ts/train.py:232 step:889K smpl:14M ep:72K epch:238.84 loss:0.001 grdn:0.097 lr:3.6e-08 updt_s:0.227 data_s:0.027 +INFO 2025-05-12 03:37:57 ts/train.py:232 step:889K smpl:14M ep:72K epch:238.89 loss:0.001 grdn:0.130 lr:3.5e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:38:46 ts/train.py:232 step:890K smpl:14M ep:72K epch:238.95 loss:0.001 grdn:0.125 lr:3.4e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:39:36 ts/train.py:232 step:890K smpl:14M ep:72K epch:239.00 loss:0.001 grdn:0.117 lr:3.2e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:40:26 ts/train.py:232 step:890K smpl:14M ep:72K epch:239.05 loss:0.001 grdn:0.102 lr:3.1e-08 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 03:41:17 ts/train.py:232 step:890K smpl:14M ep:72K epch:239.11 loss:0.001 grdn:0.126 lr:3.0e-08 updt_s:0.227 data_s:0.027 +INFO 2025-05-12 03:42:07 ts/train.py:232 step:890K smpl:14M ep:72K epch:239.16 loss:0.001 grdn:0.117 lr:2.9e-08 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 03:42:57 ts/train.py:232 step:891K smpl:14M ep:72K epch:239.22 loss:0.001 grdn:0.114 lr:2.8e-08 updt_s:0.228 data_s:0.021 +INFO 2025-05-12 03:43:46 ts/train.py:232 step:891K smpl:14M ep:72K epch:239.27 loss:0.001 grdn:0.111 lr:2.6e-08 updt_s:0.228 data_s:0.018 +INFO 2025-05-12 03:44:38 ts/train.py:232 step:891K smpl:14M ep:72K epch:239.32 loss:0.001 grdn:0.123 lr:2.5e-08 updt_s:0.228 data_s:0.028 +INFO 2025-05-12 03:45:27 ts/train.py:232 step:891K smpl:14M ep:72K epch:239.38 loss:0.002 grdn:0.132 lr:2.4e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:46:17 ts/train.py:232 step:891K smpl:14M ep:72K epch:239.43 loss:0.001 grdn:0.116 lr:2.3e-08 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 03:47:06 ts/train.py:232 step:892K smpl:14M ep:72K epch:239.48 loss:0.001 grdn:0.118 lr:2.2e-08 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 03:47:55 ts/train.py:232 step:892K smpl:14M ep:72K epch:239.54 loss:0.001 grdn:0.106 lr:2.1e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:48:45 ts/train.py:232 step:892K smpl:14M ep:72K epch:239.59 loss:0.001 grdn:0.111 lr:2.0e-08 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 03:49:34 ts/train.py:232 step:892K smpl:14M ep:72K epch:239.65 loss:0.001 grdn:0.124 lr:1.9e-08 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:50:24 ts/train.py:232 step:892K smpl:14M ep:72K epch:239.70 loss:0.001 grdn:0.132 lr:1.8e-08 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 03:51:14 ts/train.py:232 step:893K smpl:14M ep:72K epch:239.75 loss:0.001 grdn:0.124 lr:1.7e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:52:04 ts/train.py:232 step:893K smpl:14M ep:72K epch:239.81 loss:0.001 grdn:0.109 lr:1.6e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:52:53 ts/train.py:232 step:893K smpl:14M ep:72K epch:239.86 loss:0.001 grdn:0.125 lr:1.5e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:53:43 ts/train.py:232 step:893K smpl:14M ep:72K epch:239.91 loss:0.001 grdn:0.118 lr:1.5e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:54:33 ts/train.py:232 step:893K smpl:14M ep:72K epch:239.97 loss:0.001 grdn:0.103 lr:1.4e-08 updt_s:0.228 data_s:0.021 +INFO 2025-05-12 03:55:25 ts/train.py:232 step:894K smpl:14M ep:72K epch:240.02 loss:0.001 grdn:0.124 lr:1.3e-08 updt_s:0.227 data_s:0.034 +INFO 2025-05-12 03:56:15 ts/train.py:232 step:894K smpl:14M ep:72K epch:240.08 loss:0.001 grdn:0.123 lr:1.2e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:57:04 ts/train.py:232 step:894K smpl:14M ep:72K epch:240.13 loss:0.001 grdn:0.116 lr:1.1e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:57:54 ts/train.py:232 step:894K smpl:14M ep:72K epch:240.18 loss:0.001 grdn:0.126 lr:1.1e-08 updt_s:0.227 data_s:0.020 +INFO 2025-05-12 03:58:44 ts/train.py:232 step:894K smpl:14M ep:72K epch:240.24 loss:0.001 grdn:0.106 lr:9.9e-09 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 03:59:37 ts/train.py:232 step:895K smpl:14M ep:72K epch:240.29 loss:0.001 grdn:0.113 lr:9.2e-09 updt_s:0.226 data_s:0.039 +INFO 2025-05-12 04:00:27 ts/train.py:232 step:895K smpl:14M ep:72K epch:240.34 loss:0.001 grdn:0.108 lr:8.6e-09 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 04:01:16 ts/train.py:232 step:895K smpl:14M ep:72K epch:240.40 loss:0.001 grdn:0.106 lr:7.9e-09 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 04:02:06 ts/train.py:232 step:895K smpl:14M ep:72K epch:240.45 loss:0.001 grdn:0.119 lr:7.3e-09 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 04:02:56 ts/train.py:232 step:895K smpl:14M ep:72K epch:240.50 loss:0.001 grdn:0.109 lr:6.7e-09 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 04:03:46 ts/train.py:232 step:896K smpl:14M ep:72K epch:240.56 loss:0.001 grdn:0.136 lr:6.2e-09 updt_s:0.227 data_s:0.024 +INFO 2025-05-12 04:04:37 ts/train.py:232 step:896K smpl:14M ep:72K epch:240.61 loss:0.001 grdn:0.107 lr:5.6e-09 updt_s:0.228 data_s:0.024 +INFO 2025-05-12 04:05:26 ts/train.py:232 step:896K smpl:14M ep:72K epch:240.67 loss:0.001 grdn:0.108 lr:5.1e-09 updt_s:0.228 data_s:0.021 +INFO 2025-05-12 04:06:17 ts/train.py:232 step:896K smpl:14M ep:72K epch:240.72 loss:0.002 grdn:0.136 lr:4.6e-09 updt_s:0.227 data_s:0.023 +INFO 2025-05-12 04:07:07 ts/train.py:232 step:896K smpl:14M ep:72K epch:240.77 loss:0.001 grdn:0.114 lr:4.2e-09 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 04:07:57 ts/train.py:232 step:897K smpl:14M ep:72K epch:240.83 loss:0.001 grdn:0.107 lr:3.7e-09 updt_s:0.227 data_s:0.022 +INFO 2025-05-12 04:08:46 ts/train.py:232 step:897K smpl:14M ep:72K epch:240.88 loss:0.001 grdn:0.107 lr:3.3e-09 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 04:09:36 ts/train.py:232 step:897K smpl:14M ep:72K epch:240.93 loss:0.001 grdn:0.114 lr:2.9e-09 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 04:10:26 ts/train.py:232 step:897K smpl:14M ep:72K epch:240.99 loss:0.001 grdn:0.104 lr:2.6e-09 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 04:11:15 ts/train.py:232 step:897K smpl:14M ep:72K epch:241.04 loss:0.001 grdn:0.094 lr:2.2e-09 updt_s:0.228 data_s:0.020 +INFO 2025-05-12 04:12:05 ts/train.py:232 step:898K smpl:14M ep:72K epch:241.10 loss:0.001 grdn:0.107 lr:1.9e-09 updt_s:0.228 data_s:0.020 +INFO 2025-05-12 04:12:54 ts/train.py:232 step:898K smpl:14M ep:72K epch:241.15 loss:0.001 grdn:0.123 lr:1.6e-09 updt_s:0.227 data_s:0.018 +INFO 2025-05-12 04:13:44 ts/train.py:232 step:898K smpl:14M ep:72K epch:241.20 loss:0.001 grdn:0.116 lr:1.3e-09 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 04:14:36 ts/train.py:232 step:898K smpl:14M ep:72K epch:241.26 loss:0.001 grdn:0.122 lr:1.1e-09 updt_s:0.228 data_s:0.033 +INFO 2025-05-12 04:15:25 ts/train.py:232 step:898K smpl:14M ep:72K epch:241.31 loss:0.001 grdn:0.121 lr:8.8e-10 updt_s:0.227 data_s:0.019 +INFO 2025-05-12 04:16:15 ts/train.py:232 step:899K smpl:14M ep:72K epch:241.36 loss:0.001 grdn:0.092 lr:6.9e-10 updt_s:0.227 data_s:0.021 +INFO 2025-05-12 04:17:05 ts/train.py:232 step:899K smpl:14M ep:72K epch:241.42 loss:0.001 grdn:0.107 lr:5.2e-10 updt_s:0.228 data_s:0.021 +INFO 2025-05-12 04:17:56 ts/train.py:232 step:899K smpl:14M ep:72K epch:241.47 loss:0.001 grdn:0.106 lr:3.7e-10 updt_s:0.228 data_s:0.027 +INFO 2025-05-12 04:18:47 ts/train.py:232 step:899K smpl:14M ep:72K epch:241.53 loss:0.001 grdn:0.108 lr:2.5e-10 updt_s:0.228 data_s:0.027 +INFO 2025-05-12 04:19:38 ts/train.py:232 step:899K smpl:14M ep:72K epch:241.58 loss:0.001 grdn:0.113 lr:1.5e-10 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 04:20:28 ts/train.py:232 step:900K smpl:14M ep:72K epch:241.63 loss:0.001 grdn:0.129 lr:7.7e-11 updt_s:0.228 data_s:0.024 +INFO 2025-05-12 04:21:19 ts/train.py:232 step:900K smpl:14M ep:73K epch:241.69 loss:0.001 grdn:0.112 lr:2.8e-11 updt_s:0.228 data_s:0.026 +INFO 2025-05-12 04:22:11 ts/train.py:232 step:900K smpl:14M ep:73K epch:241.74 loss:0.001 grdn:0.116 lr:4.0e-12 updt_s:0.228 data_s:0.027 +INFO 2025-05-12 04:22:11 ts/train.py:241 Checkpoint policy after step 900000 +INFO 2025-05-12 04:22:15 ts/train.py:283 End of training diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/wandb-summary.json b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..1bf9284d812f0a15f5109e260f5f644c93de4ca8 --- /dev/null +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/samples":14400000,"_runtime":231765.543775861,"train/loss":0.0012944877906193141,"train/dataloading_s":0.02728448060224764,"_timestamp":1.7470237310127118e+09,"train/episodes":72522.159548751,"_wandb":{"runtime":231765},"_step":900000,"train/steps":900000,"train/grad_norm":0.11593213830608874,"train/epochs":241.74053182917004,"train/update_s":0.22757414581428748,"train/lr":4.0356373227079544e-12} \ No newline at end of file diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug-core.log b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug-core.log index cd5a401da302a5c65e1f2ac42d142f2bc0603c85..844a77461b172dc1d40c20bfb42a308db81c3c5a 100644 --- a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug-core.log +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug-core.log @@ -4,3 +4,11 @@ {"time":"2025-05-09T11:59:30.521357745Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:57280"} {"time":"2025-05-09T11:59:30.534207597Z","level":"INFO","msg":"handleInformInit: received","streamId":"el8imly4","id":"127.0.0.1:57280"} {"time":"2025-05-09T11:59:31.002997761Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"el8imly4","id":"127.0.0.1:57280"} +{"time":"2025-05-12T04:22:16.075052651Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:57280"} +{"time":"2025-05-12T04:22:16.075328383Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-05-12T04:22:16.075268879Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:57280"} +{"time":"2025-05-12T04:22:16.075620876Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:57280"} +{"time":"2025-05-12T04:22:35.265223761Z","level":"ERROR","msg":"processOutgoingData: flush error","error":"write tcp 127.0.0.1:34171->127.0.0.1:57280: use of closed network connection","id":"127.0.0.1:57280"} +{"time":"2025-05-12T04:22:35.587713787Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:57280"} +{"time":"2025-05-12T04:22:35.587806466Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:57280"} +{"time":"2025-05-12T04:22:35.587849488Z","level":"INFO","msg":"server is closed"} diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug-internal.log b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug-internal.log index b261b7a09f53560365718d3e08a49ef1eb07e7a3..3847323a6b72e5124a1aa9dbbf2fd9f584e878f8 100644 --- a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug-internal.log +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug-internal.log @@ -8,3 +8,12 @@ {"time":"2025-05-09T16:51:02.122798906Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/jaesunghan-postech/lerobot/el8imly4/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} {"time":"2025-05-09T19:47:55.073959875Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/jaesunghan-postech/lerobot/el8imly4/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} {"time":"2025-05-10T23:50:18.71286255Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"} +{"time":"2025-05-12T02:21:51.227152305Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/jaesunghan-postech/lerobot/el8imly4/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-05-12T04:22:16.076513163Z","level":"INFO","msg":"stream: closing","id":"el8imly4"} +{"time":"2025-05-12T04:22:16.076609613Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-05-12T04:22:16.078445309Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-05-12T04:22:35.315504548Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-05-12T04:22:35.582918725Z","level":"INFO","msg":"writer: Close: closed","stream_id":"el8imly4"} +{"time":"2025-05-12T04:22:35.583471795Z","level":"INFO","msg":"handler: closed","stream_id":"el8imly4"} +{"time":"2025-05-12T04:22:35.585413928Z","level":"INFO","msg":"sender: closed","stream_id":"el8imly4"} +{"time":"2025-05-12T04:22:35.585504302Z","level":"INFO","msg":"stream: closed","id":"el8imly4"} diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug.log b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug.log index 7f63278ff5c72d19cdd80cac91af806f529d6fc2..7655023cce3cae4d2c1b21971ad2cb7225bc6f3d 100644 --- a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug.log +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/logs/debug.log @@ -20,3 +20,4 @@ config: {'dataset': {'repo_id': 'shylee/so100_pengripF', 'root': '/SSD/LSY/lerob 2025-05-09 11:59:31,470 INFO MainThread:5678 [wandb_run.py:_redirect():2371] Wrapping output streams. 2025-05-09 11:59:31,470 INFO MainThread:5678 [wandb_run.py:_redirect():2394] Redirects installed. 2025-05-09 11:59:31,472 INFO MainThread:5678 [wandb_init.py:init():1056] run started, returning control to user process +2025-05-12 04:22:16,072 INFO MsgRouterThr:5678 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles. diff --git a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/run-el8imly4.wandb b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/run-el8imly4.wandb index 0c905578eac907df6ed4765bcc27aef88ba35bd4..9b4315c873ee74a1df474fb32846bb4fda1b9a2b 100644 --- a/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/run-el8imly4.wandb +++ b/DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/run-el8imly4.wandb @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9429930aa83d1f2b232bcb02b62af2bce1ef1d30a75804fd2359a8fcb6d725ad -size 43024384 +oid sha256:ed0fd55a08a213e52372c06ce2ebca353eb10aef8315ffa225c20c1482307deb +size 53945167