diff --git a/.gitattributes b/.gitattributes index 829ae0bb8ab918bca522ef827692380af8cda669..7a908a0543e2274bea074518f0c5479946d16b89 100644 --- a/.gitattributes +++ b/.gitattributes @@ -49,3 +49,4 @@ DP_pengripA_downDims1_cropNo_freeze0_32_32_ema1_1e-4/wandb/run-20250505_135012-f DP_pengripF_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250509_115930-el8imly4/run-el8imly4.wandb filter=lfs diff=lfs merge=lfs -text wandb/run-20250518_145405-bqr8y636/run-bqr8y636.wandb filter=lfs diff=lfs merge=lfs -text DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250518_145405-bqr8y636/run-bqr8y636.wandb filter=lfs diff=lfs merge=lfs -text +DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/run-fekph8dz.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..297cd6e370ff597f9d14ec02c5c7258232971448 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccdc03e56aa76cf39bd4a4605af2048c1f1eb2d9e8abcdb7ba64f348c3f0648d +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..aa003ad0b0e34a304a9af4c8f597fafbcf32f279 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.973917293499732e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cf092ab2d93bc0b6a7ad28cd7d97a8ec7c2084e8 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e0fe6ab24a480728c4ef6febb63b90490ec89b40f3717042babe0dc97fc8e4e +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a8ab525fe736f10bd171b62e26543e7c023d99d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5191452353ab75313ddc014f5dc5cf2569172ba4fdad971263bbb73cbd92b6f +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e07ff884b0d364fa893f5d7562dad3c5b7ebf570 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 20000, + "verbose": false, + "_step_count": 20001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.973917293499732e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..dc9bb47026c5d5237ca6fc5dbff6020dd122ea05 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/020000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 20000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..519c8b48fe585ad1ff4fbe48701356ad5fa4da92 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd03f2fa6c0dbb232f6350bb929e8292c5228b02a5e82d8ef7a515abf46af61 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..345d972bf402fedb115c90bd8517a7edc4162067 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.893265615722554e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a8feba65de342aa19d9956a08e353de076f83ea4 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7012d287add5206acae22e19870021ce596c47e5e263e762b639149ba9ae91ec +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..87de4d5a24efaaf0a7524989a182ba1d9273c753 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d10dbe23d34d1648573dc3a4c42ba7aa08b240741d2f5003c7f94bb456590a46 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3a51b969a2d4c582c4b9a18d77b760e2bf5b5d45 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 40000, + "verbose": false, + "_step_count": 40001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.893265615722554e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..fe40d8ff4312c7e5a8fba9bcc932a43a1384ba77 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/040000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 40000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a1d80b0dc23590cc04c2ae796818f84ab462b7b3 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46044a3ba88a76e9284fb773631d77b27bab1386ef86a86d1d2ddf1d657559ce +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1d7b57b3d2a5aa8dbee12f1a0bf1eb96e293fc17 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.758912913234665e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2aef4bc70892d8d30c30a4983624a1e5fb193093 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e795fcd0789d7b2395e3c6600694f0f471c107a11376b680594ce61b969478 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2cbd2b884f09ad2e25c7d38a874ea0575db6c97f --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:970792d1f80bf406842fb8e8bcb92f95f5113f0a38adfb8e346882f48c91e98a +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..237dbb61e3a1f02f88082684d10772bad73adc3d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 60000, + "verbose": false, + "_step_count": 60001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.758912913234665e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..d360b6037f1c8c2c85d38e951160eb02ace507dd --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/060000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 60000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..26293cd9581d81c3683d454cdc8d572f327a6a7f --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8b42d32d5a23f89273e84788a96f9b917661d4e0c53319960618d62ffdf328b +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..908e19c77d80298db94eb80b936bf02fb5d0f009 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.57233363650797e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c8236becae8c3957c5b45c5226f66af59f4fa41f --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6770c5f55d25e94bd745afbec5d9395b49363cfb1a1a4177b3480b4347e650a +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6878c3d7c68e2aa736cde0577c5d91e38ac19d31 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d60d03d8df17a0eca8807fb0ff03e7e350e0466e0f675c169c433dd40fd51aed +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..57f53aeabce0ff7806f8de9150096c47a2c348c1 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 80000, + "verbose": false, + "_step_count": 80001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.57233363650797e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..384eace4ecc2a6cba352aa7cf27f04405b7319c3 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/080000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 80000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..98abbfc4763efdd8680f261d047d7e25b71084f4 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aef061f03e352dd8ae22bbb34eb8e8fcd497e995f654e47b8274eb2698ec067 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..cb90a233f373112d9b6786195712cf7a0aeed710 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.335575395285501e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fc77cc189ff50a30ac5a7813fe1c31287710cdf3 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f05a83906aeb6b2390ba41caed84b636add4f0b852d7dc52e4af92882272d049 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fb60dada963e5feab3af340835c99d4b76db9268 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:975c6d3032b0ece7f40789793c67d553b359f0e12c7a7cf3281de77b5fa4813f +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..670853df50335e72fdd4a7d1b2e12fe2791b56ca --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 100000, + "verbose": false, + "_step_count": 100001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.335575395285501e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8eb73c13d28bca88058c08796abbf931c3f9b012 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/100000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 100000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5ee560edb269eaa3a2c52cd2874e90ffef9274a9 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5557557c896527f58abdbb567fb2595515c0e10044e89a84900fbcbb531bce2e +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..a022309f8b5790a24d2c2ae3712e9fe8ee13a355 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 9.051236487138009e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0fdf22c09c2ca39b4eced296849e75a52b48cdf7 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5bf3cdd0834be074347838a2bda0bf5621b87d319f7a087cf619da35eb90d0b +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a425018baee7987b036368e5afc7e9e95219569e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7bf032f5c6bc48df5b079a5138cf9a0a81ff0756367e7bb021355d963324861 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6cacfee16be30991cd2bd679d7590251a88d5315 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 120000, + "verbose": false, + "_step_count": 120001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 9.051236487138009e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..6bbb8719463d819c865c104597046a0165eea278 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/120000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 120000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..12880058a443573d9f6b4fe9494f788fde42c9ac --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b09a029a93a1782a1f5e92613f3d54ce5141dd67ca096939170fa87121029283 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..f58510c903f73cf4548e05a76c530b586efad5f3 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 8.722437382510338e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5cb2603d07ef9cf41c4aedee3aae644f81da4f54 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7547fa15dbf785c460a287eff6347992ae26540667e7e5705cc2157b7840392c +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..073bda7116500f53d5899ba3ec143edfc8658221 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfb51770840de721cb60e5b9a02c0936fff94661c6933c7513025f982b903f44 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ee52deda2ab8f53ea2f79efda948ecce44d4d94b --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 140000, + "verbose": false, + "_step_count": 140001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 8.722437382510338e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..f3695dfae1578a72a581cd61c827ae9be8dcaa64 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/140000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 140000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..721967e1633d8c3b55d44783e51b739edccf997c --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bdb1eb29ad58b2541fd3ff45f548e5f962cb2e093ace8ab081ceac24ebacb65 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..df717e77edcbc3832d43c2d52bce1b36da2e4a9d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 8.352786479194288e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..30a96633cdae4abc5278e8eafe761509e9266af2 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e395fa8b28984e476491cb7981a68e55e4a6ea015d31f4cecff08afe7bf3448 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f713585d52cefb8f6df4a77c520f59584d45fc5f --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f968f6f42bc4fb409f727553e80be56349f217482f0ce098c929da43a918659c +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1e4b24e76c166b56837370489e4534daa52e6f5e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 160000, + "verbose": false, + "_step_count": 160001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 8.352786479194288e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..18f112ab3b2ba641c0049e21daf4fb129368a0a0 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/160000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 160000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..15da2f50f3154900268d6550726fb0a122516602 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebacae5d32d1d345e8153494e35fe65b171897e906afc85de9161203efe2be07 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..42a429bd8a00ddcb698011fd8613aeb7769a34e7 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.946340502054646e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d64b550c14733da9ec1cfcdf805cec9df0ae1f03 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62a2ed10deecfd2bdd4d69a5f4cfc3b8d0d00c8c3971e618c7948449c2028a52 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dbe66f5fae7993fa4668195ec732268609503fc7 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39232fcd5fec0df4a3d1c859e479056a20e4c8f3aaf518df3dd7e0d3a28409cd +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a0e9ab8b908ccab659617e705b0134f2b318bb5e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 180000, + "verbose": false, + "_step_count": 180001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.946340502054646e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..3da013f3092359f6c4ebb97dfc69e8300dbb32e4 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/180000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 180000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..716d3c0e4b4610ef7413c6e62705414ae8a0ce11 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f71f4770e416cc3a442b887d6bb0105657fadc2f29d5af040adbf0be34bf2e8 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..fa28b60189e65bcfbb95e8f7368c81dbae309597 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.507559982600643e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2a7ea232c028c0a77968b8ceb6a67bbef19182ce --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22dc83be0b609eaef323262371016ecf6501780e91691fd22c3b81241ec1f34e +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3ef366ff1460548605fc718d6bca0996b2cb3b28 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54ab13c3b08bb3482f6e98191247ed92434274dd9fd73956ab4929542ed48858 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..91019bcb16306bf4a90aebee8436df7053314b29 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 200000, + "verbose": false, + "_step_count": 200001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.507559982600643e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..bcef8d8b05253341579ace0987c46cee6885c961 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/200000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 200000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8856aa5bf2b9f56d1810dcb7a06b783e270ecd07 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d435245db1de6629150575f6a46f9f1965fd14f9c1416b213dfed8243972aeea +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..085d2529e084de5d400ba141c5df556f1e7b7507 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.041260306991176e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..71b2f16dc4ca3f4f7aa7442a62be8dd24b8fbe2e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e4136088fc856e6b10694485a1b68a4d67878854a3ed36e27cb797ba41852a9 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..25b5b63e597f44b764aa217e6d31771246307a6d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7b6716c6a2a1f2541538a311172f3c7c3afc18c699624d6af53860d36679f37 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3a9c4ddc05d881f2eaaaceffac89b8c11773f503 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 220000, + "verbose": false, + "_step_count": 220001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.041260306991176e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..fec2a7fdbd24d5c19d3c0b86e829eecf3e0004b3 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/220000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 220000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5521492613f26a23f1e29c90614402237c685219 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd38c492c410fd8aac4656bc9c7e40b223e6a79c5c3660d493122eab810800a +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..5905ea4a21bdf270da763d564266f7d40af2ffbd --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.552558869696321e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b1647f564d1c85ce6ea981456dcc1f2f36e4915c --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71ae61002c436673f068389fbf261f526f073d817ed4091c2c698ca3f12498de +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..985debddb17f17a40c103b43e79321e774caf020 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe5967a6d88a3888c5e32b9e16fe75fdba6d21ba1b7bf00722599aafd4c3c720 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a7b79021017938bc41c6023516ce4dd338e203bf --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 240000, + "verbose": false, + "_step_count": 240001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.552558869696321e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..cd776dcc291fdcc40079da5c594255ca04238bce --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/240000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 240000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f43b258e8daf5ff4892a0a0df9fb3794eaf7ff6a --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f36020afb3d5ccf1fe9df8fc559f515aeb04c146938a4e0a5d767216808922dc +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e8695a3fd920e5cfb89d502c3060a72246eb6223 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "resume": false, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 600000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": null, + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..02a4c12546a584ae8c5dd95e3349e85898944cc7 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.04681891277594e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..24dde1210dbda6287bf016e259eda83926643096 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25c0f810b45a4e842bde3b31ff348765fd0efd74ee272144106217ef78ab22a4 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fc14e90b204aebb123812b43818cde811ac8af52 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:594268e32bb78b65a522adcd616bba8e1c6f7b7b47819922358a1cb3ce322495 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bef7e5f6b70c0295c439fe3d4a7243905136953e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 260000, + "verbose": false, + "_step_count": 260001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.04681891277594e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..2e44498dfcc90aab5b8d8abc1e091f54a4d70426 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/checkpoints/260000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 260000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/debug-internal.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..dac689ddf9885760f57acfe786773ab591878c31 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-05-18T14:52:55.339058052Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-core.log"} +{"time":"2025-05-18T14:52:55.67042234Z","level":"INFO","msg":"created new stream","id":"fekph8dz"} +{"time":"2025-05-18T14:52:55.670496162Z","level":"INFO","msg":"stream: started","id":"fekph8dz"} +{"time":"2025-05-18T14:52:55.670554828Z","level":"INFO","msg":"sender: started","stream_id":"fekph8dz"} +{"time":"2025-05-18T14:52:55.670556224Z","level":"INFO","msg":"writer: Do: started","stream_id":"fekph8dz"} +{"time":"2025-05-18T14:52:55.670618731Z","level":"INFO","msg":"handler: started","stream_id":"fekph8dz"} +{"time":"2025-05-18T14:52:56.221542859Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-05-19T14:27:28.106339931Z","level":"INFO","msg":"stream: closing","id":"fekph8dz"} +{"time":"2025-05-19T14:27:28.106399575Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-05-19T14:27:28.106543378Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-05-19T14:27:29.143033663Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-05-19T14:27:29.418524413Z","level":"INFO","msg":"handler: closed","stream_id":"fekph8dz"} +{"time":"2025-05-19T14:27:29.418600121Z","level":"INFO","msg":"writer: Close: closed","stream_id":"fekph8dz"} +{"time":"2025-05-19T14:27:29.418619047Z","level":"INFO","msg":"sender: closed","stream_id":"fekph8dz"} +{"time":"2025-05-19T14:27:29.418794627Z","level":"INFO","msg":"stream: closed","id":"fekph8dz"} diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/debug.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0fa511745ddef851e37b373283ff9f07c3755070 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/debug.log @@ -0,0 +1,23 @@ +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Configure stats pid to 18114 +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug.log +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-internal.log +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:init():781] calling init triggers +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'shylee/so100_pengripG', 'root': '/SSD/LSY/lerobot_dataset/pengripG', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ac': False, 'ac_threshold': 0.99, 'use_pcg': False, 'pcg_weight': 0.1, 'use_bid': False, 'bid_batch_size': 15, 'bid_mode_size': 3, 'bid_temporal_coherence': 0.5, 'weak_model_path': None, 'weak_model_config': None}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain', 'job_name': 'DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 600000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:init():809] starting backend +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:init():813] sending inform_init request +2025-05-18 14:52:55,337 INFO MainThread:18114 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-05-18 14:52:55,337 INFO MainThread:18114 [wandb_init.py:init():823] backend started and connected +2025-05-18 14:52:55,472 INFO MainThread:18114 [wandb_init.py:init():915] updated telemetry +2025-05-18 14:52:55,737 INFO MainThread:18114 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-05-18 14:52:56,212 INFO MainThread:18114 [wandb_init.py:init():1014] starting run threads in backend +2025-05-18 14:52:56,355 INFO MainThread:18114 [wandb_run.py:_console_start():2454] atexit reg +2025-05-18 14:52:56,355 INFO MainThread:18114 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-05-18 14:52:56,355 INFO MainThread:18114 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-05-18 14:52:56,355 INFO MainThread:18114 [wandb_run.py:_redirect():2394] Redirects installed. +2025-05-18 14:52:56,356 INFO MainThread:18114 [wandb_init.py:init():1056] run started, returning control to user process +2025-05-19 14:27:28,103 INFO MsgRouterThr:18114 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/config.yaml b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc1c27ba7823ab3e97b2ca72f1a90e8c5c648b6d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/config.yaml @@ -0,0 +1,197 @@ +_wandb: + value: + cli_version: 0.19.9 + m: [] + python_version: 3.10.17 + t: + "1": + - 1 + - 41 + - 49 + - 51 + - 55 + "2": + - 1 + - 41 + - 49 + - 51 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.10.17 + "5": 0.19.9 + "8": + - 5 + "12": 0.19.9 + "13": linux-x86_64 +batch_size: + value: 16 +dataset: + value: + episodes: null + image_transforms: + enable: false + max_num_transforms: 3 + random_order: false + tfs: + brightness: + kwargs: + brightness: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + contrast: + kwargs: + contrast: + - 0.8 + - 1.2 + type: ColorJitter + weight: 1 + hue: + kwargs: + hue: + - -0.05 + - 0.05 + type: ColorJitter + weight: 1 + saturation: + kwargs: + saturation: + - 0.5 + - 1.5 + type: ColorJitter + weight: 1 + sharpness: + kwargs: + sharpness: + - 0.5 + - 1.5 + type: SharpnessJitter + weight: 1 + repo_id: shylee/so100_pengripG + revision: null + root: /SSD/LSY/lerobot_dataset/pengripG + use_imagenet_stats: true + video_backend: torchcodec +env: + value: null +eval: + value: + batch_size: 50 + n_episodes: 50 + use_async_envs: false +eval_freq: + value: 20000 +job_name: + value: DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain +log_freq: + value: 200 +num_workers: + value: 4 +optimizer: + value: + betas: + - 0.95 + - 0.999 + eps: 1e-08 + grad_clip_norm: 10 + lr: 0.0001 + type: adam + weight_decay: 1e-06 +output_dir: + value: /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain +policy: + value: + ac_threshold: 0.99 + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + bid_batch_size: 15 + bid_mode_size: 3 + bid_temporal_coherence: 0.5 + clip_sample: true + clip_sample_range: 1 + crop_is_random: false + crop_shape: + - 224 + - 224 + device: cuda + diffusion_step_embed_dim: 128 + do_mask_loss_for_padding: false + down_dims: + - 256 + - 512 + - 1024 + drop_n_last_frames: 7 + freeze_vision_backbone: false + horizon: 16 + kernel_size: 5 + n_action_steps: 8 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler_type: DDIM + normalization_mapping: + ACTION: MIN_MAX + STATE: MIN_MAX + VISUAL: MEAN_STD + num_inference_steps: 10 + num_train_timesteps: 100 + optimizer_betas: + - 0.95 + - 0.999 + optimizer_eps: 1e-08 + optimizer_lr: 0.0001 + optimizer_weight_decay: 1e-06 + pcg_weight: 0.1 + pre_resize_shape: + - 224 + - 224 + prediction_type: epsilon + pretrained_backbone_weights: ResNet50_Weights.IMAGENET1K_V1 + scheduler_name: cosine + scheduler_warmup_steps: 500 + spatial_softmax_num_keypoints: 32 + type: diffusion + use_ac: false + use_amp: false + use_bid: false + use_film_scale_modulation: true + use_group_norm: false + use_pcg: false + use_separate_rgb_encoder_per_camera: false + vision_backbone: resnet50 + weak_model_config: null + weak_model_path: null +resume: + value: false +save_checkpoint: + value: true +save_freq: + value: 20000 +scheduler: + value: + name: cosine + num_warmup_steps: 500 + type: diffuser +seed: + value: 1000 +steps: + value: 600000 +use_policy_training_preset: + value: true +wandb: + value: + disable_artifact: false + enable: true + entity: null + mode: null + notes: null + project: lerobot + run_id: null diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/output.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..79701353682f6bf535ede3b448baeb75496114d8 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/output.log @@ -0,0 +1,1487 @@ +Logs will be synced with wandb. +INFO 2025-05-18 14:52:56 ndb_utils.py:96 Track this run --> https://wandb.ai/marchmelo0923-postech/lerobot/runs/fekph8dz +INFO 2025-05-18 14:52:56 ts/train.py:127 Creating dataset +Resolving data files: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 300/300 [00:00<00:00, 185369.95it/s] +INFO 2025-05-18 14:52:58 ts/train.py:138 Creating policy +INFO 2025-05-18 14:52:59 ts/train.py:144 Creating optimizer and scheduler +INFO 2025-05-18 14:52:59 ts/train.py:156 Output dir: /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain +INFO 2025-05-18 14:52:59 ts/train.py:159 cfg.steps=600000 (600K) +INFO 2025-05-18 14:52:59 ts/train.py:160 dataset.num_frames=63435 (63K) +INFO 2025-05-18 14:52:59 ts/train.py:161 dataset.num_episodes=300 +INFO 2025-05-18 14:52:59 ts/train.py:162 num_learnable_params=92242982 (92M) +INFO 2025-05-18 14:52:59 ts/train.py:163 num_total_params=92243036 (92M) +INFO 2025-05-18 14:52:59 ts/train.py:202 Start offline training on a fixed dataset +INFO 2025-05-18 14:54:06 ts/train.py:232 step:200 smpl:3K ep:15 epch:0.05 loss:0.795 grdn:3.545 lr:2.0e-05 updt_s:0.243 data_s:0.090 +INFO 2025-05-18 14:55:10 ts/train.py:232 step:400 smpl:6K ep:30 epch:0.10 loss:0.162 grdn:2.700 lr:6.0e-05 updt_s:0.222 data_s:0.102 +INFO 2025-05-18 14:56:15 ts/train.py:232 step:600 smpl:10K ep:45 epch:0.15 loss:0.104 grdn:2.033 lr:9.5e-05 updt_s:0.222 data_s:0.101 +INFO 2025-05-18 14:57:20 ts/train.py:232 step:800 smpl:13K ep:61 epch:0.20 loss:0.068 grdn:1.301 lr:1.0e-04 updt_s:0.222 data_s:0.101 +INFO 2025-05-18 14:58:24 ts/train.py:232 step:1K smpl:16K ep:76 epch:0.25 loss:0.062 grdn:1.191 lr:1.0e-04 updt_s:0.222 data_s:0.096 +INFO 2025-05-18 14:59:27 ts/train.py:232 step:1K smpl:19K ep:91 epch:0.30 loss:0.052 grdn:1.005 lr:1.0e-04 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 15:00:20 ts/train.py:232 step:1K smpl:22K ep:106 epch:0.35 loss:0.046 grdn:0.898 lr:1.0e-04 updt_s:0.222 data_s:0.038 +INFO 2025-05-18 15:01:11 ts/train.py:232 step:2K smpl:26K ep:121 epch:0.40 loss:0.039 grdn:0.805 lr:1.0e-04 updt_s:0.222 data_s:0.034 +INFO 2025-05-18 15:02:02 ts/train.py:232 step:2K smpl:29K ep:136 epch:0.45 loss:0.040 grdn:0.782 lr:1.0e-04 updt_s:0.223 data_s:0.030 +INFO 2025-05-18 15:02:52 ts/train.py:232 step:2K smpl:32K ep:151 epch:0.50 loss:0.037 grdn:0.763 lr:1.0e-04 updt_s:0.222 data_s:0.028 +INFO 2025-05-18 15:03:44 ts/train.py:232 step:2K smpl:35K ep:166 epch:0.55 loss:0.036 grdn:0.733 lr:1.0e-04 updt_s:0.222 data_s:0.036 +INFO 2025-05-18 15:04:34 ts/train.py:232 step:2K smpl:38K ep:182 epch:0.61 loss:0.036 grdn:0.689 lr:1.0e-04 updt_s:0.223 data_s:0.027 +INFO 2025-05-18 15:05:24 ts/train.py:232 step:3K smpl:42K ep:197 epch:0.66 loss:0.033 grdn:0.622 lr:1.0e-04 updt_s:0.222 data_s:0.029 +INFO 2025-05-18 15:06:14 ts/train.py:232 step:3K smpl:45K ep:212 epch:0.71 loss:0.030 grdn:0.582 lr:1.0e-04 updt_s:0.222 data_s:0.026 +INFO 2025-05-18 15:07:06 ts/train.py:232 step:3K smpl:48K ep:227 epch:0.76 loss:0.033 grdn:0.627 lr:1.0e-04 updt_s:0.222 data_s:0.034 +INFO 2025-05-18 15:07:57 ts/train.py:232 step:3K smpl:51K ep:242 epch:0.81 loss:0.030 grdn:0.571 lr:1.0e-04 updt_s:0.222 data_s:0.032 +INFO 2025-05-18 15:08:46 ts/train.py:232 step:3K smpl:54K ep:257 epch:0.86 loss:0.030 grdn:0.551 lr:1.0e-04 updt_s:0.222 data_s:0.026 +INFO 2025-05-18 15:09:38 ts/train.py:232 step:4K smpl:58K ep:272 epch:0.91 loss:0.030 grdn:0.559 lr:1.0e-04 updt_s:0.222 data_s:0.034 +INFO 2025-05-18 15:10:29 ts/train.py:232 step:4K smpl:61K ep:288 epch:0.96 loss:0.029 grdn:0.538 lr:1.0e-04 updt_s:0.222 data_s:0.032 +INFO 2025-05-18 15:11:35 ts/train.py:232 step:4K smpl:64K ep:303 epch:1.01 loss:0.029 grdn:0.545 lr:1.0e-04 updt_s:0.230 data_s:0.099 +INFO 2025-05-18 15:12:39 ts/train.py:232 step:4K smpl:67K ep:318 epch:1.06 loss:0.027 grdn:0.492 lr:1.0e-04 updt_s:0.222 data_s:0.101 +INFO 2025-05-18 15:13:43 ts/train.py:232 step:4K smpl:70K ep:333 epch:1.11 loss:0.026 grdn:0.466 lr:1.0e-04 updt_s:0.222 data_s:0.096 +INFO 2025-05-18 15:14:47 ts/train.py:232 step:5K smpl:74K ep:348 epch:1.16 loss:0.028 grdn:0.472 lr:1.0e-04 updt_s:0.222 data_s:0.096 +INFO 2025-05-18 15:15:51 ts/train.py:232 step:5K smpl:77K ep:363 epch:1.21 loss:0.027 grdn:0.468 lr:1.0e-04 updt_s:0.222 data_s:0.099 +INFO 2025-05-18 15:16:55 ts/train.py:232 step:5K smpl:80K ep:378 epch:1.26 loss:0.027 grdn:0.451 lr:1.0e-04 updt_s:0.222 data_s:0.101 +INFO 2025-05-18 15:17:50 ts/train.py:232 step:5K smpl:83K ep:393 epch:1.31 loss:0.026 grdn:0.440 lr:1.0e-04 updt_s:0.222 data_s:0.048 +INFO 2025-05-18 15:18:39 ts/train.py:232 step:5K smpl:86K ep:409 epch:1.36 loss:0.026 grdn:0.437 lr:1.0e-04 updt_s:0.223 data_s:0.022 +INFO 2025-05-18 15:19:29 ts/train.py:232 step:6K smpl:90K ep:424 epch:1.41 loss:0.026 grdn:0.431 lr:1.0e-04 updt_s:0.223 data_s:0.025 +INFO 2025-05-18 15:20:18 ts/train.py:232 step:6K smpl:93K ep:439 epch:1.46 loss:0.025 grdn:0.430 lr:1.0e-04 updt_s:0.222 data_s:0.025 +INFO 2025-05-18 15:21:08 ts/train.py:232 step:6K smpl:96K ep:454 epch:1.51 loss:0.024 grdn:0.401 lr:1.0e-04 updt_s:0.222 data_s:0.024 +INFO 2025-05-18 15:21:57 ts/train.py:232 step:6K smpl:99K ep:469 epch:1.56 loss:0.023 grdn:0.392 lr:1.0e-04 updt_s:0.223 data_s:0.024 +INFO 2025-05-18 15:22:47 ts/train.py:232 step:6K smpl:102K ep:484 epch:1.61 loss:0.026 grdn:0.420 lr:1.0e-04 updt_s:0.222 data_s:0.027 +INFO 2025-05-18 15:23:36 ts/train.py:232 step:7K smpl:106K ep:499 epch:1.66 loss:0.024 grdn:0.403 lr:1.0e-04 updt_s:0.222 data_s:0.024 +INFO 2025-05-18 15:24:26 ts/train.py:232 step:7K smpl:109K ep:515 epch:1.72 loss:0.025 grdn:0.382 lr:1.0e-04 updt_s:0.223 data_s:0.025 +INFO 2025-05-18 15:25:16 ts/train.py:232 step:7K smpl:112K ep:530 epch:1.77 loss:0.024 grdn:0.378 lr:1.0e-04 updt_s:0.223 data_s:0.025 +INFO 2025-05-18 15:26:06 ts/train.py:232 step:7K smpl:115K ep:545 epch:1.82 loss:0.026 grdn:0.411 lr:1.0e-04 updt_s:0.222 data_s:0.028 +INFO 2025-05-18 15:26:55 ts/train.py:232 step:7K smpl:118K ep:560 epch:1.87 loss:0.024 grdn:0.374 lr:1.0e-04 updt_s:0.223 data_s:0.025 +INFO 2025-05-18 15:27:45 ts/train.py:232 step:8K smpl:122K ep:575 epch:1.92 loss:0.025 grdn:0.383 lr:1.0e-04 updt_s:0.222 data_s:0.026 +INFO 2025-05-18 15:28:47 ts/train.py:232 step:8K smpl:125K ep:590 epch:1.97 loss:0.021 grdn:0.346 lr:1.0e-04 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 15:29:53 ts/train.py:232 step:8K smpl:128K ep:605 epch:2.02 loss:0.024 grdn:0.358 lr:1.0e-04 updt_s:0.222 data_s:0.105 +INFO 2025-05-18 15:30:58 ts/train.py:232 step:8K smpl:131K ep:620 epch:2.07 loss:0.023 grdn:0.340 lr:1.0e-04 updt_s:0.222 data_s:0.104 +INFO 2025-05-18 15:32:04 ts/train.py:232 step:8K smpl:134K ep:636 epch:2.12 loss:0.022 grdn:0.336 lr:1.0e-04 updt_s:0.222 data_s:0.110 +INFO 2025-05-18 15:33:12 ts/train.py:232 step:9K smpl:138K ep:651 epch:2.17 loss:0.024 grdn:0.362 lr:1.0e-04 updt_s:0.222 data_s:0.114 +INFO 2025-05-18 15:34:18 ts/train.py:232 step:9K smpl:141K ep:666 epch:2.22 loss:0.025 grdn:0.372 lr:1.0e-04 updt_s:0.222 data_s:0.108 +INFO 2025-05-18 15:35:17 ts/train.py:232 step:9K smpl:144K ep:681 epch:2.27 loss:0.022 grdn:0.346 lr:1.0e-04 updt_s:0.222 data_s:0.073 +INFO 2025-05-18 15:36:12 ts/train.py:232 step:9K smpl:147K ep:696 epch:2.32 loss:0.021 grdn:0.317 lr:1.0e-04 updt_s:0.222 data_s:0.050 +INFO 2025-05-18 15:37:05 ts/train.py:232 step:9K smpl:150K ep:711 epch:2.37 loss:0.023 grdn:0.332 lr:1.0e-04 updt_s:0.222 data_s:0.042 +INFO 2025-05-18 15:37:57 ts/train.py:232 step:10K smpl:154K ep:726 epch:2.42 loss:0.021 grdn:0.314 lr:1.0e-04 updt_s:0.222 data_s:0.042 +INFO 2025-05-18 15:38:51 ts/train.py:232 step:10K smpl:157K ep:742 epch:2.47 loss:0.023 grdn:0.346 lr:1.0e-04 updt_s:0.222 data_s:0.044 +INFO 2025-05-18 15:39:46 ts/train.py:232 step:10K smpl:160K ep:757 epch:2.52 loss:0.023 grdn:0.330 lr:1.0e-04 updt_s:0.222 data_s:0.055 +INFO 2025-05-18 15:40:40 ts/train.py:232 step:10K smpl:163K ep:772 epch:2.57 loss:0.022 grdn:0.316 lr:1.0e-04 updt_s:0.222 data_s:0.044 +INFO 2025-05-18 15:41:32 ts/train.py:232 step:10K smpl:166K ep:787 epch:2.62 loss:0.021 grdn:0.305 lr:1.0e-04 updt_s:0.222 data_s:0.037 +INFO 2025-05-18 15:42:25 ts/train.py:232 step:11K smpl:170K ep:802 epch:2.67 loss:0.020 grdn:0.310 lr:1.0e-04 updt_s:0.223 data_s:0.041 +INFO 2025-05-18 15:43:17 ts/train.py:232 step:11K smpl:173K ep:817 epch:2.72 loss:0.020 grdn:0.294 lr:1.0e-04 updt_s:0.222 data_s:0.039 +INFO 2025-05-18 15:44:10 ts/train.py:232 step:11K smpl:176K ep:832 epch:2.77 loss:0.023 grdn:0.316 lr:1.0e-04 updt_s:0.222 data_s:0.043 +INFO 2025-05-18 15:45:02 ts/train.py:232 step:11K smpl:179K ep:847 epch:2.82 loss:0.021 grdn:0.301 lr:1.0e-04 updt_s:0.222 data_s:0.037 +INFO 2025-05-18 15:45:55 ts/train.py:232 step:11K smpl:182K ep:863 epch:2.88 loss:0.022 grdn:0.313 lr:1.0e-04 updt_s:0.222 data_s:0.040 +INFO 2025-05-18 15:46:55 ts/train.py:232 step:12K smpl:186K ep:878 epch:2.93 loss:0.021 grdn:0.300 lr:1.0e-04 updt_s:0.221 data_s:0.081 +INFO 2025-05-18 15:48:02 ts/train.py:232 step:12K smpl:189K ep:893 epch:2.98 loss:0.023 grdn:0.327 lr:1.0e-04 updt_s:0.222 data_s:0.112 +INFO 2025-05-18 15:49:10 ts/train.py:232 step:12K smpl:192K ep:908 epch:3.03 loss:0.021 grdn:0.308 lr:1.0e-04 updt_s:0.222 data_s:0.114 +INFO 2025-05-18 15:50:16 ts/train.py:232 step:12K smpl:195K ep:923 epch:3.08 loss:0.021 grdn:0.300 lr:1.0e-04 updt_s:0.222 data_s:0.108 +INFO 2025-05-18 15:51:21 ts/train.py:232 step:12K smpl:198K ep:938 epch:3.13 loss:0.020 grdn:0.284 lr:1.0e-04 updt_s:0.222 data_s:0.105 +INFO 2025-05-18 15:52:27 ts/train.py:232 step:13K smpl:202K ep:953 epch:3.18 loss:0.020 grdn:0.289 lr:1.0e-04 updt_s:0.222 data_s:0.106 +INFO 2025-05-18 15:53:27 ts/train.py:232 step:13K smpl:205K ep:969 epch:3.23 loss:0.019 grdn:0.280 lr:1.0e-04 updt_s:0.222 data_s:0.077 +INFO 2025-05-18 15:54:16 ts/train.py:232 step:13K smpl:208K ep:984 epch:3.28 loss:0.019 grdn:0.265 lr:1.0e-04 updt_s:0.222 data_s:0.026 +INFO 2025-05-18 15:55:06 ts/train.py:232 step:13K smpl:211K ep:999 epch:3.33 loss:0.021 grdn:0.308 lr:1.0e-04 updt_s:0.222 data_s:0.024 +INFO 2025-05-18 15:55:56 ts/train.py:232 step:13K smpl:214K ep:1K epch:3.38 loss:0.020 grdn:0.291 lr:1.0e-04 updt_s:0.222 data_s:0.027 +INFO 2025-05-18 15:56:47 ts/train.py:232 step:14K smpl:218K ep:1K epch:3.43 loss:0.020 grdn:0.279 lr:1.0e-04 updt_s:0.222 data_s:0.030 +INFO 2025-05-18 15:57:37 ts/train.py:232 step:14K smpl:221K ep:1K epch:3.48 loss:0.020 grdn:0.295 lr:1.0e-04 updt_s:0.222 data_s:0.027 +INFO 2025-05-18 15:58:27 ts/train.py:232 step:14K smpl:224K ep:1K epch:3.53 loss:0.021 grdn:0.291 lr:1.0e-04 updt_s:0.222 data_s:0.031 +INFO 2025-05-18 15:59:18 ts/train.py:232 step:14K smpl:227K ep:1K epch:3.58 loss:0.020 grdn:0.281 lr:1.0e-04 updt_s:0.222 data_s:0.031 +INFO 2025-05-18 16:00:09 ts/train.py:232 step:14K smpl:230K ep:1K epch:3.63 loss:0.019 grdn:0.273 lr:1.0e-04 updt_s:0.222 data_s:0.033 +INFO 2025-05-18 16:01:00 ts/train.py:232 step:15K smpl:234K ep:1K epch:3.68 loss:0.019 grdn:0.277 lr:1.0e-04 updt_s:0.222 data_s:0.030 +INFO 2025-05-18 16:01:51 ts/train.py:232 step:15K smpl:237K ep:1K epch:3.73 loss:0.018 grdn:0.263 lr:1.0e-04 updt_s:0.222 data_s:0.030 +INFO 2025-05-18 16:02:42 ts/train.py:232 step:15K smpl:240K ep:1K epch:3.78 loss:0.019 grdn:0.268 lr:1.0e-04 updt_s:0.222 data_s:0.032 +INFO 2025-05-18 16:03:33 ts/train.py:232 step:15K smpl:243K ep:1K epch:3.83 loss:0.019 grdn:0.268 lr:1.0e-04 updt_s:0.222 data_s:0.033 +INFO 2025-05-18 16:04:31 ts/train.py:232 step:15K smpl:246K ep:1K epch:3.88 loss:0.019 grdn:0.273 lr:1.0e-04 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 16:05:37 ts/train.py:232 step:16K smpl:250K ep:1K epch:3.93 loss:0.020 grdn:0.282 lr:1.0e-04 updt_s:0.222 data_s:0.106 +INFO 2025-05-18 16:06:42 ts/train.py:232 step:16K smpl:253K ep:1K epch:3.99 loss:0.018 grdn:0.255 lr:1.0e-04 updt_s:0.222 data_s:0.101 +INFO 2025-05-18 16:07:46 ts/train.py:232 step:16K smpl:256K ep:1K epch:4.04 loss:0.020 grdn:0.279 lr:1.0e-04 updt_s:0.222 data_s:0.099 +INFO 2025-05-18 16:08:52 ts/train.py:232 step:16K smpl:259K ep:1K epch:4.09 loss:0.019 grdn:0.266 lr:1.0e-04 updt_s:0.222 data_s:0.110 +INFO 2025-05-18 16:09:59 ts/train.py:232 step:16K smpl:262K ep:1K epch:4.14 loss:0.019 grdn:0.260 lr:1.0e-04 updt_s:0.222 data_s:0.109 +INFO 2025-05-18 16:11:03 ts/train.py:232 step:17K smpl:266K ep:1K epch:4.19 loss:0.019 grdn:0.264 lr:1.0e-04 updt_s:0.222 data_s:0.098 +INFO 2025-05-18 16:11:54 ts/train.py:232 step:17K smpl:269K ep:1K epch:4.24 loss:0.019 grdn:0.262 lr:1.0e-04 updt_s:0.223 data_s:0.034 +INFO 2025-05-18 16:12:46 ts/train.py:232 step:17K smpl:272K ep:1K epch:4.29 loss:0.017 grdn:0.251 lr:1.0e-04 updt_s:0.223 data_s:0.036 +INFO 2025-05-18 16:13:38 ts/train.py:232 step:17K smpl:275K ep:1K epch:4.34 loss:0.018 grdn:0.258 lr:1.0e-04 updt_s:0.223 data_s:0.036 +INFO 2025-05-18 16:14:30 ts/train.py:232 step:17K smpl:278K ep:1K epch:4.39 loss:0.018 grdn:0.260 lr:1.0e-04 updt_s:0.222 data_s:0.037 +INFO 2025-05-18 16:15:22 ts/train.py:232 step:18K smpl:282K ep:1K epch:4.44 loss:0.018 grdn:0.253 lr:1.0e-04 updt_s:0.222 data_s:0.036 +INFO 2025-05-18 16:16:14 ts/train.py:232 step:18K smpl:285K ep:1K epch:4.49 loss:0.017 grdn:0.242 lr:1.0e-04 updt_s:0.222 data_s:0.036 +INFO 2025-05-18 16:17:06 ts/train.py:232 step:18K smpl:288K ep:1K epch:4.54 loss:0.019 grdn:0.269 lr:1.0e-04 updt_s:0.222 data_s:0.036 +INFO 2025-05-18 16:17:57 ts/train.py:232 step:18K smpl:291K ep:1K epch:4.59 loss:0.018 grdn:0.254 lr:1.0e-04 updt_s:0.223 data_s:0.033 +INFO 2025-05-18 16:18:49 ts/train.py:232 step:18K smpl:294K ep:1K epch:4.64 loss:0.018 grdn:0.254 lr:1.0e-04 updt_s:0.222 data_s:0.035 +INFO 2025-05-18 16:19:40 ts/train.py:232 step:19K smpl:298K ep:1K epch:4.69 loss:0.018 grdn:0.261 lr:1.0e-04 updt_s:0.223 data_s:0.032 +INFO 2025-05-18 16:20:31 ts/train.py:232 step:19K smpl:301K ep:1K epch:4.74 loss:0.018 grdn:0.256 lr:1.0e-04 updt_s:0.223 data_s:0.032 +INFO 2025-05-18 16:21:23 ts/train.py:232 step:19K smpl:304K ep:1K epch:4.79 loss:0.019 grdn:0.254 lr:1.0e-04 updt_s:0.222 data_s:0.036 +INFO 2025-05-18 16:22:18 ts/train.py:232 step:19K smpl:307K ep:1K epch:4.84 loss:0.017 grdn:0.250 lr:1.0e-04 updt_s:0.222 data_s:0.056 +INFO 2025-05-18 16:23:23 ts/train.py:232 step:19K smpl:310K ep:1K epch:4.89 loss:0.017 grdn:0.247 lr:1.0e-04 updt_s:0.222 data_s:0.103 +INFO 2025-05-18 16:24:29 ts/train.py:232 step:20K smpl:314K ep:1K epch:4.94 loss:0.016 grdn:0.232 lr:1.0e-04 updt_s:0.221 data_s:0.104 +INFO 2025-05-18 16:25:34 ts/train.py:232 step:20K smpl:317K ep:1K epch:4.99 loss:0.016 grdn:0.250 lr:1.0e-04 updt_s:0.222 data_s:0.103 +INFO 2025-05-18 16:26:40 ts/train.py:232 step:20K smpl:320K ep:2K epch:5.04 loss:0.017 grdn:0.252 lr:1.0e-04 updt_s:0.222 data_s:0.108 +INFO 2025-05-18 16:26:40 ts/train.py:241 Checkpoint policy after step 20000 +INFO 2025-05-18 16:27:48 ts/train.py:232 step:20K smpl:323K ep:2K epch:5.09 loss:0.018 grdn:0.256 lr:1.0e-04 updt_s:0.221 data_s:0.095 +INFO 2025-05-18 16:28:53 ts/train.py:232 step:20K smpl:326K ep:2K epch:5.15 loss:0.018 grdn:0.261 lr:1.0e-04 updt_s:0.222 data_s:0.101 +INFO 2025-05-18 16:29:50 ts/train.py:232 step:21K smpl:330K ep:2K epch:5.20 loss:0.017 grdn:0.241 lr:1.0e-04 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 16:30:49 ts/train.py:232 step:21K smpl:333K ep:2K epch:5.25 loss:0.016 grdn:0.230 lr:1.0e-04 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 16:31:48 ts/train.py:232 step:21K smpl:336K ep:2K epch:5.30 loss:0.018 grdn:0.251 lr:1.0e-04 updt_s:0.222 data_s:0.072 +INFO 2025-05-18 16:32:47 ts/train.py:232 step:21K smpl:339K ep:2K epch:5.35 loss:0.017 grdn:0.243 lr:1.0e-04 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 16:33:45 ts/train.py:232 step:21K smpl:342K ep:2K epch:5.40 loss:0.017 grdn:0.240 lr:1.0e-04 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 16:34:46 ts/train.py:232 step:22K smpl:346K ep:2K epch:5.45 loss:0.017 grdn:0.242 lr:1.0e-04 updt_s:0.222 data_s:0.078 +INFO 2025-05-18 16:35:46 ts/train.py:232 step:22K smpl:349K ep:2K epch:5.50 loss:0.017 grdn:0.244 lr:1.0e-04 updt_s:0.222 data_s:0.079 +INFO 2025-05-18 16:36:45 ts/train.py:232 step:22K smpl:352K ep:2K epch:5.55 loss:0.017 grdn:0.246 lr:1.0e-04 updt_s:0.222 data_s:0.072 +INFO 2025-05-18 16:37:44 ts/train.py:232 step:22K smpl:355K ep:2K epch:5.60 loss:0.017 grdn:0.247 lr:1.0e-04 updt_s:0.222 data_s:0.072 +INFO 2025-05-18 16:38:43 ts/train.py:232 step:22K smpl:358K ep:2K epch:5.65 loss:0.017 grdn:0.241 lr:1.0e-04 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 16:39:42 ts/train.py:232 step:23K smpl:362K ep:2K epch:5.70 loss:0.016 grdn:0.233 lr:1.0e-04 updt_s:0.222 data_s:0.074 +INFO 2025-05-18 16:40:41 ts/train.py:232 step:23K smpl:365K ep:2K epch:5.75 loss:0.017 grdn:0.244 lr:1.0e-04 updt_s:0.222 data_s:0.074 +INFO 2025-05-18 16:41:41 ts/train.py:232 step:23K smpl:368K ep:2K epch:5.80 loss:0.016 grdn:0.231 lr:1.0e-04 updt_s:0.222 data_s:0.076 +INFO 2025-05-18 16:42:44 ts/train.py:232 step:23K smpl:371K ep:2K epch:5.85 loss:0.016 grdn:0.235 lr:1.0e-04 updt_s:0.221 data_s:0.096 +INFO 2025-05-18 16:43:47 ts/train.py:232 step:23K smpl:374K ep:2K epch:5.90 loss:0.016 grdn:0.247 lr:1.0e-04 updt_s:0.223 data_s:0.089 +INFO 2025-05-18 16:44:50 ts/train.py:232 step:24K smpl:378K ep:2K epch:5.95 loss:0.016 grdn:0.241 lr:1.0e-04 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 16:45:53 ts/train.py:232 step:24K smpl:381K ep:2K epch:6.00 loss:0.016 grdn:0.223 lr:1.0e-04 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 16:46:58 ts/train.py:232 step:24K smpl:384K ep:2K epch:6.05 loss:0.017 grdn:0.245 lr:1.0e-04 updt_s:0.222 data_s:0.098 +INFO 2025-05-18 16:48:01 ts/train.py:232 step:24K smpl:387K ep:2K epch:6.10 loss:0.015 grdn:0.223 lr:1.0e-04 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 16:49:03 ts/train.py:232 step:24K smpl:390K ep:2K epch:6.15 loss:0.015 grdn:0.243 lr:1.0e-04 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 16:50:07 ts/train.py:232 step:25K smpl:394K ep:2K epch:6.20 loss:0.016 grdn:0.226 lr:1.0e-04 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 16:51:11 ts/train.py:232 step:25K smpl:397K ep:2K epch:6.26 loss:0.015 grdn:0.232 lr:1.0e-04 updt_s:0.222 data_s:0.098 +INFO 2025-05-18 16:52:14 ts/train.py:232 step:25K smpl:400K ep:2K epch:6.31 loss:0.016 grdn:0.228 lr:1.0e-04 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 16:53:17 ts/train.py:232 step:25K smpl:403K ep:2K epch:6.36 loss:0.017 grdn:0.235 lr:1.0e-04 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 16:54:19 ts/train.py:232 step:25K smpl:406K ep:2K epch:6.41 loss:0.016 grdn:0.232 lr:1.0e-04 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 16:55:22 ts/train.py:232 step:26K smpl:410K ep:2K epch:6.46 loss:0.016 grdn:0.251 lr:1.0e-04 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 16:56:25 ts/train.py:232 step:26K smpl:413K ep:2K epch:6.51 loss:0.016 grdn:0.224 lr:1.0e-04 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 16:57:28 ts/train.py:232 step:26K smpl:416K ep:2K epch:6.56 loss:0.015 grdn:0.215 lr:1.0e-04 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 16:58:30 ts/train.py:232 step:26K smpl:419K ep:2K epch:6.61 loss:0.014 grdn:0.209 lr:1.0e-04 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 16:59:32 ts/train.py:232 step:26K smpl:422K ep:2K epch:6.66 loss:0.015 grdn:0.231 lr:1.0e-04 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 17:00:35 ts/train.py:232 step:27K smpl:426K ep:2K epch:6.71 loss:0.015 grdn:0.225 lr:1.0e-04 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 17:01:38 ts/train.py:232 step:27K smpl:429K ep:2K epch:6.76 loss:0.015 grdn:0.223 lr:1.0e-04 updt_s:0.222 data_s:0.096 +INFO 2025-05-18 17:02:43 ts/train.py:232 step:27K smpl:432K ep:2K epch:6.81 loss:0.014 grdn:0.210 lr:1.0e-04 updt_s:0.221 data_s:0.099 +INFO 2025-05-18 17:03:43 ts/train.py:232 step:27K smpl:435K ep:2K epch:6.86 loss:0.016 grdn:0.239 lr:1.0e-04 updt_s:0.222 data_s:0.079 +INFO 2025-05-18 17:04:41 ts/train.py:232 step:27K smpl:438K ep:2K epch:6.91 loss:0.015 grdn:0.216 lr:1.0e-04 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 17:05:41 ts/train.py:232 step:28K smpl:442K ep:2K epch:6.96 loss:0.016 grdn:0.234 lr:1.0e-04 updt_s:0.222 data_s:0.076 +INFO 2025-05-18 17:06:40 ts/train.py:232 step:28K smpl:445K ep:2K epch:7.01 loss:0.016 grdn:0.229 lr:9.9e-05 updt_s:0.222 data_s:0.073 +INFO 2025-05-18 17:07:39 ts/train.py:232 step:28K smpl:448K ep:2K epch:7.06 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-18 17:08:37 ts/train.py:232 step:28K smpl:451K ep:2K epch:7.11 loss:0.015 grdn:0.215 lr:9.9e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 17:09:34 ts/train.py:232 step:28K smpl:454K ep:2K epch:7.16 loss:0.016 grdn:0.225 lr:9.9e-05 updt_s:0.222 data_s:0.060 +INFO 2025-05-18 17:10:30 ts/train.py:232 step:29K smpl:458K ep:2K epch:7.21 loss:0.015 grdn:0.219 lr:9.9e-05 updt_s:0.222 data_s:0.059 +INFO 2025-05-18 17:11:26 ts/train.py:232 step:29K smpl:461K ep:2K epch:7.26 loss:0.015 grdn:0.230 lr:9.9e-05 updt_s:0.222 data_s:0.055 +INFO 2025-05-18 17:12:21 ts/train.py:232 step:29K smpl:464K ep:2K epch:7.31 loss:0.016 grdn:0.238 lr:9.9e-05 updt_s:0.222 data_s:0.054 +INFO 2025-05-18 17:13:17 ts/train.py:232 step:29K smpl:467K ep:2K epch:7.37 loss:0.015 grdn:0.225 lr:9.9e-05 updt_s:0.222 data_s:0.056 +INFO 2025-05-18 17:14:14 ts/train.py:232 step:29K smpl:470K ep:2K epch:7.42 loss:0.016 grdn:0.229 lr:9.9e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-18 17:15:11 ts/train.py:232 step:30K smpl:474K ep:2K epch:7.47 loss:0.016 grdn:0.224 lr:9.9e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 17:16:09 ts/train.py:232 step:30K smpl:477K ep:2K epch:7.52 loss:0.014 grdn:0.215 lr:9.9e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 17:17:07 ts/train.py:232 step:30K smpl:480K ep:2K epch:7.57 loss:0.015 grdn:0.227 lr:9.9e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 17:18:03 ts/train.py:232 step:30K smpl:483K ep:2K epch:7.62 loss:0.015 grdn:0.227 lr:9.9e-05 updt_s:0.222 data_s:0.059 +INFO 2025-05-18 17:19:00 ts/train.py:232 step:30K smpl:486K ep:2K epch:7.67 loss:0.015 grdn:0.219 lr:9.9e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-18 17:19:57 ts/train.py:232 step:31K smpl:490K ep:2K epch:7.72 loss:0.016 grdn:0.236 lr:9.9e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 17:20:58 ts/train.py:232 step:31K smpl:493K ep:2K epch:7.77 loss:0.015 grdn:0.217 lr:9.9e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-18 17:22:01 ts/train.py:232 step:31K smpl:496K ep:2K epch:7.82 loss:0.014 grdn:0.215 lr:9.9e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 17:23:03 ts/train.py:232 step:31K smpl:499K ep:2K epch:7.87 loss:0.017 grdn:0.229 lr:9.9e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 17:24:05 ts/train.py:232 step:31K smpl:502K ep:2K epch:7.92 loss:0.014 grdn:0.210 lr:9.9e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 17:25:09 ts/train.py:232 step:32K smpl:506K ep:2K epch:7.97 loss:0.014 grdn:0.223 lr:9.9e-05 updt_s:0.222 data_s:0.097 +INFO 2025-05-18 17:26:13 ts/train.py:232 step:32K smpl:509K ep:2K epch:8.02 loss:0.015 grdn:0.218 lr:9.9e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 17:27:12 ts/train.py:232 step:32K smpl:512K ep:2K epch:8.07 loss:0.015 grdn:0.220 lr:9.9e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-18 17:28:09 ts/train.py:232 step:32K smpl:515K ep:2K epch:8.12 loss:0.014 grdn:0.223 lr:9.9e-05 updt_s:0.223 data_s:0.061 +INFO 2025-05-18 17:29:06 ts/train.py:232 step:32K smpl:518K ep:2K epch:8.17 loss:0.015 grdn:0.217 lr:9.9e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-18 17:30:03 ts/train.py:232 step:33K smpl:522K ep:2K epch:8.22 loss:0.015 grdn:0.216 lr:9.9e-05 updt_s:0.222 data_s:0.066 +INFO 2025-05-18 17:30:59 ts/train.py:232 step:33K smpl:525K ep:2K epch:8.27 loss:0.013 grdn:0.216 lr:9.9e-05 updt_s:0.222 data_s:0.057 +INFO 2025-05-18 17:31:55 ts/train.py:232 step:33K smpl:528K ep:2K epch:8.32 loss:0.014 grdn:0.236 lr:9.9e-05 updt_s:0.222 data_s:0.056 +INFO 2025-05-18 17:32:53 ts/train.py:232 step:33K smpl:531K ep:3K epch:8.37 loss:0.015 grdn:0.224 lr:9.9e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 17:33:49 ts/train.py:232 step:33K smpl:534K ep:3K epch:8.42 loss:0.014 grdn:0.215 lr:9.9e-05 updt_s:0.222 data_s:0.058 +INFO 2025-05-18 17:34:46 ts/train.py:232 step:34K smpl:538K ep:3K epch:8.47 loss:0.014 grdn:0.219 lr:9.9e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 17:35:43 ts/train.py:232 step:34K smpl:541K ep:3K epch:8.53 loss:0.014 grdn:0.207 lr:9.9e-05 updt_s:0.222 data_s:0.060 +INFO 2025-05-18 17:36:39 ts/train.py:232 step:34K smpl:544K ep:3K epch:8.58 loss:0.014 grdn:0.220 lr:9.9e-05 updt_s:0.222 data_s:0.057 +INFO 2025-05-18 17:37:35 ts/train.py:232 step:34K smpl:547K ep:3K epch:8.63 loss:0.015 grdn:0.224 lr:9.9e-05 updt_s:0.222 data_s:0.059 +INFO 2025-05-18 17:38:32 ts/train.py:232 step:34K smpl:550K ep:3K epch:8.68 loss:0.014 grdn:0.214 lr:9.9e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-18 17:39:34 ts/train.py:232 step:35K smpl:554K ep:3K epch:8.73 loss:0.014 grdn:0.227 lr:9.9e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 17:40:38 ts/train.py:232 step:35K smpl:557K ep:3K epch:8.78 loss:0.015 grdn:0.225 lr:9.9e-05 updt_s:0.222 data_s:0.097 +INFO 2025-05-18 17:41:42 ts/train.py:232 step:35K smpl:560K ep:3K epch:8.83 loss:0.013 grdn:0.204 lr:9.9e-05 updt_s:0.222 data_s:0.098 +INFO 2025-05-18 17:42:46 ts/train.py:232 step:35K smpl:563K ep:3K epch:8.88 loss:0.014 grdn:0.209 lr:9.9e-05 updt_s:0.222 data_s:0.097 +INFO 2025-05-18 17:43:49 ts/train.py:232 step:35K smpl:566K ep:3K epch:8.93 loss:0.014 grdn:0.219 lr:9.9e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 17:44:51 ts/train.py:232 step:36K smpl:570K ep:3K epch:8.98 loss:0.014 grdn:0.221 lr:9.9e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 17:45:50 ts/train.py:232 step:36K smpl:573K ep:3K epch:9.03 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.222 data_s:0.076 +INFO 2025-05-18 17:46:48 ts/train.py:232 step:36K smpl:576K ep:3K epch:9.08 loss:0.014 grdn:0.221 lr:9.9e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 17:47:45 ts/train.py:232 step:36K smpl:579K ep:3K epch:9.13 loss:0.014 grdn:0.210 lr:9.9e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-18 17:48:41 ts/train.py:232 step:36K smpl:582K ep:3K epch:9.18 loss:0.014 grdn:0.201 lr:9.9e-05 updt_s:0.222 data_s:0.060 +INFO 2025-05-18 17:49:38 ts/train.py:232 step:37K smpl:586K ep:3K epch:9.23 loss:0.014 grdn:0.211 lr:9.9e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 17:50:35 ts/train.py:232 step:37K smpl:589K ep:3K epch:9.28 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.222 data_s:0.060 +INFO 2025-05-18 17:51:32 ts/train.py:232 step:37K smpl:592K ep:3K epch:9.33 loss:0.014 grdn:0.214 lr:9.9e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 17:52:30 ts/train.py:232 step:37K smpl:595K ep:3K epch:9.38 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 17:53:28 ts/train.py:232 step:37K smpl:598K ep:3K epch:9.43 loss:0.014 grdn:0.215 lr:9.9e-05 updt_s:0.222 data_s:0.066 +INFO 2025-05-18 17:54:26 ts/train.py:232 step:38K smpl:602K ep:3K epch:9.48 loss:0.013 grdn:0.202 lr:9.9e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 17:55:23 ts/train.py:232 step:38K smpl:605K ep:3K epch:9.53 loss:0.013 grdn:0.208 lr:9.9e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 17:56:21 ts/train.py:232 step:38K smpl:608K ep:3K epch:9.58 loss:0.014 grdn:0.222 lr:9.9e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 17:57:18 ts/train.py:232 step:38K smpl:611K ep:3K epch:9.64 loss:0.014 grdn:0.213 lr:9.9e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 17:58:19 ts/train.py:232 step:38K smpl:614K ep:3K epch:9.69 loss:0.013 grdn:0.196 lr:9.9e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-18 17:59:21 ts/train.py:232 step:39K smpl:618K ep:3K epch:9.74 loss:0.013 grdn:0.200 lr:9.9e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 18:00:24 ts/train.py:232 step:39K smpl:621K ep:3K epch:9.79 loss:0.013 grdn:0.208 lr:9.9e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 18:01:26 ts/train.py:232 step:39K smpl:624K ep:3K epch:9.84 loss:0.014 grdn:0.203 lr:9.9e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 18:02:28 ts/train.py:232 step:39K smpl:627K ep:3K epch:9.89 loss:0.014 grdn:0.206 lr:9.9e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 18:03:30 ts/train.py:232 step:39K smpl:630K ep:3K epch:9.94 loss:0.014 grdn:0.224 lr:9.9e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 18:04:32 ts/train.py:232 step:40K smpl:634K ep:3K epch:9.99 loss:0.013 grdn:0.214 lr:9.9e-05 updt_s:0.222 data_s:0.085 +INFO 2025-05-18 18:05:29 ts/train.py:232 step:40K smpl:637K ep:3K epch:10.04 loss:0.013 grdn:0.219 lr:9.9e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 18:06:28 ts/train.py:232 step:40K smpl:640K ep:3K epch:10.09 loss:0.014 grdn:0.219 lr:9.9e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 18:06:28 ts/train.py:241 Checkpoint policy after step 40000 +INFO 2025-05-18 18:07:29 ts/train.py:232 step:40K smpl:643K ep:3K epch:10.14 loss:0.014 grdn:0.218 lr:9.9e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-18 18:08:27 ts/train.py:232 step:40K smpl:646K ep:3K epch:10.19 loss:0.014 grdn:0.216 lr:9.9e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 18:09:25 ts/train.py:232 step:41K smpl:650K ep:3K epch:10.24 loss:0.013 grdn:0.207 lr:9.9e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 18:10:24 ts/train.py:232 step:41K smpl:653K ep:3K epch:10.29 loss:0.013 grdn:0.200 lr:9.9e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 18:11:23 ts/train.py:232 step:41K smpl:656K ep:3K epch:10.34 loss:0.013 grdn:0.208 lr:9.9e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-18 18:12:21 ts/train.py:232 step:41K smpl:659K ep:3K epch:10.39 loss:0.012 grdn:0.198 lr:9.9e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 18:13:20 ts/train.py:232 step:41K smpl:662K ep:3K epch:10.44 loss:0.013 grdn:0.205 lr:9.9e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 18:14:19 ts/train.py:232 step:42K smpl:666K ep:3K epch:10.49 loss:0.015 grdn:0.219 lr:9.9e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 18:15:17 ts/train.py:232 step:42K smpl:669K ep:3K epch:10.54 loss:0.013 grdn:0.216 lr:9.9e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 18:16:17 ts/train.py:232 step:42K smpl:672K ep:3K epch:10.59 loss:0.014 grdn:0.200 lr:9.9e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-18 18:17:17 ts/train.py:232 step:42K smpl:675K ep:3K epch:10.64 loss:0.013 grdn:0.216 lr:9.9e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-18 18:18:19 ts/train.py:232 step:42K smpl:678K ep:3K epch:10.69 loss:0.013 grdn:0.206 lr:9.9e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 18:19:23 ts/train.py:232 step:43K smpl:682K ep:3K epch:10.74 loss:0.013 grdn:0.215 lr:9.9e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 18:20:24 ts/train.py:232 step:43K smpl:685K ep:3K epch:10.80 loss:0.014 grdn:0.222 lr:9.9e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 18:21:27 ts/train.py:232 step:43K smpl:688K ep:3K epch:10.85 loss:0.013 grdn:0.208 lr:9.9e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 18:22:28 ts/train.py:232 step:43K smpl:691K ep:3K epch:10.90 loss:0.013 grdn:0.204 lr:9.9e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 18:23:31 ts/train.py:232 step:43K smpl:694K ep:3K epch:10.95 loss:0.013 grdn:0.205 lr:9.9e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 18:24:33 ts/train.py:232 step:44K smpl:698K ep:3K epch:11.00 loss:0.012 grdn:0.207 lr:9.9e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 18:25:36 ts/train.py:232 step:44K smpl:701K ep:3K epch:11.05 loss:0.013 grdn:0.202 lr:9.9e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 18:26:38 ts/train.py:232 step:44K smpl:704K ep:3K epch:11.10 loss:0.012 grdn:0.203 lr:9.9e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 18:27:41 ts/train.py:232 step:44K smpl:707K ep:3K epch:11.15 loss:0.014 grdn:0.215 lr:9.9e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 18:28:43 ts/train.py:232 step:44K smpl:710K ep:3K epch:11.20 loss:0.014 grdn:0.208 lr:9.9e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 18:29:46 ts/train.py:232 step:45K smpl:714K ep:3K epch:11.25 loss:0.012 grdn:0.208 lr:9.9e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 18:30:49 ts/train.py:232 step:45K smpl:717K ep:3K epch:11.30 loss:0.013 grdn:0.204 lr:9.9e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 18:31:51 ts/train.py:232 step:45K smpl:720K ep:3K epch:11.35 loss:0.014 grdn:0.214 lr:9.9e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 18:32:53 ts/train.py:232 step:45K smpl:723K ep:3K epch:11.40 loss:0.013 grdn:0.206 lr:9.9e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 18:33:56 ts/train.py:232 step:45K smpl:726K ep:3K epch:11.45 loss:0.013 grdn:0.207 lr:9.9e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 18:34:59 ts/train.py:232 step:46K smpl:730K ep:3K epch:11.50 loss:0.012 grdn:0.206 lr:9.9e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 18:36:03 ts/train.py:232 step:46K smpl:733K ep:3K epch:11.55 loss:0.013 grdn:0.213 lr:9.9e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 18:37:05 ts/train.py:232 step:46K smpl:736K ep:3K epch:11.60 loss:0.013 grdn:0.212 lr:9.9e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 18:38:11 ts/train.py:232 step:46K smpl:739K ep:3K epch:11.65 loss:0.013 grdn:0.207 lr:9.9e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-18 18:39:15 ts/train.py:232 step:46K smpl:742K ep:4K epch:11.70 loss:0.013 grdn:0.210 lr:9.9e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 18:40:19 ts/train.py:232 step:47K smpl:746K ep:4K epch:11.75 loss:0.012 grdn:0.193 lr:9.9e-05 updt_s:0.222 data_s:0.098 +INFO 2025-05-18 18:41:22 ts/train.py:232 step:47K smpl:749K ep:4K epch:11.80 loss:0.013 grdn:0.208 lr:9.9e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 18:42:26 ts/train.py:232 step:47K smpl:752K ep:4K epch:11.85 loss:0.013 grdn:0.201 lr:9.9e-05 updt_s:0.222 data_s:0.097 +INFO 2025-05-18 18:43:30 ts/train.py:232 step:47K smpl:755K ep:4K epch:11.91 loss:0.013 grdn:0.213 lr:9.9e-05 updt_s:0.222 data_s:0.101 +INFO 2025-05-18 18:44:27 ts/train.py:232 step:47K smpl:758K ep:4K epch:11.96 loss:0.013 grdn:0.200 lr:9.9e-05 updt_s:0.222 data_s:0.059 +INFO 2025-05-18 18:45:25 ts/train.py:232 step:48K smpl:762K ep:4K epch:12.01 loss:0.012 grdn:0.197 lr:9.8e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 18:46:22 ts/train.py:232 step:48K smpl:765K ep:4K epch:12.06 loss:0.013 grdn:0.211 lr:9.8e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-18 18:47:19 ts/train.py:232 step:48K smpl:768K ep:4K epch:12.11 loss:0.013 grdn:0.198 lr:9.8e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 18:48:17 ts/train.py:232 step:48K smpl:771K ep:4K epch:12.16 loss:0.012 grdn:0.195 lr:9.8e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 18:49:15 ts/train.py:232 step:48K smpl:774K ep:4K epch:12.21 loss:0.013 grdn:0.207 lr:9.8e-05 updt_s:0.222 data_s:0.066 +INFO 2025-05-18 18:50:12 ts/train.py:232 step:49K smpl:778K ep:4K epch:12.26 loss:0.013 grdn:0.213 lr:9.8e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 18:51:10 ts/train.py:232 step:49K smpl:781K ep:4K epch:12.31 loss:0.013 grdn:0.204 lr:9.8e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 18:52:06 ts/train.py:232 step:49K smpl:784K ep:4K epch:12.36 loss:0.014 grdn:0.216 lr:9.8e-05 updt_s:0.222 data_s:0.058 +INFO 2025-05-18 18:53:02 ts/train.py:232 step:49K smpl:787K ep:4K epch:12.41 loss:0.013 grdn:0.211 lr:9.8e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-18 18:53:59 ts/train.py:232 step:49K smpl:790K ep:4K epch:12.46 loss:0.012 grdn:0.193 lr:9.8e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-18 18:54:56 ts/train.py:232 step:50K smpl:794K ep:4K epch:12.51 loss:0.012 grdn:0.197 lr:9.8e-05 updt_s:0.222 data_s:0.060 +INFO 2025-05-18 18:55:53 ts/train.py:232 step:50K smpl:797K ep:4K epch:12.56 loss:0.012 grdn:0.193 lr:9.8e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-18 18:56:57 ts/train.py:232 step:50K smpl:800K ep:4K epch:12.61 loss:0.012 grdn:0.196 lr:9.8e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-18 18:58:01 ts/train.py:232 step:50K smpl:803K ep:4K epch:12.66 loss:0.012 grdn:0.197 lr:9.8e-05 updt_s:0.222 data_s:0.098 +INFO 2025-05-18 18:59:03 ts/train.py:232 step:50K smpl:806K ep:4K epch:12.71 loss:0.011 grdn:0.194 lr:9.8e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 19:00:07 ts/train.py:232 step:51K smpl:810K ep:4K epch:12.76 loss:0.013 grdn:0.194 lr:9.8e-05 updt_s:0.222 data_s:0.097 +INFO 2025-05-18 19:01:10 ts/train.py:232 step:51K smpl:813K ep:4K epch:12.81 loss:0.011 grdn:0.201 lr:9.8e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 19:02:14 ts/train.py:232 step:51K smpl:816K ep:4K epch:12.86 loss:0.012 grdn:0.206 lr:9.8e-05 updt_s:0.222 data_s:0.099 +INFO 2025-05-18 19:03:16 ts/train.py:232 step:51K smpl:819K ep:4K epch:12.91 loss:0.012 grdn:0.207 lr:9.8e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 19:04:18 ts/train.py:232 step:51K smpl:822K ep:4K epch:12.96 loss:0.012 grdn:0.193 lr:9.8e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 19:05:20 ts/train.py:232 step:52K smpl:826K ep:4K epch:13.01 loss:0.012 grdn:0.197 lr:9.8e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 19:06:21 ts/train.py:232 step:52K smpl:829K ep:4K epch:13.07 loss:0.013 grdn:0.208 lr:9.8e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-18 19:07:24 ts/train.py:232 step:52K smpl:832K ep:4K epch:13.12 loss:0.012 grdn:0.204 lr:9.8e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 19:08:26 ts/train.py:232 step:52K smpl:835K ep:4K epch:13.17 loss:0.013 grdn:0.226 lr:9.8e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 19:09:29 ts/train.py:232 step:52K smpl:838K ep:4K epch:13.22 loss:0.013 grdn:0.206 lr:9.8e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 19:10:30 ts/train.py:232 step:53K smpl:842K ep:4K epch:13.27 loss:0.012 grdn:0.192 lr:9.8e-05 updt_s:0.222 data_s:0.084 +INFO 2025-05-18 19:11:31 ts/train.py:232 step:53K smpl:845K ep:4K epch:13.32 loss:0.012 grdn:0.189 lr:9.8e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 19:12:35 ts/train.py:232 step:53K smpl:848K ep:4K epch:13.37 loss:0.013 grdn:0.205 lr:9.8e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 19:13:37 ts/train.py:232 step:53K smpl:851K ep:4K epch:13.42 loss:0.012 grdn:0.199 lr:9.8e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 19:14:40 ts/train.py:232 step:53K smpl:854K ep:4K epch:13.47 loss:0.012 grdn:0.194 lr:9.8e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 19:15:42 ts/train.py:232 step:54K smpl:858K ep:4K epch:13.52 loss:0.012 grdn:0.205 lr:9.8e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 19:16:47 ts/train.py:232 step:54K smpl:861K ep:4K epch:13.57 loss:0.012 grdn:0.197 lr:9.8e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-18 19:17:50 ts/train.py:232 step:54K smpl:864K ep:4K epch:13.62 loss:0.013 grdn:0.207 lr:9.8e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 19:18:53 ts/train.py:232 step:54K smpl:867K ep:4K epch:13.67 loss:0.012 grdn:0.191 lr:9.8e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 19:19:56 ts/train.py:232 step:54K smpl:870K ep:4K epch:13.72 loss:0.011 grdn:0.191 lr:9.8e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 19:20:58 ts/train.py:232 step:55K smpl:874K ep:4K epch:13.77 loss:0.012 grdn:0.193 lr:9.8e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 19:22:01 ts/train.py:232 step:55K smpl:877K ep:4K epch:13.82 loss:0.011 grdn:0.197 lr:9.8e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 19:23:03 ts/train.py:232 step:55K smpl:880K ep:4K epch:13.87 loss:0.012 grdn:0.203 lr:9.8e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 19:24:08 ts/train.py:232 step:55K smpl:883K ep:4K epch:13.92 loss:0.013 grdn:0.206 lr:9.8e-05 updt_s:0.222 data_s:0.099 +INFO 2025-05-18 19:25:11 ts/train.py:232 step:55K smpl:886K ep:4K epch:13.97 loss:0.012 grdn:0.201 lr:9.8e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 19:26:14 ts/train.py:232 step:56K smpl:890K ep:4K epch:14.02 loss:0.012 grdn:0.191 lr:9.8e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 19:27:19 ts/train.py:232 step:56K smpl:893K ep:4K epch:14.07 loss:0.012 grdn:0.206 lr:9.8e-05 updt_s:0.222 data_s:0.099 +INFO 2025-05-18 19:28:23 ts/train.py:232 step:56K smpl:896K ep:4K epch:14.12 loss:0.012 grdn:0.195 lr:9.8e-05 updt_s:0.222 data_s:0.097 +INFO 2025-05-18 19:29:26 ts/train.py:232 step:56K smpl:899K ep:4K epch:14.18 loss:0.012 grdn:0.212 lr:9.8e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 19:30:29 ts/train.py:232 step:56K smpl:902K ep:4K epch:14.23 loss:0.012 grdn:0.200 lr:9.8e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 19:31:32 ts/train.py:232 step:57K smpl:906K ep:4K epch:14.28 loss:0.013 grdn:0.208 lr:9.8e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 19:32:37 ts/train.py:232 step:57K smpl:909K ep:4K epch:14.33 loss:0.012 grdn:0.209 lr:9.8e-05 updt_s:0.222 data_s:0.102 +INFO 2025-05-18 19:33:40 ts/train.py:232 step:57K smpl:912K ep:4K epch:14.38 loss:0.013 grdn:0.205 lr:9.8e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 19:34:43 ts/train.py:232 step:57K smpl:915K ep:4K epch:14.43 loss:0.012 grdn:0.204 lr:9.8e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 19:35:43 ts/train.py:232 step:57K smpl:918K ep:4K epch:14.48 loss:0.011 grdn:0.195 lr:9.8e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-18 19:36:47 ts/train.py:232 step:58K smpl:922K ep:4K epch:14.53 loss:0.012 grdn:0.195 lr:9.8e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-18 19:37:50 ts/train.py:232 step:58K smpl:925K ep:4K epch:14.58 loss:0.012 grdn:0.196 lr:9.8e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 19:38:54 ts/train.py:232 step:58K smpl:928K ep:4K epch:14.63 loss:0.012 grdn:0.200 lr:9.8e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 19:39:57 ts/train.py:232 step:58K smpl:931K ep:4K epch:14.68 loss:0.011 grdn:0.185 lr:9.8e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 19:41:00 ts/train.py:232 step:58K smpl:934K ep:4K epch:14.73 loss:0.012 grdn:0.201 lr:9.8e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 19:42:03 ts/train.py:232 step:59K smpl:938K ep:4K epch:14.78 loss:0.012 grdn:0.204 lr:9.8e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 19:43:05 ts/train.py:232 step:59K smpl:941K ep:4K epch:14.83 loss:0.012 grdn:0.195 lr:9.8e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 19:44:07 ts/train.py:232 step:59K smpl:944K ep:4K epch:14.88 loss:0.011 grdn:0.184 lr:9.8e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 19:45:09 ts/train.py:232 step:59K smpl:947K ep:4K epch:14.93 loss:0.011 grdn:0.199 lr:9.8e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 19:46:11 ts/train.py:232 step:59K smpl:950K ep:4K epch:14.98 loss:0.012 grdn:0.195 lr:9.8e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 19:47:13 ts/train.py:232 step:60K smpl:954K ep:5K epch:15.03 loss:0.012 grdn:0.193 lr:9.8e-05 updt_s:0.222 data_s:0.085 +INFO 2025-05-18 19:48:16 ts/train.py:232 step:60K smpl:957K ep:5K epch:15.08 loss:0.013 grdn:0.202 lr:9.8e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 19:49:16 ts/train.py:232 step:60K smpl:960K ep:5K epch:15.13 loss:0.012 grdn:0.201 lr:9.8e-05 updt_s:0.222 data_s:0.079 +INFO 2025-05-18 19:49:16 ts/train.py:241 Checkpoint policy after step 60000 +INFO 2025-05-18 19:50:22 ts/train.py:232 step:60K smpl:963K ep:5K epch:15.18 loss:0.012 grdn:0.204 lr:9.8e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-18 19:51:23 ts/train.py:232 step:60K smpl:966K ep:5K epch:15.23 loss:0.011 grdn:0.198 lr:9.8e-05 updt_s:0.222 data_s:0.084 +INFO 2025-05-18 19:52:24 ts/train.py:232 step:61K smpl:970K ep:5K epch:15.28 loss:0.012 grdn:0.194 lr:9.8e-05 updt_s:0.222 data_s:0.083 +INFO 2025-05-18 19:53:25 ts/train.py:232 step:61K smpl:973K ep:5K epch:15.34 loss:0.011 grdn:0.195 lr:9.8e-05 updt_s:0.222 data_s:0.084 +INFO 2025-05-18 19:54:27 ts/train.py:232 step:61K smpl:976K ep:5K epch:15.39 loss:0.011 grdn:0.192 lr:9.8e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 19:55:28 ts/train.py:232 step:61K smpl:979K ep:5K epch:15.44 loss:0.011 grdn:0.207 lr:9.7e-05 updt_s:0.222 data_s:0.085 +INFO 2025-05-18 19:56:32 ts/train.py:232 step:61K smpl:982K ep:5K epch:15.49 loss:0.011 grdn:0.204 lr:9.7e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-18 19:57:35 ts/train.py:232 step:62K smpl:986K ep:5K epch:15.54 loss:0.011 grdn:0.186 lr:9.7e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 19:58:37 ts/train.py:232 step:62K smpl:989K ep:5K epch:15.59 loss:0.011 grdn:0.194 lr:9.7e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 19:59:38 ts/train.py:232 step:62K smpl:992K ep:5K epch:15.64 loss:0.011 grdn:0.192 lr:9.7e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-18 20:00:40 ts/train.py:232 step:62K smpl:995K ep:5K epch:15.69 loss:0.011 grdn:0.196 lr:9.7e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 20:01:42 ts/train.py:232 step:62K smpl:998K ep:5K epch:15.74 loss:0.012 grdn:0.197 lr:9.7e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 20:02:43 ts/train.py:232 step:63K smpl:1M ep:5K epch:15.79 loss:0.010 grdn:0.181 lr:9.7e-05 updt_s:0.222 data_s:0.080 +INFO 2025-05-18 20:03:39 ts/train.py:232 step:63K smpl:1M ep:5K epch:15.84 loss:0.011 grdn:0.186 lr:9.7e-05 updt_s:0.222 data_s:0.057 +INFO 2025-05-18 20:04:36 ts/train.py:232 step:63K smpl:1M ep:5K epch:15.89 loss:0.012 grdn:0.195 lr:9.7e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 20:05:35 ts/train.py:232 step:63K smpl:1M ep:5K epch:15.94 loss:0.010 grdn:0.185 lr:9.7e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 20:06:32 ts/train.py:232 step:63K smpl:1M ep:5K epch:15.99 loss:0.012 grdn:0.199 lr:9.7e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 20:07:31 ts/train.py:232 step:64K smpl:1M ep:5K epch:16.04 loss:0.012 grdn:0.200 lr:9.7e-05 updt_s:0.223 data_s:0.069 +INFO 2025-05-18 20:08:29 ts/train.py:232 step:64K smpl:1M ep:5K epch:16.09 loss:0.012 grdn:0.199 lr:9.7e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 20:09:27 ts/train.py:232 step:64K smpl:1M ep:5K epch:16.14 loss:0.012 grdn:0.195 lr:9.7e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 20:10:26 ts/train.py:232 step:64K smpl:1M ep:5K epch:16.19 loss:0.011 grdn:0.196 lr:9.7e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 20:11:24 ts/train.py:232 step:64K smpl:1M ep:5K epch:16.24 loss:0.010 grdn:0.179 lr:9.7e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 20:12:22 ts/train.py:232 step:65K smpl:1M ep:5K epch:16.29 loss:0.010 grdn:0.184 lr:9.7e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 20:13:20 ts/train.py:232 step:65K smpl:1M ep:5K epch:16.34 loss:0.012 grdn:0.203 lr:9.7e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 20:14:19 ts/train.py:232 step:65K smpl:1M ep:5K epch:16.39 loss:0.011 grdn:0.180 lr:9.7e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 20:15:18 ts/train.py:232 step:65K smpl:1M ep:5K epch:16.45 loss:0.010 grdn:0.183 lr:9.7e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-18 20:16:21 ts/train.py:232 step:65K smpl:1M ep:5K epch:16.50 loss:0.011 grdn:0.201 lr:9.7e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 20:17:24 ts/train.py:232 step:66K smpl:1M ep:5K epch:16.55 loss:0.011 grdn:0.199 lr:9.7e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 20:18:26 ts/train.py:232 step:66K smpl:1M ep:5K epch:16.60 loss:0.011 grdn:0.196 lr:9.7e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 20:19:29 ts/train.py:232 step:66K smpl:1M ep:5K epch:16.65 loss:0.012 grdn:0.195 lr:9.7e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 20:20:33 ts/train.py:232 step:66K smpl:1M ep:5K epch:16.70 loss:0.011 grdn:0.190 lr:9.7e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 20:21:36 ts/train.py:232 step:66K smpl:1M ep:5K epch:16.75 loss:0.011 grdn:0.192 lr:9.7e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 20:22:37 ts/train.py:232 step:67K smpl:1M ep:5K epch:16.80 loss:0.011 grdn:0.186 lr:9.7e-05 updt_s:0.222 data_s:0.083 +INFO 2025-05-18 20:23:39 ts/train.py:232 step:67K smpl:1M ep:5K epch:16.85 loss:0.012 grdn:0.192 lr:9.7e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 20:24:41 ts/train.py:232 step:67K smpl:1M ep:5K epch:16.90 loss:0.011 grdn:0.189 lr:9.7e-05 updt_s:0.222 data_s:0.084 +INFO 2025-05-18 20:25:42 ts/train.py:232 step:67K smpl:1M ep:5K epch:16.95 loss:0.011 grdn:0.187 lr:9.7e-05 updt_s:0.222 data_s:0.083 +INFO 2025-05-18 20:26:44 ts/train.py:232 step:67K smpl:1M ep:5K epch:17.00 loss:0.012 grdn:0.196 lr:9.7e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 20:27:47 ts/train.py:232 step:68K smpl:1M ep:5K epch:17.05 loss:0.011 grdn:0.194 lr:9.7e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 20:28:51 ts/train.py:232 step:68K smpl:1M ep:5K epch:17.10 loss:0.011 grdn:0.190 lr:9.7e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 20:29:53 ts/train.py:232 step:68K smpl:1M ep:5K epch:17.15 loss:0.011 grdn:0.190 lr:9.7e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 20:30:55 ts/train.py:232 step:68K smpl:1M ep:5K epch:17.20 loss:0.012 grdn:0.196 lr:9.7e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 20:31:59 ts/train.py:232 step:68K smpl:1M ep:5K epch:17.25 loss:0.012 grdn:0.201 lr:9.7e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 20:33:01 ts/train.py:232 step:69K smpl:1M ep:5K epch:17.30 loss:0.011 grdn:0.196 lr:9.7e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 20:34:03 ts/train.py:232 step:69K smpl:1M ep:5K epch:17.35 loss:0.011 grdn:0.188 lr:9.7e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 20:35:06 ts/train.py:232 step:69K smpl:1M ep:5K epch:17.40 loss:0.012 grdn:0.202 lr:9.7e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 20:36:11 ts/train.py:232 step:69K smpl:1M ep:5K epch:17.45 loss:0.011 grdn:0.193 lr:9.7e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-18 20:37:14 ts/train.py:232 step:69K smpl:1M ep:5K epch:17.50 loss:0.011 grdn:0.190 lr:9.7e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 20:38:18 ts/train.py:232 step:70K smpl:1M ep:5K epch:17.55 loss:0.011 grdn:0.192 lr:9.7e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 20:39:21 ts/train.py:232 step:70K smpl:1M ep:5K epch:17.61 loss:0.011 grdn:0.182 lr:9.7e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 20:40:23 ts/train.py:232 step:70K smpl:1M ep:5K epch:17.66 loss:0.011 grdn:0.194 lr:9.7e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 20:41:26 ts/train.py:232 step:70K smpl:1M ep:5K epch:17.71 loss:0.011 grdn:0.194 lr:9.7e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 20:42:25 ts/train.py:232 step:70K smpl:1M ep:5K epch:17.76 loss:0.012 grdn:0.194 lr:9.7e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 20:43:22 ts/train.py:232 step:71K smpl:1M ep:5K epch:17.81 loss:0.011 grdn:0.184 lr:9.7e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 20:44:20 ts/train.py:232 step:71K smpl:1M ep:5K epch:17.86 loss:0.011 grdn:0.189 lr:9.7e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 20:45:19 ts/train.py:232 step:71K smpl:1M ep:5K epch:17.91 loss:0.011 grdn:0.187 lr:9.7e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 20:46:17 ts/train.py:232 step:71K smpl:1M ep:5K epch:17.96 loss:0.012 grdn:0.188 lr:9.7e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 20:47:15 ts/train.py:232 step:71K smpl:1M ep:5K epch:18.01 loss:0.011 grdn:0.189 lr:9.7e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 20:48:13 ts/train.py:232 step:72K smpl:1M ep:5K epch:18.06 loss:0.011 grdn:0.194 lr:9.7e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 20:49:11 ts/train.py:232 step:72K smpl:1M ep:5K epch:18.11 loss:0.011 grdn:0.191 lr:9.7e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 20:50:09 ts/train.py:232 step:72K smpl:1M ep:5K epch:18.16 loss:0.011 grdn:0.197 lr:9.7e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 20:51:06 ts/train.py:232 step:72K smpl:1M ep:5K epch:18.21 loss:0.011 grdn:0.183 lr:9.7e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-18 20:52:05 ts/train.py:232 step:72K smpl:1M ep:5K epch:18.26 loss:0.011 grdn:0.187 lr:9.7e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 20:53:03 ts/train.py:232 step:73K smpl:1M ep:5K epch:18.31 loss:0.011 grdn:0.194 lr:9.6e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 20:54:00 ts/train.py:232 step:73K smpl:1M ep:6K epch:18.36 loss:0.011 grdn:0.194 lr:9.6e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 20:55:04 ts/train.py:232 step:73K smpl:1M ep:6K epch:18.41 loss:0.010 grdn:0.178 lr:9.6e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-18 20:56:05 ts/train.py:232 step:73K smpl:1M ep:6K epch:18.46 loss:0.011 grdn:0.183 lr:9.6e-05 updt_s:0.222 data_s:0.085 +INFO 2025-05-18 20:57:07 ts/train.py:232 step:73K smpl:1M ep:6K epch:18.51 loss:0.010 grdn:0.191 lr:9.6e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 20:58:09 ts/train.py:232 step:74K smpl:1M ep:6K epch:18.56 loss:0.010 grdn:0.188 lr:9.6e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 20:59:11 ts/train.py:232 step:74K smpl:1M ep:6K epch:18.61 loss:0.010 grdn:0.183 lr:9.6e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 21:00:14 ts/train.py:232 step:74K smpl:1M ep:6K epch:18.66 loss:0.011 grdn:0.190 lr:9.6e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 21:01:13 ts/train.py:232 step:74K smpl:1M ep:6K epch:18.72 loss:0.011 grdn:0.201 lr:9.6e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 21:02:11 ts/train.py:232 step:74K smpl:1M ep:6K epch:18.77 loss:0.011 grdn:0.190 lr:9.6e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 21:03:10 ts/train.py:232 step:75K smpl:1M ep:6K epch:18.82 loss:0.010 grdn:0.179 lr:9.6e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 21:04:08 ts/train.py:232 step:75K smpl:1M ep:6K epch:18.87 loss:0.011 grdn:0.184 lr:9.6e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 21:05:05 ts/train.py:232 step:75K smpl:1M ep:6K epch:18.92 loss:0.011 grdn:0.195 lr:9.6e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 21:06:02 ts/train.py:232 step:75K smpl:1M ep:6K epch:18.97 loss:0.010 grdn:0.183 lr:9.6e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 21:06:59 ts/train.py:232 step:75K smpl:1M ep:6K epch:19.02 loss:0.011 grdn:0.191 lr:9.6e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 21:07:57 ts/train.py:232 step:76K smpl:1M ep:6K epch:19.07 loss:0.010 grdn:0.192 lr:9.6e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 21:08:54 ts/train.py:232 step:76K smpl:1M ep:6K epch:19.12 loss:0.011 grdn:0.195 lr:9.6e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 21:09:52 ts/train.py:232 step:76K smpl:1M ep:6K epch:19.17 loss:0.011 grdn:0.187 lr:9.6e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 21:10:49 ts/train.py:232 step:76K smpl:1M ep:6K epch:19.22 loss:0.011 grdn:0.187 lr:9.6e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 21:11:48 ts/train.py:232 step:76K smpl:1M ep:6K epch:19.27 loss:0.011 grdn:0.197 lr:9.6e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 21:12:46 ts/train.py:232 step:77K smpl:1M ep:6K epch:19.32 loss:0.011 grdn:0.188 lr:9.6e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 21:13:48 ts/train.py:232 step:77K smpl:1M ep:6K epch:19.37 loss:0.011 grdn:0.188 lr:9.6e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-18 21:14:50 ts/train.py:232 step:77K smpl:1M ep:6K epch:19.42 loss:0.010 grdn:0.175 lr:9.6e-05 updt_s:0.222 data_s:0.085 +INFO 2025-05-18 21:15:51 ts/train.py:232 step:77K smpl:1M ep:6K epch:19.47 loss:0.010 grdn:0.190 lr:9.6e-05 updt_s:0.222 data_s:0.085 +INFO 2025-05-18 21:16:54 ts/train.py:232 step:77K smpl:1M ep:6K epch:19.52 loss:0.011 grdn:0.190 lr:9.6e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 21:17:56 ts/train.py:232 step:78K smpl:1M ep:6K epch:19.57 loss:0.011 grdn:0.186 lr:9.6e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 21:18:57 ts/train.py:232 step:78K smpl:1M ep:6K epch:19.62 loss:0.011 grdn:0.190 lr:9.6e-05 updt_s:0.222 data_s:0.085 +INFO 2025-05-18 21:20:00 ts/train.py:232 step:78K smpl:1M ep:6K epch:19.67 loss:0.010 grdn:0.187 lr:9.6e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 21:21:01 ts/train.py:232 step:78K smpl:1M ep:6K epch:19.72 loss:0.011 grdn:0.192 lr:9.6e-05 updt_s:0.222 data_s:0.085 +INFO 2025-05-18 21:22:03 ts/train.py:232 step:78K smpl:1M ep:6K epch:19.77 loss:0.010 grdn:0.179 lr:9.6e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 21:23:05 ts/train.py:232 step:79K smpl:1M ep:6K epch:19.83 loss:0.010 grdn:0.186 lr:9.6e-05 updt_s:0.222 data_s:0.083 +INFO 2025-05-18 21:24:07 ts/train.py:232 step:79K smpl:1M ep:6K epch:19.88 loss:0.010 grdn:0.180 lr:9.6e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 21:25:08 ts/train.py:232 step:79K smpl:1M ep:6K epch:19.93 loss:0.011 grdn:0.199 lr:9.6e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 21:26:11 ts/train.py:232 step:79K smpl:1M ep:6K epch:19.98 loss:0.010 grdn:0.179 lr:9.6e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 21:27:14 ts/train.py:232 step:79K smpl:1M ep:6K epch:20.03 loss:0.011 grdn:0.192 lr:9.6e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 21:28:16 ts/train.py:232 step:80K smpl:1M ep:6K epch:20.08 loss:0.010 grdn:0.183 lr:9.6e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 21:29:17 ts/train.py:232 step:80K smpl:1M ep:6K epch:20.13 loss:0.011 grdn:0.191 lr:9.6e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 21:30:20 ts/train.py:232 step:80K smpl:1M ep:6K epch:20.18 loss:0.011 grdn:0.196 lr:9.6e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 21:30:20 ts/train.py:241 Checkpoint policy after step 80000 +INFO 2025-05-18 21:31:25 ts/train.py:232 step:80K smpl:1M ep:6K epch:20.23 loss:0.010 grdn:0.173 lr:9.6e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-18 21:32:28 ts/train.py:232 step:80K smpl:1M ep:6K epch:20.28 loss:0.010 grdn:0.177 lr:9.6e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 21:33:32 ts/train.py:232 step:81K smpl:1M ep:6K epch:20.33 loss:0.010 grdn:0.179 lr:9.6e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-18 21:34:35 ts/train.py:232 step:81K smpl:1M ep:6K epch:20.38 loss:0.010 grdn:0.186 lr:9.6e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 21:35:38 ts/train.py:232 step:81K smpl:1M ep:6K epch:20.43 loss:0.011 grdn:0.202 lr:9.6e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 21:36:41 ts/train.py:232 step:81K smpl:1M ep:6K epch:20.48 loss:0.011 grdn:0.193 lr:9.6e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 21:37:44 ts/train.py:232 step:81K smpl:1M ep:6K epch:20.53 loss:0.009 grdn:0.179 lr:9.6e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 21:38:48 ts/train.py:232 step:82K smpl:1M ep:6K epch:20.58 loss:0.010 grdn:0.180 lr:9.6e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 21:39:51 ts/train.py:232 step:82K smpl:1M ep:6K epch:20.63 loss:0.011 grdn:0.198 lr:9.6e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 21:40:54 ts/train.py:232 step:82K smpl:1M ep:6K epch:20.68 loss:0.010 grdn:0.191 lr:9.6e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 21:41:57 ts/train.py:232 step:82K smpl:1M ep:6K epch:20.73 loss:0.010 grdn:0.175 lr:9.5e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 21:43:00 ts/train.py:232 step:82K smpl:1M ep:6K epch:20.78 loss:0.011 grdn:0.191 lr:9.5e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 21:44:02 ts/train.py:232 step:83K smpl:1M ep:6K epch:20.83 loss:0.010 grdn:0.177 lr:9.5e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 21:45:05 ts/train.py:232 step:83K smpl:1M ep:6K epch:20.88 loss:0.011 grdn:0.185 lr:9.5e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 21:46:05 ts/train.py:232 step:83K smpl:1M ep:6K epch:20.93 loss:0.010 grdn:0.180 lr:9.5e-05 updt_s:0.222 data_s:0.081 +INFO 2025-05-18 21:47:08 ts/train.py:232 step:83K smpl:1M ep:6K epch:20.99 loss:0.010 grdn:0.189 lr:9.5e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 21:48:10 ts/train.py:232 step:83K smpl:1M ep:6K epch:21.04 loss:0.010 grdn:0.197 lr:9.5e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 21:49:13 ts/train.py:232 step:84K smpl:1M ep:6K epch:21.09 loss:0.010 grdn:0.189 lr:9.5e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 21:50:15 ts/train.py:232 step:84K smpl:1M ep:6K epch:21.14 loss:0.010 grdn:0.180 lr:9.5e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 21:51:16 ts/train.py:232 step:84K smpl:1M ep:6K epch:21.19 loss:0.010 grdn:0.183 lr:9.5e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-18 21:52:19 ts/train.py:232 step:84K smpl:1M ep:6K epch:21.24 loss:0.011 grdn:0.192 lr:9.5e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 21:53:22 ts/train.py:232 step:84K smpl:1M ep:6K epch:21.29 loss:0.010 grdn:0.182 lr:9.5e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-18 21:54:24 ts/train.py:232 step:85K smpl:1M ep:6K epch:21.34 loss:0.010 grdn:0.181 lr:9.5e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 21:55:26 ts/train.py:232 step:85K smpl:1M ep:6K epch:21.39 loss:0.009 grdn:0.177 lr:9.5e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 21:56:28 ts/train.py:232 step:85K smpl:1M ep:6K epch:21.44 loss:0.010 grdn:0.178 lr:9.5e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 21:57:29 ts/train.py:232 step:85K smpl:1M ep:6K epch:21.49 loss:0.010 grdn:0.180 lr:9.5e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 21:58:32 ts/train.py:232 step:85K smpl:1M ep:6K epch:21.54 loss:0.011 grdn:0.186 lr:9.5e-05 updt_s:0.222 data_s:0.092 +INFO 2025-05-18 21:59:34 ts/train.py:232 step:86K smpl:1M ep:6K epch:21.59 loss:0.011 grdn:0.193 lr:9.5e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 22:00:34 ts/train.py:232 step:86K smpl:1M ep:6K epch:21.64 loss:0.011 grdn:0.198 lr:9.5e-05 updt_s:0.222 data_s:0.076 +INFO 2025-05-18 22:01:33 ts/train.py:232 step:86K smpl:1M ep:7K epch:21.69 loss:0.011 grdn:0.180 lr:9.5e-05 updt_s:0.222 data_s:0.075 +INFO 2025-05-18 22:02:33 ts/train.py:232 step:86K smpl:1M ep:7K epch:21.74 loss:0.011 grdn:0.202 lr:9.5e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-18 22:03:33 ts/train.py:232 step:86K smpl:1M ep:7K epch:21.79 loss:0.010 grdn:0.183 lr:9.5e-05 updt_s:0.222 data_s:0.078 +INFO 2025-05-18 22:04:33 ts/train.py:232 step:87K smpl:1M ep:7K epch:21.84 loss:0.010 grdn:0.189 lr:9.5e-05 updt_s:0.222 data_s:0.075 +INFO 2025-05-18 22:05:33 ts/train.py:232 step:87K smpl:1M ep:7K epch:21.89 loss:0.010 grdn:0.185 lr:9.5e-05 updt_s:0.222 data_s:0.080 +INFO 2025-05-18 22:06:33 ts/train.py:232 step:87K smpl:1M ep:7K epch:21.94 loss:0.011 grdn:0.186 lr:9.5e-05 updt_s:0.222 data_s:0.079 +INFO 2025-05-18 22:07:33 ts/train.py:232 step:87K smpl:1M ep:7K epch:21.99 loss:0.010 grdn:0.182 lr:9.5e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-18 22:08:33 ts/train.py:232 step:87K smpl:1M ep:7K epch:22.04 loss:0.011 grdn:0.193 lr:9.5e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-18 22:09:32 ts/train.py:232 step:88K smpl:1M ep:7K epch:22.10 loss:0.010 grdn:0.178 lr:9.5e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-18 22:10:33 ts/train.py:232 step:88K smpl:1M ep:7K epch:22.15 loss:0.010 grdn:0.182 lr:9.5e-05 updt_s:0.222 data_s:0.078 +INFO 2025-05-18 22:11:33 ts/train.py:232 step:88K smpl:1M ep:7K epch:22.20 loss:0.009 grdn:0.176 lr:9.5e-05 updt_s:0.222 data_s:0.079 +INFO 2025-05-18 22:12:34 ts/train.py:232 step:88K smpl:1M ep:7K epch:22.25 loss:0.011 grdn:0.194 lr:9.5e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-18 22:13:38 ts/train.py:232 step:88K smpl:1M ep:7K epch:22.30 loss:0.010 grdn:0.181 lr:9.5e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 22:14:41 ts/train.py:232 step:89K smpl:1M ep:7K epch:22.35 loss:0.009 grdn:0.173 lr:9.5e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 22:15:44 ts/train.py:232 step:89K smpl:1M ep:7K epch:22.40 loss:0.009 grdn:0.178 lr:9.5e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 22:16:47 ts/train.py:232 step:89K smpl:1M ep:7K epch:22.45 loss:0.010 grdn:0.177 lr:9.5e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 22:17:50 ts/train.py:232 step:89K smpl:1M ep:7K epch:22.50 loss:0.009 grdn:0.180 lr:9.5e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-18 22:18:53 ts/train.py:232 step:89K smpl:1M ep:7K epch:22.55 loss:0.010 grdn:0.177 lr:9.5e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 22:19:50 ts/train.py:232 step:90K smpl:1M ep:7K epch:22.60 loss:0.011 grdn:0.189 lr:9.5e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-18 22:20:48 ts/train.py:232 step:90K smpl:1M ep:7K epch:22.65 loss:0.010 grdn:0.178 lr:9.5e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 22:21:46 ts/train.py:232 step:90K smpl:1M ep:7K epch:22.70 loss:0.010 grdn:0.185 lr:9.5e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 22:22:43 ts/train.py:232 step:90K smpl:1M ep:7K epch:22.75 loss:0.010 grdn:0.183 lr:9.5e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 22:23:39 ts/train.py:232 step:90K smpl:1M ep:7K epch:22.80 loss:0.010 grdn:0.181 lr:9.5e-05 updt_s:0.222 data_s:0.055 +INFO 2025-05-18 22:24:36 ts/train.py:232 step:91K smpl:1M ep:7K epch:22.85 loss:0.010 grdn:0.181 lr:9.5e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 22:25:33 ts/train.py:232 step:91K smpl:1M ep:7K epch:22.90 loss:0.010 grdn:0.183 lr:9.5e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 22:26:31 ts/train.py:232 step:91K smpl:1M ep:7K epch:22.95 loss:0.010 grdn:0.175 lr:9.4e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 22:27:28 ts/train.py:232 step:91K smpl:1M ep:7K epch:23.00 loss:0.010 grdn:0.187 lr:9.4e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 22:28:26 ts/train.py:232 step:91K smpl:1M ep:7K epch:23.05 loss:0.010 grdn:0.183 lr:9.4e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 22:29:24 ts/train.py:232 step:92K smpl:1M ep:7K epch:23.10 loss:0.011 grdn:0.195 lr:9.4e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 22:30:23 ts/train.py:232 step:92K smpl:1M ep:7K epch:23.15 loss:0.010 grdn:0.188 lr:9.4e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 22:31:21 ts/train.py:232 step:92K smpl:1M ep:7K epch:23.20 loss:0.010 grdn:0.174 lr:9.4e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 22:32:26 ts/train.py:232 step:92K smpl:1M ep:7K epch:23.26 loss:0.010 grdn:0.179 lr:9.4e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-18 22:33:28 ts/train.py:232 step:92K smpl:1M ep:7K epch:23.31 loss:0.010 grdn:0.176 lr:9.4e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-18 22:34:31 ts/train.py:232 step:93K smpl:1M ep:7K epch:23.36 loss:0.010 grdn:0.189 lr:9.4e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-18 22:35:33 ts/train.py:232 step:93K smpl:1M ep:7K epch:23.41 loss:0.010 grdn:0.190 lr:9.4e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-18 22:36:36 ts/train.py:232 step:93K smpl:1M ep:7K epch:23.46 loss:0.010 grdn:0.183 lr:9.4e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-18 22:37:39 ts/train.py:232 step:93K smpl:1M ep:7K epch:23.51 loss:0.009 grdn:0.175 lr:9.4e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-18 22:38:38 ts/train.py:232 step:93K smpl:1M ep:7K epch:23.56 loss:0.009 grdn:0.167 lr:9.4e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 22:39:37 ts/train.py:232 step:94K smpl:1M ep:7K epch:23.61 loss:0.009 grdn:0.178 lr:9.4e-05 updt_s:0.222 data_s:0.073 +INFO 2025-05-18 22:40:36 ts/train.py:232 step:94K smpl:2M ep:7K epch:23.66 loss:0.010 grdn:0.184 lr:9.4e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-18 22:41:36 ts/train.py:232 step:94K smpl:2M ep:7K epch:23.71 loss:0.010 grdn:0.179 lr:9.4e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-18 22:42:36 ts/train.py:232 step:94K smpl:2M ep:7K epch:23.76 loss:0.009 grdn:0.179 lr:9.4e-05 updt_s:0.222 data_s:0.079 +INFO 2025-05-18 22:43:36 ts/train.py:232 step:94K smpl:2M ep:7K epch:23.81 loss:0.010 grdn:0.182 lr:9.4e-05 updt_s:0.222 data_s:0.076 +INFO 2025-05-18 22:44:36 ts/train.py:232 step:95K smpl:2M ep:7K epch:23.86 loss:0.009 grdn:0.166 lr:9.4e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-18 22:45:32 ts/train.py:232 step:95K smpl:2M ep:7K epch:23.91 loss:0.010 grdn:0.182 lr:9.4e-05 updt_s:0.222 data_s:0.058 +INFO 2025-05-18 22:46:30 ts/train.py:232 step:95K smpl:2M ep:7K epch:23.96 loss:0.010 grdn:0.181 lr:9.4e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 22:47:28 ts/train.py:232 step:95K smpl:2M ep:7K epch:24.01 loss:0.010 grdn:0.186 lr:9.4e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 22:48:27 ts/train.py:232 step:95K smpl:2M ep:7K epch:24.06 loss:0.009 grdn:0.168 lr:9.4e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-18 22:49:26 ts/train.py:232 step:96K smpl:2M ep:7K epch:24.11 loss:0.010 grdn:0.188 lr:9.4e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 22:50:24 ts/train.py:232 step:96K smpl:2M ep:7K epch:24.16 loss:0.011 grdn:0.188 lr:9.4e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 22:51:26 ts/train.py:232 step:96K smpl:2M ep:7K epch:24.21 loss:0.009 grdn:0.174 lr:9.4e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-18 22:52:28 ts/train.py:232 step:96K smpl:2M ep:7K epch:24.26 loss:0.010 grdn:0.171 lr:9.4e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-18 22:53:30 ts/train.py:232 step:96K smpl:2M ep:7K epch:24.31 loss:0.009 grdn:0.177 lr:9.4e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-18 22:54:33 ts/train.py:232 step:97K smpl:2M ep:7K epch:24.37 loss:0.010 grdn:0.188 lr:9.4e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-18 22:55:36 ts/train.py:232 step:97K smpl:2M ep:7K epch:24.42 loss:0.010 grdn:0.179 lr:9.4e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-18 22:56:37 ts/train.py:232 step:97K smpl:2M ep:7K epch:24.47 loss:0.009 grdn:0.175 lr:9.4e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-18 22:57:37 ts/train.py:232 step:97K smpl:2M ep:7K epch:24.52 loss:0.008 grdn:0.161 lr:9.4e-05 updt_s:0.222 data_s:0.073 +INFO 2025-05-18 22:58:35 ts/train.py:232 step:97K smpl:2M ep:7K epch:24.57 loss:0.010 grdn:0.191 lr:9.4e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 22:59:32 ts/train.py:232 step:98K smpl:2M ep:7K epch:24.62 loss:0.010 grdn:0.195 lr:9.4e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 23:00:30 ts/train.py:232 step:98K smpl:2M ep:7K epch:24.67 loss:0.010 grdn:0.183 lr:9.4e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 23:01:28 ts/train.py:232 step:98K smpl:2M ep:7K epch:24.72 loss:0.010 grdn:0.184 lr:9.4e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 23:02:25 ts/train.py:232 step:98K smpl:2M ep:7K epch:24.77 loss:0.010 grdn:0.188 lr:9.4e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 23:03:24 ts/train.py:232 step:98K smpl:2M ep:7K epch:24.82 loss:0.010 grdn:0.175 lr:9.4e-05 updt_s:0.222 data_s:0.073 +INFO 2025-05-18 23:04:21 ts/train.py:232 step:99K smpl:2M ep:7K epch:24.87 loss:0.010 grdn:0.187 lr:9.4e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 23:05:20 ts/train.py:232 step:99K smpl:2M ep:7K epch:24.92 loss:0.010 grdn:0.191 lr:9.4e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 23:06:18 ts/train.py:232 step:99K smpl:2M ep:7K epch:24.97 loss:0.010 grdn:0.177 lr:9.3e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 23:07:17 ts/train.py:232 step:99K smpl:2M ep:8K epch:25.02 loss:0.009 grdn:0.168 lr:9.3e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-18 23:08:16 ts/train.py:232 step:99K smpl:2M ep:8K epch:25.07 loss:0.010 grdn:0.191 lr:9.3e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-18 23:09:16 ts/train.py:232 step:100K smpl:2M ep:8K epch:25.12 loss:0.010 grdn:0.183 lr:9.3e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-18 23:10:20 ts/train.py:232 step:100K smpl:2M ep:8K epch:25.17 loss:0.010 grdn:0.182 lr:9.3e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-18 23:11:25 ts/train.py:232 step:100K smpl:2M ep:8K epch:25.22 loss:0.010 grdn:0.187 lr:9.3e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-18 23:11:25 ts/train.py:241 Checkpoint policy after step 100000 +INFO 2025-05-18 23:12:33 ts/train.py:232 step:100K smpl:2M ep:8K epch:25.27 loss:0.009 grdn:0.177 lr:9.3e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-18 23:13:38 ts/train.py:232 step:100K smpl:2M ep:8K epch:25.32 loss:0.010 grdn:0.174 lr:9.3e-05 updt_s:0.222 data_s:0.102 +INFO 2025-05-18 23:14:43 ts/train.py:232 step:101K smpl:2M ep:8K epch:25.37 loss:0.009 grdn:0.175 lr:9.3e-05 updt_s:0.222 data_s:0.103 +INFO 2025-05-18 23:15:46 ts/train.py:232 step:101K smpl:2M ep:8K epch:25.42 loss:0.010 grdn:0.178 lr:9.3e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-18 23:16:47 ts/train.py:232 step:101K smpl:2M ep:8K epch:25.47 loss:0.009 grdn:0.180 lr:9.3e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-18 23:17:45 ts/train.py:232 step:101K smpl:2M ep:8K epch:25.53 loss:0.010 grdn:0.182 lr:9.3e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 23:18:41 ts/train.py:232 step:101K smpl:2M ep:8K epch:25.58 loss:0.010 grdn:0.188 lr:9.3e-05 updt_s:0.222 data_s:0.059 +INFO 2025-05-18 23:19:38 ts/train.py:232 step:102K smpl:2M ep:8K epch:25.63 loss:0.010 grdn:0.182 lr:9.3e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 23:20:35 ts/train.py:232 step:102K smpl:2M ep:8K epch:25.68 loss:0.011 grdn:0.192 lr:9.3e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-18 23:21:33 ts/train.py:232 step:102K smpl:2M ep:8K epch:25.73 loss:0.009 grdn:0.175 lr:9.3e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 23:22:31 ts/train.py:232 step:102K smpl:2M ep:8K epch:25.78 loss:0.010 grdn:0.178 lr:9.3e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 23:23:29 ts/train.py:232 step:102K smpl:2M ep:8K epch:25.83 loss:0.009 grdn:0.179 lr:9.3e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 23:24:26 ts/train.py:232 step:103K smpl:2M ep:8K epch:25.88 loss:0.010 grdn:0.184 lr:9.3e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 23:25:23 ts/train.py:232 step:103K smpl:2M ep:8K epch:25.93 loss:0.010 grdn:0.186 lr:9.3e-05 updt_s:0.221 data_s:0.064 +INFO 2025-05-18 23:26:20 ts/train.py:232 step:103K smpl:2M ep:8K epch:25.98 loss:0.010 grdn:0.182 lr:9.3e-05 updt_s:0.221 data_s:0.061 +INFO 2025-05-18 23:27:17 ts/train.py:232 step:103K smpl:2M ep:8K epch:26.03 loss:0.010 grdn:0.183 lr:9.3e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 23:28:14 ts/train.py:232 step:103K smpl:2M ep:8K epch:26.08 loss:0.010 grdn:0.182 lr:9.3e-05 updt_s:0.221 data_s:0.062 +INFO 2025-05-18 23:29:15 ts/train.py:232 step:104K smpl:2M ep:8K epch:26.13 loss:0.009 grdn:0.178 lr:9.3e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-18 23:30:15 ts/train.py:232 step:104K smpl:2M ep:8K epch:26.18 loss:0.009 grdn:0.162 lr:9.3e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-18 23:31:17 ts/train.py:232 step:104K smpl:2M ep:8K epch:26.23 loss:0.010 grdn:0.187 lr:9.3e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-18 23:32:19 ts/train.py:232 step:104K smpl:2M ep:8K epch:26.28 loss:0.010 grdn:0.184 lr:9.3e-05 updt_s:0.222 data_s:0.084 +INFO 2025-05-18 23:33:21 ts/train.py:232 step:104K smpl:2M ep:8K epch:26.33 loss:0.009 grdn:0.176 lr:9.3e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-18 23:34:23 ts/train.py:232 step:105K smpl:2M ep:8K epch:26.38 loss:0.009 grdn:0.173 lr:9.3e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-18 23:35:23 ts/train.py:232 step:105K smpl:2M ep:8K epch:26.43 loss:0.010 grdn:0.195 lr:9.3e-05 updt_s:0.222 data_s:0.081 +INFO 2025-05-18 23:36:21 ts/train.py:232 step:105K smpl:2M ep:8K epch:26.48 loss:0.009 grdn:0.172 lr:9.3e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 23:37:16 ts/train.py:232 step:105K smpl:2M ep:8K epch:26.53 loss:0.010 grdn:0.176 lr:9.3e-05 updt_s:0.222 data_s:0.055 +INFO 2025-05-18 23:38:14 ts/train.py:232 step:105K smpl:2M ep:8K epch:26.58 loss:0.009 grdn:0.180 lr:9.3e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-18 23:39:12 ts/train.py:232 step:106K smpl:2M ep:8K epch:26.64 loss:0.010 grdn:0.179 lr:9.3e-05 updt_s:0.222 data_s:0.066 +INFO 2025-05-18 23:40:10 ts/train.py:232 step:106K smpl:2M ep:8K epch:26.69 loss:0.010 grdn:0.178 lr:9.3e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 23:41:08 ts/train.py:232 step:106K smpl:2M ep:8K epch:26.74 loss:0.010 grdn:0.191 lr:9.3e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-18 23:42:06 ts/train.py:232 step:106K smpl:2M ep:8K epch:26.79 loss:0.009 grdn:0.180 lr:9.3e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-18 23:43:03 ts/train.py:232 step:106K smpl:2M ep:8K epch:26.84 loss:0.010 grdn:0.179 lr:9.3e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-18 23:44:01 ts/train.py:232 step:107K smpl:2M ep:8K epch:26.89 loss:0.009 grdn:0.179 lr:9.2e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-18 23:45:00 ts/train.py:232 step:107K smpl:2M ep:8K epch:26.94 loss:0.011 grdn:0.199 lr:9.2e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 23:45:58 ts/train.py:232 step:107K smpl:2M ep:8K epch:26.99 loss:0.011 grdn:0.192 lr:9.2e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-18 23:46:56 ts/train.py:232 step:107K smpl:2M ep:8K epch:27.04 loss:0.009 grdn:0.172 lr:9.2e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-18 23:47:56 ts/train.py:232 step:107K smpl:2M ep:8K epch:27.09 loss:0.009 grdn:0.179 lr:9.2e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-18 23:48:56 ts/train.py:232 step:108K smpl:2M ep:8K epch:27.14 loss:0.010 grdn:0.188 lr:9.2e-05 updt_s:0.222 data_s:0.078 +INFO 2025-05-18 23:49:55 ts/train.py:232 step:108K smpl:2M ep:8K epch:27.19 loss:0.010 grdn:0.176 lr:9.2e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-18 23:50:55 ts/train.py:232 step:108K smpl:2M ep:8K epch:27.24 loss:0.010 grdn:0.185 lr:9.2e-05 updt_s:0.222 data_s:0.078 +INFO 2025-05-18 23:51:55 ts/train.py:232 step:108K smpl:2M ep:8K epch:27.29 loss:0.008 grdn:0.175 lr:9.2e-05 updt_s:0.222 data_s:0.076 +INFO 2025-05-18 23:52:55 ts/train.py:232 step:108K smpl:2M ep:8K epch:27.34 loss:0.010 grdn:0.179 lr:9.2e-05 updt_s:0.222 data_s:0.078 +INFO 2025-05-18 23:53:54 ts/train.py:232 step:109K smpl:2M ep:8K epch:27.39 loss:0.010 grdn:0.185 lr:9.2e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-18 23:54:50 ts/train.py:232 step:109K smpl:2M ep:8K epch:27.44 loss:0.009 grdn:0.171 lr:9.2e-05 updt_s:0.222 data_s:0.059 +INFO 2025-05-18 23:55:47 ts/train.py:232 step:109K smpl:2M ep:8K epch:27.49 loss:0.009 grdn:0.174 lr:9.2e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-18 23:56:44 ts/train.py:232 step:109K smpl:2M ep:8K epch:27.54 loss:0.009 grdn:0.178 lr:9.2e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-18 23:57:41 ts/train.py:232 step:109K smpl:2M ep:8K epch:27.59 loss:0.009 grdn:0.176 lr:9.2e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-18 23:58:37 ts/train.py:232 step:110K smpl:2M ep:8K epch:27.64 loss:0.009 grdn:0.184 lr:9.2e-05 updt_s:0.222 data_s:0.060 +INFO 2025-05-18 23:59:34 ts/train.py:232 step:110K smpl:2M ep:8K epch:27.69 loss:0.009 grdn:0.175 lr:9.2e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-19 00:00:31 ts/train.py:232 step:110K smpl:2M ep:8K epch:27.74 loss:0.009 grdn:0.168 lr:9.2e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-19 00:01:28 ts/train.py:232 step:110K smpl:2M ep:8K epch:27.80 loss:0.009 grdn:0.178 lr:9.2e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-19 00:02:25 ts/train.py:232 step:110K smpl:2M ep:8K epch:27.85 loss:0.010 grdn:0.177 lr:9.2e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-19 00:03:22 ts/train.py:232 step:111K smpl:2M ep:8K epch:27.90 loss:0.009 grdn:0.172 lr:9.2e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-19 00:04:20 ts/train.py:232 step:111K smpl:2M ep:8K epch:27.95 loss:0.009 grdn:0.175 lr:9.2e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 00:05:17 ts/train.py:232 step:111K smpl:2M ep:8K epch:28.00 loss:0.008 grdn:0.174 lr:9.2e-05 updt_s:0.222 data_s:0.066 +INFO 2025-05-19 00:06:18 ts/train.py:232 step:111K smpl:2M ep:8K epch:28.05 loss:0.009 grdn:0.176 lr:9.2e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-19 00:07:23 ts/train.py:232 step:111K smpl:2M ep:8K epch:28.10 loss:0.009 grdn:0.174 lr:9.2e-05 updt_s:0.222 data_s:0.101 +INFO 2025-05-19 00:08:27 ts/train.py:232 step:112K smpl:2M ep:8K epch:28.15 loss:0.010 grdn:0.174 lr:9.2e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 00:09:31 ts/train.py:232 step:112K smpl:2M ep:8K epch:28.20 loss:0.010 grdn:0.185 lr:9.2e-05 updt_s:0.222 data_s:0.099 +INFO 2025-05-19 00:10:34 ts/train.py:232 step:112K smpl:2M ep:8K epch:28.25 loss:0.009 grdn:0.175 lr:9.2e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-19 00:11:38 ts/train.py:232 step:112K smpl:2M ep:8K epch:28.30 loss:0.010 grdn:0.186 lr:9.2e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-19 00:12:40 ts/train.py:232 step:112K smpl:2M ep:9K epch:28.35 loss:0.009 grdn:0.178 lr:9.2e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 00:13:38 ts/train.py:232 step:113K smpl:2M ep:9K epch:28.40 loss:0.009 grdn:0.180 lr:9.2e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-19 00:14:36 ts/train.py:232 step:113K smpl:2M ep:9K epch:28.45 loss:0.010 grdn:0.169 lr:9.2e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 00:15:34 ts/train.py:232 step:113K smpl:2M ep:9K epch:28.50 loss:0.010 grdn:0.179 lr:9.2e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-19 00:16:31 ts/train.py:232 step:113K smpl:2M ep:9K epch:28.55 loss:0.009 grdn:0.177 lr:9.2e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-19 00:17:29 ts/train.py:232 step:113K smpl:2M ep:9K epch:28.60 loss:0.009 grdn:0.179 lr:9.2e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-19 00:18:27 ts/train.py:232 step:114K smpl:2M ep:9K epch:28.65 loss:0.009 grdn:0.173 lr:9.1e-05 updt_s:0.222 data_s:0.066 +INFO 2025-05-19 00:19:25 ts/train.py:232 step:114K smpl:2M ep:9K epch:28.70 loss:0.010 grdn:0.182 lr:9.1e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-19 00:20:24 ts/train.py:232 step:114K smpl:2M ep:9K epch:28.75 loss:0.010 grdn:0.186 lr:9.1e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-19 00:21:22 ts/train.py:232 step:114K smpl:2M ep:9K epch:28.80 loss:0.010 grdn:0.179 lr:9.1e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-19 00:22:19 ts/train.py:232 step:114K smpl:2M ep:9K epch:28.85 loss:0.009 grdn:0.180 lr:9.1e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-19 00:23:16 ts/train.py:232 step:115K smpl:2M ep:9K epch:28.91 loss:0.009 grdn:0.177 lr:9.1e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-19 00:24:15 ts/train.py:232 step:115K smpl:2M ep:9K epch:28.96 loss:0.009 grdn:0.171 lr:9.1e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-19 00:25:12 ts/train.py:232 step:115K smpl:2M ep:9K epch:29.01 loss:0.010 grdn:0.190 lr:9.1e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-19 00:26:15 ts/train.py:232 step:115K smpl:2M ep:9K epch:29.06 loss:0.009 grdn:0.175 lr:9.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 00:27:18 ts/train.py:232 step:115K smpl:2M ep:9K epch:29.11 loss:0.009 grdn:0.174 lr:9.1e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 00:28:21 ts/train.py:232 step:116K smpl:2M ep:9K epch:29.16 loss:0.009 grdn:0.182 lr:9.1e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 00:29:22 ts/train.py:232 step:116K smpl:2M ep:9K epch:29.21 loss:0.009 grdn:0.168 lr:9.1e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 00:30:26 ts/train.py:232 step:116K smpl:2M ep:9K epch:29.26 loss:0.009 grdn:0.179 lr:9.1e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 00:31:28 ts/train.py:232 step:116K smpl:2M ep:9K epch:29.31 loss:0.010 grdn:0.183 lr:9.1e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 00:32:25 ts/train.py:232 step:116K smpl:2M ep:9K epch:29.36 loss:0.009 grdn:0.178 lr:9.1e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-19 00:33:23 ts/train.py:232 step:117K smpl:2M ep:9K epch:29.41 loss:0.009 grdn:0.171 lr:9.1e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 00:34:19 ts/train.py:232 step:117K smpl:2M ep:9K epch:29.46 loss:0.010 grdn:0.185 lr:9.1e-05 updt_s:0.222 data_s:0.058 +INFO 2025-05-19 00:35:16 ts/train.py:232 step:117K smpl:2M ep:9K epch:29.51 loss:0.010 grdn:0.184 lr:9.1e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-19 00:36:13 ts/train.py:232 step:117K smpl:2M ep:9K epch:29.56 loss:0.009 grdn:0.176 lr:9.1e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 00:37:10 ts/train.py:232 step:117K smpl:2M ep:9K epch:29.61 loss:0.009 grdn:0.182 lr:9.1e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-19 00:38:08 ts/train.py:232 step:118K smpl:2M ep:9K epch:29.66 loss:0.009 grdn:0.178 lr:9.1e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-19 00:39:06 ts/train.py:232 step:118K smpl:2M ep:9K epch:29.71 loss:0.010 grdn:0.184 lr:9.1e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 00:40:03 ts/train.py:232 step:118K smpl:2M ep:9K epch:29.76 loss:0.009 grdn:0.172 lr:9.1e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 00:41:01 ts/train.py:232 step:118K smpl:2M ep:9K epch:29.81 loss:0.009 grdn:0.180 lr:9.1e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-19 00:41:58 ts/train.py:232 step:118K smpl:2M ep:9K epch:29.86 loss:0.009 grdn:0.169 lr:9.1e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 00:42:56 ts/train.py:232 step:119K smpl:2M ep:9K epch:29.91 loss:0.009 grdn:0.171 lr:9.1e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-19 00:43:53 ts/train.py:232 step:119K smpl:2M ep:9K epch:29.96 loss:0.009 grdn:0.169 lr:9.1e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-19 00:44:57 ts/train.py:232 step:119K smpl:2M ep:9K epch:30.01 loss:0.008 grdn:0.169 lr:9.1e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 00:45:59 ts/train.py:232 step:119K smpl:2M ep:9K epch:30.07 loss:0.009 grdn:0.179 lr:9.1e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 00:47:03 ts/train.py:232 step:119K smpl:2M ep:9K epch:30.12 loss:0.010 grdn:0.181 lr:9.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 00:48:07 ts/train.py:232 step:120K smpl:2M ep:9K epch:30.17 loss:0.009 grdn:0.187 lr:9.1e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 00:49:10 ts/train.py:232 step:120K smpl:2M ep:9K epch:30.22 loss:0.008 grdn:0.161 lr:9.1e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 00:50:13 ts/train.py:232 step:120K smpl:2M ep:9K epch:30.27 loss:0.009 grdn:0.174 lr:9.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 00:50:13 ts/train.py:241 Checkpoint policy after step 120000 +INFO 2025-05-19 00:51:17 ts/train.py:232 step:120K smpl:2M ep:9K epch:30.32 loss:0.008 grdn:0.162 lr:9.0e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-19 00:52:15 ts/train.py:232 step:120K smpl:2M ep:9K epch:30.37 loss:0.009 grdn:0.184 lr:9.0e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-19 00:53:13 ts/train.py:232 step:121K smpl:2M ep:9K epch:30.42 loss:0.009 grdn:0.174 lr:9.0e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-19 00:54:10 ts/train.py:232 step:121K smpl:2M ep:9K epch:30.47 loss:0.009 grdn:0.175 lr:9.0e-05 updt_s:0.222 data_s:0.059 +INFO 2025-05-19 00:55:07 ts/train.py:232 step:121K smpl:2M ep:9K epch:30.52 loss:0.009 grdn:0.169 lr:9.0e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 00:56:04 ts/train.py:232 step:121K smpl:2M ep:9K epch:30.57 loss:0.009 grdn:0.181 lr:9.0e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-19 00:57:02 ts/train.py:232 step:121K smpl:2M ep:9K epch:30.62 loss:0.009 grdn:0.173 lr:9.0e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 00:57:58 ts/train.py:232 step:122K smpl:2M ep:9K epch:30.67 loss:0.009 grdn:0.170 lr:9.0e-05 updt_s:0.222 data_s:0.058 +INFO 2025-05-19 00:58:54 ts/train.py:232 step:122K smpl:2M ep:9K epch:30.72 loss:0.009 grdn:0.179 lr:9.0e-05 updt_s:0.222 data_s:0.060 +INFO 2025-05-19 00:59:52 ts/train.py:232 step:122K smpl:2M ep:9K epch:30.77 loss:0.009 grdn:0.179 lr:9.0e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-19 01:00:48 ts/train.py:232 step:122K smpl:2M ep:9K epch:30.82 loss:0.009 grdn:0.179 lr:9.0e-05 updt_s:0.222 data_s:0.060 +INFO 2025-05-19 01:01:46 ts/train.py:232 step:122K smpl:2M ep:9K epch:30.87 loss:0.009 grdn:0.169 lr:9.0e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-19 01:02:42 ts/train.py:232 step:123K smpl:2M ep:9K epch:30.92 loss:0.009 grdn:0.181 lr:9.0e-05 updt_s:0.222 data_s:0.061 +INFO 2025-05-19 01:03:44 ts/train.py:232 step:123K smpl:2M ep:9K epch:30.97 loss:0.008 grdn:0.176 lr:9.0e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 01:04:47 ts/train.py:232 step:123K smpl:2M ep:9K epch:31.02 loss:0.009 grdn:0.187 lr:9.0e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-19 01:05:49 ts/train.py:232 step:123K smpl:2M ep:9K epch:31.07 loss:0.010 grdn:0.183 lr:9.0e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-19 01:06:52 ts/train.py:232 step:123K smpl:2M ep:9K epch:31.12 loss:0.009 grdn:0.188 lr:9.0e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 01:07:54 ts/train.py:232 step:124K smpl:2M ep:9K epch:31.18 loss:0.009 grdn:0.174 lr:9.0e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 01:08:57 ts/train.py:232 step:124K smpl:2M ep:9K epch:31.23 loss:0.009 grdn:0.168 lr:9.0e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 01:09:58 ts/train.py:232 step:124K smpl:2M ep:9K epch:31.28 loss:0.009 grdn:0.179 lr:9.0e-05 updt_s:0.222 data_s:0.084 +INFO 2025-05-19 01:10:57 ts/train.py:232 step:124K smpl:2M ep:9K epch:31.33 loss:0.010 grdn:0.187 lr:9.0e-05 updt_s:0.222 data_s:0.069 +INFO 2025-05-19 01:11:55 ts/train.py:232 step:124K smpl:2M ep:9K epch:31.38 loss:0.010 grdn:0.183 lr:9.0e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-19 01:12:54 ts/train.py:232 step:125K smpl:2M ep:9K epch:31.43 loss:0.008 grdn:0.169 lr:9.0e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-19 01:13:53 ts/train.py:232 step:125K smpl:2M ep:9K epch:31.48 loss:0.009 grdn:0.178 lr:9.0e-05 updt_s:0.222 data_s:0.073 +INFO 2025-05-19 01:14:52 ts/train.py:232 step:125K smpl:2M ep:9K epch:31.53 loss:0.010 grdn:0.188 lr:9.0e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-19 01:15:51 ts/train.py:232 step:125K smpl:2M ep:9K epch:31.58 loss:0.008 grdn:0.159 lr:9.0e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-19 01:16:50 ts/train.py:232 step:125K smpl:2M ep:9K epch:31.63 loss:0.009 grdn:0.176 lr:9.0e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 01:17:49 ts/train.py:232 step:126K smpl:2M ep:10K epch:31.68 loss:0.009 grdn:0.174 lr:9.0e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-19 01:18:47 ts/train.py:232 step:126K smpl:2M ep:10K epch:31.73 loss:0.009 grdn:0.178 lr:9.0e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 01:19:46 ts/train.py:232 step:126K smpl:2M ep:10K epch:31.78 loss:0.009 grdn:0.178 lr:9.0e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 01:20:44 ts/train.py:232 step:126K smpl:2M ep:10K epch:31.83 loss:0.010 grdn:0.189 lr:9.0e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 01:21:43 ts/train.py:232 step:126K smpl:2M ep:10K epch:31.88 loss:0.008 grdn:0.166 lr:9.0e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 01:22:43 ts/train.py:232 step:127K smpl:2M ep:10K epch:31.93 loss:0.009 grdn:0.171 lr:8.9e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 01:23:44 ts/train.py:232 step:127K smpl:2M ep:10K epch:31.98 loss:0.010 grdn:0.181 lr:8.9e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 01:24:44 ts/train.py:232 step:127K smpl:2M ep:10K epch:32.03 loss:0.009 grdn:0.173 lr:8.9e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 01:25:45 ts/train.py:232 step:127K smpl:2M ep:10K epch:32.08 loss:0.009 grdn:0.174 lr:8.9e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 01:26:45 ts/train.py:232 step:127K smpl:2M ep:10K epch:32.13 loss:0.009 grdn:0.179 lr:8.9e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 01:27:46 ts/train.py:232 step:128K smpl:2M ep:10K epch:32.18 loss:0.009 grdn:0.172 lr:8.9e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 01:28:46 ts/train.py:232 step:128K smpl:2M ep:10K epch:32.23 loss:0.009 grdn:0.164 lr:8.9e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 01:29:45 ts/train.py:232 step:128K smpl:2M ep:10K epch:32.29 loss:0.009 grdn:0.172 lr:8.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 01:30:45 ts/train.py:232 step:128K smpl:2M ep:10K epch:32.34 loss:0.009 grdn:0.186 lr:8.9e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 01:31:44 ts/train.py:232 step:128K smpl:2M ep:10K epch:32.39 loss:0.009 grdn:0.183 lr:8.9e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-19 01:32:43 ts/train.py:232 step:129K smpl:2M ep:10K epch:32.44 loss:0.009 grdn:0.180 lr:8.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 01:33:43 ts/train.py:232 step:129K smpl:2M ep:10K epch:32.49 loss:0.008 grdn:0.162 lr:8.9e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 01:34:42 ts/train.py:232 step:129K smpl:2M ep:10K epch:32.54 loss:0.009 grdn:0.171 lr:8.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 01:35:41 ts/train.py:232 step:129K smpl:2M ep:10K epch:32.59 loss:0.009 grdn:0.181 lr:8.9e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 01:36:41 ts/train.py:232 step:129K smpl:2M ep:10K epch:32.64 loss:0.008 grdn:0.167 lr:8.9e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 01:37:40 ts/train.py:232 step:130K smpl:2M ep:10K epch:32.69 loss:0.009 grdn:0.173 lr:8.9e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 01:38:39 ts/train.py:232 step:130K smpl:2M ep:10K epch:32.74 loss:0.009 grdn:0.177 lr:8.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 01:39:38 ts/train.py:232 step:130K smpl:2M ep:10K epch:32.79 loss:0.009 grdn:0.187 lr:8.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 01:40:37 ts/train.py:232 step:130K smpl:2M ep:10K epch:32.84 loss:0.008 grdn:0.161 lr:8.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 01:41:39 ts/train.py:232 step:130K smpl:2M ep:10K epch:32.89 loss:0.008 grdn:0.161 lr:8.9e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 01:42:42 ts/train.py:232 step:131K smpl:2M ep:10K epch:32.94 loss:0.009 grdn:0.174 lr:8.9e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 01:43:46 ts/train.py:232 step:131K smpl:2M ep:10K epch:32.99 loss:0.009 grdn:0.173 lr:8.9e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 01:44:48 ts/train.py:232 step:131K smpl:2M ep:10K epch:33.04 loss:0.009 grdn:0.179 lr:8.9e-05 updt_s:0.222 data_s:0.088 +INFO 2025-05-19 01:45:50 ts/train.py:232 step:131K smpl:2M ep:10K epch:33.09 loss:0.009 grdn:0.174 lr:8.9e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 01:46:55 ts/train.py:232 step:131K smpl:2M ep:10K epch:33.14 loss:0.009 grdn:0.178 lr:8.9e-05 updt_s:0.222 data_s:0.101 +INFO 2025-05-19 01:48:00 ts/train.py:232 step:132K smpl:2M ep:10K epch:33.19 loss:0.009 grdn:0.168 lr:8.9e-05 updt_s:0.222 data_s:0.100 +INFO 2025-05-19 01:49:00 ts/train.py:232 step:132K smpl:2M ep:10K epch:33.24 loss:0.008 grdn:0.168 lr:8.9e-05 updt_s:0.222 data_s:0.078 +INFO 2025-05-19 01:50:00 ts/train.py:232 step:132K smpl:2M ep:10K epch:33.29 loss:0.009 grdn:0.181 lr:8.9e-05 updt_s:0.222 data_s:0.080 +INFO 2025-05-19 01:51:01 ts/train.py:232 step:132K smpl:2M ep:10K epch:33.34 loss:0.009 grdn:0.174 lr:8.9e-05 updt_s:0.222 data_s:0.080 +INFO 2025-05-19 01:52:01 ts/train.py:232 step:132K smpl:2M ep:10K epch:33.39 loss:0.010 grdn:0.192 lr:8.9e-05 updt_s:0.222 data_s:0.079 +INFO 2025-05-19 01:53:00 ts/train.py:232 step:133K smpl:2M ep:10K epch:33.45 loss:0.009 grdn:0.180 lr:8.9e-05 updt_s:0.222 data_s:0.075 +INFO 2025-05-19 01:54:00 ts/train.py:232 step:133K smpl:2M ep:10K epch:33.50 loss:0.008 grdn:0.173 lr:8.8e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 01:55:00 ts/train.py:232 step:133K smpl:2M ep:10K epch:33.55 loss:0.009 grdn:0.169 lr:8.8e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 01:55:59 ts/train.py:232 step:133K smpl:2M ep:10K epch:33.60 loss:0.009 grdn:0.176 lr:8.8e-05 updt_s:0.222 data_s:0.075 +INFO 2025-05-19 01:56:59 ts/train.py:232 step:133K smpl:2M ep:10K epch:33.65 loss:0.009 grdn:0.179 lr:8.8e-05 updt_s:0.222 data_s:0.075 +INFO 2025-05-19 01:57:59 ts/train.py:232 step:134K smpl:2M ep:10K epch:33.70 loss:0.009 grdn:0.191 lr:8.8e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 01:58:58 ts/train.py:232 step:134K smpl:2M ep:10K epch:33.75 loss:0.008 grdn:0.169 lr:8.8e-05 updt_s:0.222 data_s:0.075 +INFO 2025-05-19 01:59:58 ts/train.py:232 step:134K smpl:2M ep:10K epch:33.80 loss:0.008 grdn:0.173 lr:8.8e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-19 02:00:59 ts/train.py:232 step:134K smpl:2M ep:10K epch:33.85 loss:0.008 grdn:0.175 lr:8.8e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 02:01:55 ts/train.py:232 step:134K smpl:2M ep:10K epch:33.90 loss:0.008 grdn:0.165 lr:8.8e-05 updt_s:0.221 data_s:0.060 +INFO 2025-05-19 02:02:54 ts/train.py:232 step:135K smpl:2M ep:10K epch:33.95 loss:0.009 grdn:0.173 lr:8.8e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 02:03:50 ts/train.py:232 step:135K smpl:2M ep:10K epch:34.00 loss:0.008 grdn:0.160 lr:8.8e-05 updt_s:0.221 data_s:0.059 +INFO 2025-05-19 02:04:43 ts/train.py:232 step:135K smpl:2M ep:10K epch:34.05 loss:0.008 grdn:0.173 lr:8.8e-05 updt_s:0.222 data_s:0.045 +INFO 2025-05-19 02:05:41 ts/train.py:232 step:135K smpl:2M ep:10K epch:34.10 loss:0.009 grdn:0.173 lr:8.8e-05 updt_s:0.222 data_s:0.063 +INFO 2025-05-19 02:06:39 ts/train.py:232 step:135K smpl:2M ep:10K epch:34.15 loss:0.008 grdn:0.181 lr:8.8e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 02:07:40 ts/train.py:232 step:136K smpl:2M ep:10K epch:34.20 loss:0.009 grdn:0.174 lr:8.8e-05 updt_s:0.222 data_s:0.083 +INFO 2025-05-19 02:08:40 ts/train.py:232 step:136K smpl:2M ep:10K epch:34.25 loss:0.008 grdn:0.174 lr:8.8e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 02:09:39 ts/train.py:232 step:136K smpl:2M ep:10K epch:34.30 loss:0.009 grdn:0.181 lr:8.8e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 02:10:38 ts/train.py:232 step:136K smpl:2M ep:10K epch:34.35 loss:0.008 grdn:0.163 lr:8.8e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 02:11:38 ts/train.py:232 step:136K smpl:2M ep:10K epch:34.40 loss:0.008 grdn:0.171 lr:8.8e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 02:12:36 ts/train.py:232 step:137K smpl:2M ep:10K epch:34.45 loss:0.009 grdn:0.174 lr:8.8e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 02:13:36 ts/train.py:232 step:137K smpl:2M ep:10K epch:34.50 loss:0.009 grdn:0.171 lr:8.8e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 02:14:36 ts/train.py:232 step:137K smpl:2M ep:10K epch:34.56 loss:0.008 grdn:0.169 lr:8.8e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 02:15:35 ts/train.py:232 step:137K smpl:2M ep:10K epch:34.61 loss:0.009 grdn:0.170 lr:8.8e-05 updt_s:0.222 data_s:0.075 +INFO 2025-05-19 02:16:35 ts/train.py:232 step:137K smpl:2M ep:10K epch:34.66 loss:0.009 grdn:0.179 lr:8.8e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 02:17:36 ts/train.py:232 step:138K smpl:2M ep:10K epch:34.71 loss:0.008 grdn:0.174 lr:8.8e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-19 02:18:34 ts/train.py:232 step:138K smpl:2M ep:10K epch:34.76 loss:0.009 grdn:0.171 lr:8.8e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 02:19:33 ts/train.py:232 step:138K smpl:2M ep:10K epch:34.81 loss:0.008 grdn:0.173 lr:8.8e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 02:20:39 ts/train.py:232 step:138K smpl:2M ep:10K epch:34.86 loss:0.009 grdn:0.174 lr:8.8e-05 updt_s:0.221 data_s:0.110 +INFO 2025-05-19 02:21:44 ts/train.py:232 step:138K smpl:2M ep:10K epch:34.91 loss:0.009 grdn:0.171 lr:8.8e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 02:22:48 ts/train.py:232 step:139K smpl:2M ep:10K epch:34.96 loss:0.009 grdn:0.172 lr:8.7e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 02:23:52 ts/train.py:232 step:139K smpl:2M ep:11K epch:35.01 loss:0.008 grdn:0.162 lr:8.7e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 02:24:56 ts/train.py:232 step:139K smpl:2M ep:11K epch:35.06 loss:0.008 grdn:0.167 lr:8.7e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 02:26:00 ts/train.py:232 step:139K smpl:2M ep:11K epch:35.11 loss:0.009 grdn:0.184 lr:8.7e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 02:27:00 ts/train.py:232 step:139K smpl:2M ep:11K epch:35.16 loss:0.008 grdn:0.167 lr:8.7e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 02:27:58 ts/train.py:232 step:140K smpl:2M ep:11K epch:35.21 loss:0.008 grdn:0.174 lr:8.7e-05 updt_s:0.221 data_s:0.068 +INFO 2025-05-19 02:28:56 ts/train.py:232 step:140K smpl:2M ep:11K epch:35.26 loss:0.008 grdn:0.172 lr:8.7e-05 updt_s:0.222 data_s:0.066 +INFO 2025-05-19 02:29:54 ts/train.py:232 step:140K smpl:2M ep:11K epch:35.31 loss:0.008 grdn:0.170 lr:8.7e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 02:29:54 ts/train.py:241 Checkpoint policy after step 140000 +INFO 2025-05-19 02:30:56 ts/train.py:232 step:140K smpl:2M ep:11K epch:35.36 loss:0.008 grdn:0.167 lr:8.7e-05 updt_s:0.221 data_s:0.064 +INFO 2025-05-19 02:31:53 ts/train.py:232 step:140K smpl:2M ep:11K epch:35.41 loss:0.008 grdn:0.164 lr:8.7e-05 updt_s:0.221 data_s:0.066 +INFO 2025-05-19 02:32:52 ts/train.py:232 step:141K smpl:2M ep:11K epch:35.46 loss:0.009 grdn:0.176 lr:8.7e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 02:33:50 ts/train.py:232 step:141K smpl:2M ep:11K epch:35.51 loss:0.008 grdn:0.170 lr:8.7e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 02:34:49 ts/train.py:232 step:141K smpl:2M ep:11K epch:35.56 loss:0.009 grdn:0.182 lr:8.7e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 02:35:48 ts/train.py:232 step:141K smpl:2M ep:11K epch:35.61 loss:0.008 grdn:0.165 lr:8.7e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 02:36:47 ts/train.py:232 step:141K smpl:2M ep:11K epch:35.66 loss:0.008 grdn:0.165 lr:8.7e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 02:37:46 ts/train.py:232 step:142K smpl:2M ep:11K epch:35.72 loss:0.009 grdn:0.174 lr:8.7e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 02:38:46 ts/train.py:232 step:142K smpl:2M ep:11K epch:35.77 loss:0.008 grdn:0.166 lr:8.7e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 02:39:50 ts/train.py:232 step:142K smpl:2M ep:11K epch:35.82 loss:0.009 grdn:0.185 lr:8.7e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 02:40:54 ts/train.py:232 step:142K smpl:2M ep:11K epch:35.87 loss:0.008 grdn:0.172 lr:8.7e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-19 02:41:57 ts/train.py:232 step:142K smpl:2M ep:11K epch:35.92 loss:0.008 grdn:0.163 lr:8.7e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 02:43:00 ts/train.py:232 step:143K smpl:2M ep:11K epch:35.97 loss:0.009 grdn:0.177 lr:8.7e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 02:44:04 ts/train.py:232 step:143K smpl:2M ep:11K epch:36.02 loss:0.008 grdn:0.164 lr:8.7e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 02:45:08 ts/train.py:232 step:143K smpl:2M ep:11K epch:36.07 loss:0.008 grdn:0.178 lr:8.7e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 02:46:08 ts/train.py:232 step:143K smpl:2M ep:11K epch:36.12 loss:0.008 grdn:0.165 lr:8.7e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 02:47:07 ts/train.py:232 step:143K smpl:2M ep:11K epch:36.17 loss:0.009 grdn:0.176 lr:8.7e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-19 02:48:06 ts/train.py:232 step:144K smpl:2M ep:11K epch:36.22 loss:0.008 grdn:0.169 lr:8.7e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-19 02:49:05 ts/train.py:232 step:144K smpl:2M ep:11K epch:36.27 loss:0.009 grdn:0.169 lr:8.7e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-19 02:50:04 ts/train.py:232 step:144K smpl:2M ep:11K epch:36.32 loss:0.009 grdn:0.177 lr:8.7e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 02:51:02 ts/train.py:232 step:144K smpl:2M ep:11K epch:36.37 loss:0.008 grdn:0.168 lr:8.6e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-19 02:52:00 ts/train.py:232 step:144K smpl:2M ep:11K epch:36.42 loss:0.008 grdn:0.171 lr:8.6e-05 updt_s:0.221 data_s:0.067 +INFO 2025-05-19 02:52:58 ts/train.py:232 step:145K smpl:2M ep:11K epch:36.47 loss:0.009 grdn:0.175 lr:8.6e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-19 02:53:57 ts/train.py:232 step:145K smpl:2M ep:11K epch:36.52 loss:0.008 grdn:0.174 lr:8.6e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 02:54:55 ts/train.py:232 step:145K smpl:2M ep:11K epch:36.57 loss:0.008 grdn:0.179 lr:8.6e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-19 02:55:53 ts/train.py:232 step:145K smpl:2M ep:11K epch:36.62 loss:0.008 grdn:0.177 lr:8.6e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 02:56:51 ts/train.py:232 step:145K smpl:2M ep:11K epch:36.67 loss:0.008 grdn:0.170 lr:8.6e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-19 02:57:49 ts/train.py:232 step:146K smpl:2M ep:11K epch:36.72 loss:0.008 grdn:0.172 lr:8.6e-05 updt_s:0.221 data_s:0.068 +INFO 2025-05-19 02:58:51 ts/train.py:232 step:146K smpl:2M ep:11K epch:36.77 loss:0.008 grdn:0.169 lr:8.6e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 02:59:54 ts/train.py:232 step:146K smpl:2M ep:11K epch:36.83 loss:0.008 grdn:0.166 lr:8.6e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 03:00:57 ts/train.py:232 step:146K smpl:2M ep:11K epch:36.88 loss:0.008 grdn:0.171 lr:8.6e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 03:01:58 ts/train.py:232 step:146K smpl:2M ep:11K epch:36.93 loss:0.009 grdn:0.173 lr:8.6e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 03:03:01 ts/train.py:232 step:147K smpl:2M ep:11K epch:36.98 loss:0.008 grdn:0.158 lr:8.6e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 03:04:04 ts/train.py:232 step:147K smpl:2M ep:11K epch:37.03 loss:0.008 grdn:0.179 lr:8.6e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 03:05:07 ts/train.py:232 step:147K smpl:2M ep:11K epch:37.08 loss:0.009 grdn:0.177 lr:8.6e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 03:06:08 ts/train.py:232 step:147K smpl:2M ep:11K epch:37.13 loss:0.009 grdn:0.173 lr:8.6e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 03:07:10 ts/train.py:232 step:147K smpl:2M ep:11K epch:37.18 loss:0.007 grdn:0.162 lr:8.6e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 03:08:15 ts/train.py:232 step:148K smpl:2M ep:11K epch:37.23 loss:0.008 grdn:0.174 lr:8.6e-05 updt_s:0.222 data_s:0.099 +INFO 2025-05-19 03:09:18 ts/train.py:232 step:148K smpl:2M ep:11K epch:37.28 loss:0.008 grdn:0.164 lr:8.6e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-19 03:10:22 ts/train.py:232 step:148K smpl:2M ep:11K epch:37.33 loss:0.008 grdn:0.178 lr:8.6e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 03:11:25 ts/train.py:232 step:148K smpl:2M ep:11K epch:37.38 loss:0.009 grdn:0.163 lr:8.6e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 03:12:27 ts/train.py:232 step:148K smpl:2M ep:11K epch:37.43 loss:0.007 grdn:0.163 lr:8.6e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 03:13:29 ts/train.py:232 step:149K smpl:2M ep:11K epch:37.48 loss:0.009 grdn:0.176 lr:8.6e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-19 03:14:31 ts/train.py:232 step:149K smpl:2M ep:11K epch:37.53 loss:0.009 grdn:0.174 lr:8.6e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 03:15:35 ts/train.py:232 step:149K smpl:2M ep:11K epch:37.58 loss:0.008 grdn:0.171 lr:8.6e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-19 03:16:37 ts/train.py:232 step:149K smpl:2M ep:11K epch:37.63 loss:0.009 grdn:0.173 lr:8.6e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 03:17:40 ts/train.py:232 step:149K smpl:2M ep:11K epch:37.68 loss:0.008 grdn:0.170 lr:8.6e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 03:18:46 ts/train.py:232 step:150K smpl:2M ep:11K epch:37.73 loss:0.008 grdn:0.166 lr:8.6e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 03:19:50 ts/train.py:232 step:150K smpl:2M ep:11K epch:37.78 loss:0.007 grdn:0.157 lr:8.5e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 03:20:55 ts/train.py:232 step:150K smpl:2M ep:11K epch:37.83 loss:0.008 grdn:0.176 lr:8.5e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 03:21:59 ts/train.py:232 step:150K smpl:2M ep:11K epch:37.88 loss:0.009 grdn:0.197 lr:8.5e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 03:23:03 ts/train.py:232 step:150K smpl:2M ep:11K epch:37.93 loss:0.008 grdn:0.172 lr:8.5e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 03:24:06 ts/train.py:232 step:151K smpl:2M ep:11K epch:37.99 loss:0.009 grdn:0.174 lr:8.5e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 03:25:08 ts/train.py:232 step:151K smpl:2M ep:11K epch:38.04 loss:0.008 grdn:0.171 lr:8.5e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 03:26:10 ts/train.py:232 step:151K smpl:2M ep:11K epch:38.09 loss:0.008 grdn:0.171 lr:8.5e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 03:27:13 ts/train.py:232 step:151K smpl:2M ep:11K epch:38.14 loss:0.007 grdn:0.169 lr:8.5e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 03:28:16 ts/train.py:232 step:151K smpl:2M ep:11K epch:38.19 loss:0.009 grdn:0.181 lr:8.5e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-19 03:29:19 ts/train.py:232 step:152K smpl:2M ep:11K epch:38.24 loss:0.008 grdn:0.178 lr:8.5e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-19 03:30:22 ts/train.py:232 step:152K smpl:2M ep:11K epch:38.29 loss:0.008 grdn:0.164 lr:8.5e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 03:31:25 ts/train.py:232 step:152K smpl:2M ep:12K epch:38.34 loss:0.008 grdn:0.174 lr:8.5e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 03:32:28 ts/train.py:232 step:152K smpl:2M ep:12K epch:38.39 loss:0.009 grdn:0.172 lr:8.5e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 03:33:32 ts/train.py:232 step:152K smpl:2M ep:12K epch:38.44 loss:0.009 grdn:0.172 lr:8.5e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 03:34:35 ts/train.py:232 step:153K smpl:2M ep:12K epch:38.49 loss:0.008 grdn:0.171 lr:8.5e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 03:35:39 ts/train.py:232 step:153K smpl:2M ep:12K epch:38.54 loss:0.008 grdn:0.173 lr:8.5e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 03:36:42 ts/train.py:232 step:153K smpl:2M ep:12K epch:38.59 loss:0.008 grdn:0.172 lr:8.5e-05 updt_s:0.222 data_s:0.096 +INFO 2025-05-19 03:37:46 ts/train.py:232 step:153K smpl:2M ep:12K epch:38.64 loss:0.008 grdn:0.173 lr:8.5e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 03:38:50 ts/train.py:232 step:153K smpl:2M ep:12K epch:38.69 loss:0.009 grdn:0.174 lr:8.5e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 03:39:53 ts/train.py:232 step:154K smpl:2M ep:12K epch:38.74 loss:0.008 grdn:0.170 lr:8.5e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-19 03:40:56 ts/train.py:232 step:154K smpl:2M ep:12K epch:38.79 loss:0.009 grdn:0.179 lr:8.5e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 03:42:00 ts/train.py:232 step:154K smpl:2M ep:12K epch:38.84 loss:0.008 grdn:0.170 lr:8.5e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 03:43:02 ts/train.py:232 step:154K smpl:2M ep:12K epch:38.89 loss:0.009 grdn:0.177 lr:8.5e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 03:44:05 ts/train.py:232 step:154K smpl:2M ep:12K epch:38.94 loss:0.008 grdn:0.169 lr:8.5e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 03:45:08 ts/train.py:232 step:155K smpl:2M ep:12K epch:38.99 loss:0.008 grdn:0.172 lr:8.5e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 03:46:11 ts/train.py:232 step:155K smpl:2M ep:12K epch:39.04 loss:0.007 grdn:0.162 lr:8.5e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 03:47:14 ts/train.py:232 step:155K smpl:2M ep:12K epch:39.10 loss:0.009 grdn:0.183 lr:8.5e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 03:48:17 ts/train.py:232 step:155K smpl:2M ep:12K epch:39.15 loss:0.008 grdn:0.165 lr:8.4e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 03:49:21 ts/train.py:232 step:155K smpl:2M ep:12K epch:39.20 loss:0.008 grdn:0.161 lr:8.4e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 03:50:24 ts/train.py:232 step:156K smpl:2M ep:12K epch:39.25 loss:0.008 grdn:0.172 lr:8.4e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 03:51:28 ts/train.py:232 step:156K smpl:2M ep:12K epch:39.30 loss:0.009 grdn:0.169 lr:8.4e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 03:52:31 ts/train.py:232 step:156K smpl:2M ep:12K epch:39.35 loss:0.008 grdn:0.165 lr:8.4e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 03:53:34 ts/train.py:232 step:156K smpl:2M ep:12K epch:39.40 loss:0.008 grdn:0.168 lr:8.4e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 03:54:38 ts/train.py:232 step:156K smpl:3M ep:12K epch:39.45 loss:0.008 grdn:0.174 lr:8.4e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 03:55:40 ts/train.py:232 step:157K smpl:3M ep:12K epch:39.50 loss:0.008 grdn:0.167 lr:8.4e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 03:56:43 ts/train.py:232 step:157K smpl:3M ep:12K epch:39.55 loss:0.009 grdn:0.167 lr:8.4e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 03:57:47 ts/train.py:232 step:157K smpl:3M ep:12K epch:39.60 loss:0.007 grdn:0.156 lr:8.4e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 03:58:52 ts/train.py:232 step:157K smpl:3M ep:12K epch:39.65 loss:0.008 grdn:0.172 lr:8.4e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 03:59:56 ts/train.py:232 step:157K smpl:3M ep:12K epch:39.70 loss:0.009 grdn:0.177 lr:8.4e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 04:00:59 ts/train.py:232 step:158K smpl:3M ep:12K epch:39.75 loss:0.008 grdn:0.178 lr:8.4e-05 updt_s:0.222 data_s:0.095 +INFO 2025-05-19 04:02:03 ts/train.py:232 step:158K smpl:3M ep:12K epch:39.80 loss:0.008 grdn:0.165 lr:8.4e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 04:03:06 ts/train.py:232 step:158K smpl:3M ep:12K epch:39.85 loss:0.008 grdn:0.164 lr:8.4e-05 updt_s:0.222 data_s:0.090 +INFO 2025-05-19 04:04:09 ts/train.py:232 step:158K smpl:3M ep:12K epch:39.90 loss:0.008 grdn:0.176 lr:8.4e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 04:05:13 ts/train.py:232 step:158K smpl:3M ep:12K epch:39.95 loss:0.008 grdn:0.177 lr:8.4e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 04:06:16 ts/train.py:232 step:159K smpl:3M ep:12K epch:40.00 loss:0.008 grdn:0.165 lr:8.4e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 04:07:20 ts/train.py:232 step:159K smpl:3M ep:12K epch:40.05 loss:0.008 grdn:0.168 lr:8.4e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 04:08:23 ts/train.py:232 step:159K smpl:3M ep:12K epch:40.10 loss:0.008 grdn:0.170 lr:8.4e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 04:09:26 ts/train.py:232 step:159K smpl:3M ep:12K epch:40.15 loss:0.008 grdn:0.163 lr:8.4e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 04:10:28 ts/train.py:232 step:159K smpl:3M ep:12K epch:40.20 loss:0.009 grdn:0.173 lr:8.4e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 04:11:31 ts/train.py:232 step:160K smpl:3M ep:12K epch:40.26 loss:0.008 grdn:0.173 lr:8.4e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 04:12:34 ts/train.py:232 step:160K smpl:3M ep:12K epch:40.31 loss:0.008 grdn:0.171 lr:8.4e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 04:13:36 ts/train.py:232 step:160K smpl:3M ep:12K epch:40.36 loss:0.008 grdn:0.171 lr:8.4e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 04:13:36 ts/train.py:241 Checkpoint policy after step 160000 +INFO 2025-05-19 04:14:41 ts/train.py:232 step:160K smpl:3M ep:12K epch:40.41 loss:0.008 grdn:0.164 lr:8.4e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 04:15:44 ts/train.py:232 step:160K smpl:3M ep:12K epch:40.46 loss:0.008 grdn:0.174 lr:8.3e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 04:16:45 ts/train.py:232 step:161K smpl:3M ep:12K epch:40.51 loss:0.009 grdn:0.179 lr:8.3e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 04:17:47 ts/train.py:232 step:161K smpl:3M ep:12K epch:40.56 loss:0.008 grdn:0.162 lr:8.3e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 04:18:50 ts/train.py:232 step:161K smpl:3M ep:12K epch:40.61 loss:0.008 grdn:0.159 lr:8.3e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 04:19:56 ts/train.py:232 step:161K smpl:3M ep:12K epch:40.66 loss:0.009 grdn:0.173 lr:8.3e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 04:21:00 ts/train.py:232 step:161K smpl:3M ep:12K epch:40.71 loss:0.007 grdn:0.157 lr:8.3e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 04:22:02 ts/train.py:232 step:162K smpl:3M ep:12K epch:40.76 loss:0.009 grdn:0.172 lr:8.3e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 04:23:05 ts/train.py:232 step:162K smpl:3M ep:12K epch:40.81 loss:0.009 grdn:0.177 lr:8.3e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 04:24:08 ts/train.py:232 step:162K smpl:3M ep:12K epch:40.86 loss:0.008 grdn:0.175 lr:8.3e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 04:25:11 ts/train.py:232 step:162K smpl:3M ep:12K epch:40.91 loss:0.008 grdn:0.174 lr:8.3e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-19 04:26:14 ts/train.py:232 step:162K smpl:3M ep:12K epch:40.96 loss:0.009 grdn:0.183 lr:8.3e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 04:27:17 ts/train.py:232 step:163K smpl:3M ep:12K epch:41.01 loss:0.008 grdn:0.167 lr:8.3e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 04:28:20 ts/train.py:232 step:163K smpl:3M ep:12K epch:41.06 loss:0.009 grdn:0.177 lr:8.3e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 04:29:25 ts/train.py:232 step:163K smpl:3M ep:12K epch:41.11 loss:0.009 grdn:0.178 lr:8.3e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 04:30:29 ts/train.py:232 step:163K smpl:3M ep:12K epch:41.16 loss:0.008 grdn:0.170 lr:8.3e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 04:31:32 ts/train.py:232 step:163K smpl:3M ep:12K epch:41.21 loss:0.008 grdn:0.168 lr:8.3e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 04:32:37 ts/train.py:232 step:164K smpl:3M ep:12K epch:41.26 loss:0.008 grdn:0.171 lr:8.3e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 04:33:40 ts/train.py:232 step:164K smpl:3M ep:12K epch:41.31 loss:0.008 grdn:0.169 lr:8.3e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 04:34:45 ts/train.py:232 step:164K smpl:3M ep:12K epch:41.37 loss:0.008 grdn:0.173 lr:8.3e-05 updt_s:0.222 data_s:0.100 +INFO 2025-05-19 04:35:48 ts/train.py:232 step:164K smpl:3M ep:12K epch:41.42 loss:0.007 grdn:0.153 lr:8.3e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 04:36:51 ts/train.py:232 step:164K smpl:3M ep:12K epch:41.47 loss:0.008 grdn:0.174 lr:8.3e-05 updt_s:0.222 data_s:0.096 +INFO 2025-05-19 04:37:55 ts/train.py:232 step:165K smpl:3M ep:12K epch:41.52 loss:0.008 grdn:0.168 lr:8.3e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 04:38:58 ts/train.py:232 step:165K smpl:3M ep:12K epch:41.57 loss:0.007 grdn:0.157 lr:8.3e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 04:40:03 ts/train.py:232 step:165K smpl:3M ep:12K epch:41.62 loss:0.008 grdn:0.170 lr:8.3e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-19 04:41:08 ts/train.py:232 step:165K smpl:3M ep:13K epch:41.67 loss:0.008 grdn:0.161 lr:8.3e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 04:42:12 ts/train.py:232 step:165K smpl:3M ep:13K epch:41.72 loss:0.008 grdn:0.174 lr:8.2e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 04:43:15 ts/train.py:232 step:166K smpl:3M ep:13K epch:41.77 loss:0.008 grdn:0.177 lr:8.2e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 04:44:19 ts/train.py:232 step:166K smpl:3M ep:13K epch:41.82 loss:0.008 grdn:0.167 lr:8.2e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 04:45:23 ts/train.py:232 step:166K smpl:3M ep:13K epch:41.87 loss:0.008 grdn:0.171 lr:8.2e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 04:46:25 ts/train.py:232 step:166K smpl:3M ep:13K epch:41.92 loss:0.009 grdn:0.174 lr:8.2e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 04:47:24 ts/train.py:232 step:166K smpl:3M ep:13K epch:41.97 loss:0.008 grdn:0.169 lr:8.2e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 04:48:23 ts/train.py:232 step:167K smpl:3M ep:13K epch:42.02 loss:0.008 grdn:0.169 lr:8.2e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-19 04:49:23 ts/train.py:232 step:167K smpl:3M ep:13K epch:42.07 loss:0.008 grdn:0.175 lr:8.2e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 04:50:21 ts/train.py:232 step:167K smpl:3M ep:13K epch:42.12 loss:0.008 grdn:0.165 lr:8.2e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 04:51:19 ts/train.py:232 step:167K smpl:3M ep:13K epch:42.17 loss:0.008 grdn:0.175 lr:8.2e-05 updt_s:0.221 data_s:0.066 +INFO 2025-05-19 04:52:18 ts/train.py:232 step:167K smpl:3M ep:13K epch:42.22 loss:0.008 grdn:0.168 lr:8.2e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 04:53:16 ts/train.py:232 step:168K smpl:3M ep:13K epch:42.27 loss:0.008 grdn:0.172 lr:8.2e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 04:54:14 ts/train.py:232 step:168K smpl:3M ep:13K epch:42.32 loss:0.008 grdn:0.168 lr:8.2e-05 updt_s:0.221 data_s:0.067 +INFO 2025-05-19 04:55:12 ts/train.py:232 step:168K smpl:3M ep:13K epch:42.37 loss:0.008 grdn:0.168 lr:8.2e-05 updt_s:0.221 data_s:0.068 +INFO 2025-05-19 04:56:09 ts/train.py:232 step:168K smpl:3M ep:13K epch:42.42 loss:0.007 grdn:0.158 lr:8.2e-05 updt_s:0.221 data_s:0.065 +INFO 2025-05-19 04:57:05 ts/train.py:232 step:168K smpl:3M ep:13K epch:42.47 loss:0.007 grdn:0.168 lr:8.2e-05 updt_s:0.221 data_s:0.060 +INFO 2025-05-19 04:58:03 ts/train.py:232 step:169K smpl:3M ep:13K epch:42.53 loss:0.008 grdn:0.161 lr:8.2e-05 updt_s:0.221 data_s:0.065 +INFO 2025-05-19 04:59:04 ts/train.py:232 step:169K smpl:3M ep:13K epch:42.58 loss:0.007 grdn:0.164 lr:8.2e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 05:00:07 ts/train.py:232 step:169K smpl:3M ep:13K epch:42.63 loss:0.007 grdn:0.168 lr:8.2e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 05:01:09 ts/train.py:232 step:169K smpl:3M ep:13K epch:42.68 loss:0.008 grdn:0.167 lr:8.2e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 05:02:12 ts/train.py:232 step:169K smpl:3M ep:13K epch:42.73 loss:0.008 grdn:0.170 lr:8.2e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 05:03:14 ts/train.py:232 step:170K smpl:3M ep:13K epch:42.78 loss:0.008 grdn:0.170 lr:8.2e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 05:04:16 ts/train.py:232 step:170K smpl:3M ep:13K epch:42.83 loss:0.008 grdn:0.172 lr:8.2e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 05:05:19 ts/train.py:232 step:170K smpl:3M ep:13K epch:42.88 loss:0.008 grdn:0.168 lr:8.2e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 05:06:20 ts/train.py:232 step:170K smpl:3M ep:13K epch:42.93 loss:0.008 grdn:0.168 lr:8.2e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 05:07:22 ts/train.py:232 step:170K smpl:3M ep:13K epch:42.98 loss:0.008 grdn:0.174 lr:8.1e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 05:08:26 ts/train.py:232 step:171K smpl:3M ep:13K epch:43.03 loss:0.008 grdn:0.163 lr:8.1e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 05:09:28 ts/train.py:232 step:171K smpl:3M ep:13K epch:43.08 loss:0.008 grdn:0.163 lr:8.1e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 05:10:31 ts/train.py:232 step:171K smpl:3M ep:13K epch:43.13 loss:0.008 grdn:0.169 lr:8.1e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-19 05:11:34 ts/train.py:232 step:171K smpl:3M ep:13K epch:43.18 loss:0.007 grdn:0.163 lr:8.1e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 05:12:37 ts/train.py:232 step:171K smpl:3M ep:13K epch:43.23 loss:0.008 grdn:0.170 lr:8.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 05:13:39 ts/train.py:232 step:172K smpl:3M ep:13K epch:43.28 loss:0.008 grdn:0.174 lr:8.1e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 05:14:43 ts/train.py:232 step:172K smpl:3M ep:13K epch:43.33 loss:0.008 grdn:0.168 lr:8.1e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 05:15:46 ts/train.py:232 step:172K smpl:3M ep:13K epch:43.38 loss:0.008 grdn:0.166 lr:8.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 05:16:49 ts/train.py:232 step:172K smpl:3M ep:13K epch:43.43 loss:0.008 grdn:0.176 lr:8.1e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 05:17:53 ts/train.py:232 step:172K smpl:3M ep:13K epch:43.48 loss:0.007 grdn:0.169 lr:8.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 05:18:54 ts/train.py:232 step:173K smpl:3M ep:13K epch:43.53 loss:0.008 grdn:0.179 lr:8.1e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 05:19:48 ts/train.py:232 step:173K smpl:3M ep:13K epch:43.58 loss:0.009 grdn:0.180 lr:8.1e-05 updt_s:0.222 data_s:0.047 +INFO 2025-05-19 05:20:41 ts/train.py:232 step:173K smpl:3M ep:13K epch:43.64 loss:0.007 grdn:0.168 lr:8.1e-05 updt_s:0.222 data_s:0.044 +INFO 2025-05-19 05:21:34 ts/train.py:232 step:173K smpl:3M ep:13K epch:43.69 loss:0.007 grdn:0.160 lr:8.1e-05 updt_s:0.222 data_s:0.042 +INFO 2025-05-19 05:22:30 ts/train.py:232 step:173K smpl:3M ep:13K epch:43.74 loss:0.008 grdn:0.172 lr:8.1e-05 updt_s:0.222 data_s:0.055 +INFO 2025-05-19 05:23:21 ts/train.py:232 step:174K smpl:3M ep:13K epch:43.79 loss:0.008 grdn:0.163 lr:8.1e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-19 05:24:14 ts/train.py:232 step:174K smpl:3M ep:13K epch:43.84 loss:0.007 grdn:0.168 lr:8.1e-05 updt_s:0.222 data_s:0.039 +INFO 2025-05-19 05:25:07 ts/train.py:232 step:174K smpl:3M ep:13K epch:43.89 loss:0.007 grdn:0.163 lr:8.1e-05 updt_s:0.222 data_s:0.043 +INFO 2025-05-19 05:25:59 ts/train.py:232 step:174K smpl:3M ep:13K epch:43.94 loss:0.008 grdn:0.172 lr:8.1e-05 updt_s:0.222 data_s:0.038 +INFO 2025-05-19 05:26:49 ts/train.py:232 step:174K smpl:3M ep:13K epch:43.99 loss:0.008 grdn:0.165 lr:8.1e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-19 05:27:40 ts/train.py:232 step:175K smpl:3M ep:13K epch:44.04 loss:0.008 grdn:0.173 lr:8.1e-05 updt_s:0.222 data_s:0.032 +INFO 2025-05-19 05:28:33 ts/train.py:232 step:175K smpl:3M ep:13K epch:44.09 loss:0.008 grdn:0.176 lr:8.1e-05 updt_s:0.222 data_s:0.044 +INFO 2025-05-19 05:29:25 ts/train.py:232 step:175K smpl:3M ep:13K epch:44.14 loss:0.008 grdn:0.161 lr:8.1e-05 updt_s:0.222 data_s:0.036 +INFO 2025-05-19 05:30:16 ts/train.py:232 step:175K smpl:3M ep:13K epch:44.19 loss:0.008 grdn:0.175 lr:8.0e-05 updt_s:0.222 data_s:0.033 +INFO 2025-05-19 05:31:09 ts/train.py:232 step:175K smpl:3M ep:13K epch:44.24 loss:0.007 grdn:0.162 lr:8.0e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-19 05:32:01 ts/train.py:232 step:176K smpl:3M ep:13K epch:44.29 loss:0.007 grdn:0.163 lr:8.0e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-19 05:32:53 ts/train.py:232 step:176K smpl:3M ep:13K epch:44.34 loss:0.008 grdn:0.173 lr:8.0e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-19 05:33:46 ts/train.py:232 step:176K smpl:3M ep:13K epch:44.39 loss:0.008 grdn:0.174 lr:8.0e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-19 05:34:37 ts/train.py:232 step:176K smpl:3M ep:13K epch:44.44 loss:0.007 grdn:0.165 lr:8.0e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-19 05:35:32 ts/train.py:232 step:176K smpl:3M ep:13K epch:44.49 loss:0.008 grdn:0.177 lr:8.0e-05 updt_s:0.221 data_s:0.054 +INFO 2025-05-19 05:36:32 ts/train.py:232 step:177K smpl:3M ep:13K epch:44.54 loss:0.007 grdn:0.167 lr:8.0e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 05:37:31 ts/train.py:232 step:177K smpl:3M ep:13K epch:44.59 loss:0.008 grdn:0.175 lr:8.0e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 05:38:31 ts/train.py:232 step:177K smpl:3M ep:13K epch:44.64 loss:0.008 grdn:0.170 lr:8.0e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 05:39:31 ts/train.py:232 step:177K smpl:3M ep:13K epch:44.69 loss:0.008 grdn:0.161 lr:8.0e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 05:40:31 ts/train.py:232 step:177K smpl:3M ep:13K epch:44.75 loss:0.008 grdn:0.171 lr:8.0e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 05:41:31 ts/train.py:232 step:178K smpl:3M ep:13K epch:44.80 loss:0.008 grdn:0.161 lr:8.0e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 05:42:30 ts/train.py:232 step:178K smpl:3M ep:13K epch:44.85 loss:0.007 grdn:0.161 lr:8.0e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 05:43:29 ts/train.py:232 step:178K smpl:3M ep:13K epch:44.90 loss:0.008 grdn:0.163 lr:8.0e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 05:44:29 ts/train.py:232 step:178K smpl:3M ep:13K epch:44.95 loss:0.007 grdn:0.156 lr:8.0e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 05:45:28 ts/train.py:232 step:178K smpl:3M ep:13K epch:45.00 loss:0.008 grdn:0.175 lr:8.0e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 05:46:28 ts/train.py:232 step:179K smpl:3M ep:14K epch:45.05 loss:0.007 grdn:0.160 lr:8.0e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 05:47:28 ts/train.py:232 step:179K smpl:3M ep:14K epch:45.10 loss:0.008 grdn:0.175 lr:8.0e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 05:48:28 ts/train.py:232 step:179K smpl:3M ep:14K epch:45.15 loss:0.008 grdn:0.164 lr:8.0e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 05:49:27 ts/train.py:232 step:179K smpl:3M ep:14K epch:45.20 loss:0.008 grdn:0.179 lr:8.0e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 05:50:27 ts/train.py:232 step:179K smpl:3M ep:14K epch:45.25 loss:0.008 grdn:0.167 lr:8.0e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 05:51:27 ts/train.py:232 step:180K smpl:3M ep:14K epch:45.30 loss:0.008 grdn:0.169 lr:8.0e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 05:52:28 ts/train.py:232 step:180K smpl:3M ep:14K epch:45.35 loss:0.007 grdn:0.157 lr:8.0e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 05:53:28 ts/train.py:232 step:180K smpl:3M ep:14K epch:45.40 loss:0.008 grdn:0.167 lr:7.9e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 05:53:28 ts/train.py:241 Checkpoint policy after step 180000 +INFO 2025-05-19 05:54:33 ts/train.py:232 step:180K smpl:3M ep:14K epch:45.45 loss:0.007 grdn:0.165 lr:7.9e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 05:55:38 ts/train.py:232 step:180K smpl:3M ep:14K epch:45.50 loss:0.007 grdn:0.176 lr:7.9e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 05:56:41 ts/train.py:232 step:181K smpl:3M ep:14K epch:45.55 loss:0.007 grdn:0.162 lr:7.9e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 05:57:44 ts/train.py:232 step:181K smpl:3M ep:14K epch:45.60 loss:0.007 grdn:0.162 lr:7.9e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 05:58:46 ts/train.py:232 step:181K smpl:3M ep:14K epch:45.65 loss:0.008 grdn:0.160 lr:7.9e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 05:59:49 ts/train.py:232 step:181K smpl:3M ep:14K epch:45.70 loss:0.008 grdn:0.167 lr:7.9e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 06:00:51 ts/train.py:232 step:181K smpl:3M ep:14K epch:45.75 loss:0.008 grdn:0.169 lr:7.9e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 06:01:51 ts/train.py:232 step:182K smpl:3M ep:14K epch:45.80 loss:0.008 grdn:0.174 lr:7.9e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 06:02:49 ts/train.py:232 step:182K smpl:3M ep:14K epch:45.85 loss:0.008 grdn:0.175 lr:7.9e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 06:03:47 ts/train.py:232 step:182K smpl:3M ep:14K epch:45.91 loss:0.007 grdn:0.157 lr:7.9e-05 updt_s:0.221 data_s:0.066 +INFO 2025-05-19 06:04:45 ts/train.py:232 step:182K smpl:3M ep:14K epch:45.96 loss:0.008 grdn:0.172 lr:7.9e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 06:05:43 ts/train.py:232 step:182K smpl:3M ep:14K epch:46.01 loss:0.007 grdn:0.159 lr:7.9e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-19 06:06:42 ts/train.py:232 step:183K smpl:3M ep:14K epch:46.06 loss:0.007 grdn:0.165 lr:7.9e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 06:07:40 ts/train.py:232 step:183K smpl:3M ep:14K epch:46.11 loss:0.008 grdn:0.162 lr:7.9e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 06:08:38 ts/train.py:232 step:183K smpl:3M ep:14K epch:46.16 loss:0.008 grdn:0.173 lr:7.9e-05 updt_s:0.221 data_s:0.067 +INFO 2025-05-19 06:09:37 ts/train.py:232 step:183K smpl:3M ep:14K epch:46.21 loss:0.008 grdn:0.177 lr:7.9e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 06:10:35 ts/train.py:232 step:183K smpl:3M ep:14K epch:46.26 loss:0.008 grdn:0.165 lr:7.9e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 06:11:33 ts/train.py:232 step:184K smpl:3M ep:14K epch:46.31 loss:0.008 grdn:0.171 lr:7.9e-05 updt_s:0.221 data_s:0.067 +INFO 2025-05-19 06:12:30 ts/train.py:232 step:184K smpl:3M ep:14K epch:46.36 loss:0.008 grdn:0.169 lr:7.9e-05 updt_s:0.221 data_s:0.067 +INFO 2025-05-19 06:13:29 ts/train.py:232 step:184K smpl:3M ep:14K epch:46.41 loss:0.008 grdn:0.176 lr:7.9e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 06:14:31 ts/train.py:232 step:184K smpl:3M ep:14K epch:46.46 loss:0.008 grdn:0.169 lr:7.9e-05 updt_s:0.220 data_s:0.090 +INFO 2025-05-19 06:15:32 ts/train.py:232 step:184K smpl:3M ep:14K epch:46.51 loss:0.008 grdn:0.170 lr:7.9e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 06:16:33 ts/train.py:232 step:185K smpl:3M ep:14K epch:46.56 loss:0.007 grdn:0.164 lr:7.9e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 06:17:35 ts/train.py:232 step:185K smpl:3M ep:14K epch:46.61 loss:0.007 grdn:0.159 lr:7.8e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 06:18:35 ts/train.py:232 step:185K smpl:3M ep:14K epch:46.66 loss:0.007 grdn:0.169 lr:7.8e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 06:19:36 ts/train.py:232 step:185K smpl:3M ep:14K epch:46.71 loss:0.007 grdn:0.167 lr:7.8e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 06:20:36 ts/train.py:232 step:185K smpl:3M ep:14K epch:46.76 loss:0.008 grdn:0.172 lr:7.8e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 06:21:34 ts/train.py:232 step:186K smpl:3M ep:14K epch:46.81 loss:0.007 grdn:0.167 lr:7.8e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 06:22:35 ts/train.py:232 step:186K smpl:3M ep:14K epch:46.86 loss:0.008 grdn:0.173 lr:7.8e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 06:23:35 ts/train.py:232 step:186K smpl:3M ep:14K epch:46.91 loss:0.008 grdn:0.169 lr:7.8e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 06:24:35 ts/train.py:232 step:186K smpl:3M ep:14K epch:46.96 loss:0.007 grdn:0.158 lr:7.8e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 06:25:35 ts/train.py:232 step:186K smpl:3M ep:14K epch:47.02 loss:0.008 grdn:0.167 lr:7.8e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 06:26:34 ts/train.py:232 step:187K smpl:3M ep:14K epch:47.07 loss:0.007 grdn:0.161 lr:7.8e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 06:27:35 ts/train.py:232 step:187K smpl:3M ep:14K epch:47.12 loss:0.007 grdn:0.153 lr:7.8e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 06:28:35 ts/train.py:232 step:187K smpl:3M ep:14K epch:47.17 loss:0.007 grdn:0.153 lr:7.8e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 06:29:33 ts/train.py:232 step:187K smpl:3M ep:14K epch:47.22 loss:0.008 grdn:0.171 lr:7.8e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 06:30:33 ts/train.py:232 step:187K smpl:3M ep:14K epch:47.27 loss:0.007 grdn:0.168 lr:7.8e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 06:31:33 ts/train.py:232 step:188K smpl:3M ep:14K epch:47.32 loss:0.007 grdn:0.166 lr:7.8e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 06:32:33 ts/train.py:232 step:188K smpl:3M ep:14K epch:47.37 loss:0.007 grdn:0.165 lr:7.8e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 06:33:37 ts/train.py:232 step:188K smpl:3M ep:14K epch:47.42 loss:0.007 grdn:0.164 lr:7.8e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 06:34:43 ts/train.py:232 step:188K smpl:3M ep:14K epch:47.47 loss:0.008 grdn:0.174 lr:7.8e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 06:35:47 ts/train.py:232 step:188K smpl:3M ep:14K epch:47.52 loss:0.008 grdn:0.174 lr:7.8e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 06:36:52 ts/train.py:232 step:189K smpl:3M ep:14K epch:47.57 loss:0.007 grdn:0.158 lr:7.8e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 06:37:58 ts/train.py:232 step:189K smpl:3M ep:14K epch:47.62 loss:0.008 grdn:0.172 lr:7.8e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 06:39:02 ts/train.py:232 step:189K smpl:3M ep:14K epch:47.67 loss:0.007 grdn:0.167 lr:7.8e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 06:40:05 ts/train.py:232 step:189K smpl:3M ep:14K epch:47.72 loss:0.007 grdn:0.166 lr:7.8e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 06:41:03 ts/train.py:232 step:189K smpl:3M ep:14K epch:47.77 loss:0.008 grdn:0.175 lr:7.7e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 06:42:04 ts/train.py:232 step:190K smpl:3M ep:14K epch:47.82 loss:0.007 grdn:0.162 lr:7.7e-05 updt_s:0.222 data_s:0.081 +INFO 2025-05-19 06:43:05 ts/train.py:232 step:190K smpl:3M ep:14K epch:47.87 loss:0.008 grdn:0.172 lr:7.7e-05 updt_s:0.222 data_s:0.080 +INFO 2025-05-19 06:44:05 ts/train.py:232 step:190K smpl:3M ep:14K epch:47.92 loss:0.006 grdn:0.155 lr:7.7e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 06:45:04 ts/train.py:232 step:190K smpl:3M ep:14K epch:47.97 loss:0.008 grdn:0.170 lr:7.7e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 06:46:03 ts/train.py:232 step:190K smpl:3M ep:14K epch:48.02 loss:0.008 grdn:0.182 lr:7.7e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-19 06:47:03 ts/train.py:232 step:191K smpl:3M ep:14K epch:48.07 loss:0.007 grdn:0.155 lr:7.7e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 06:48:03 ts/train.py:232 step:191K smpl:3M ep:14K epch:48.12 loss:0.007 grdn:0.164 lr:7.7e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 06:49:02 ts/train.py:232 step:191K smpl:3M ep:14K epch:48.18 loss:0.008 grdn:0.166 lr:7.7e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-19 06:50:02 ts/train.py:232 step:191K smpl:3M ep:14K epch:48.23 loss:0.007 grdn:0.165 lr:7.7e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 06:51:02 ts/train.py:232 step:191K smpl:3M ep:14K epch:48.28 loss:0.008 grdn:0.166 lr:7.7e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 06:52:04 ts/train.py:232 step:192K smpl:3M ep:14K epch:48.33 loss:0.009 grdn:0.183 lr:7.7e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-19 06:53:08 ts/train.py:232 step:192K smpl:3M ep:15K epch:48.38 loss:0.007 grdn:0.167 lr:7.7e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 06:54:12 ts/train.py:232 step:192K smpl:3M ep:15K epch:48.43 loss:0.007 grdn:0.157 lr:7.7e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 06:55:16 ts/train.py:232 step:192K smpl:3M ep:15K epch:48.48 loss:0.007 grdn:0.158 lr:7.7e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 06:56:21 ts/train.py:232 step:192K smpl:3M ep:15K epch:48.53 loss:0.008 grdn:0.179 lr:7.7e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 06:57:26 ts/train.py:232 step:193K smpl:3M ep:15K epch:48.58 loss:0.008 grdn:0.165 lr:7.7e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 06:58:31 ts/train.py:232 step:193K smpl:3M ep:15K epch:48.63 loss:0.008 grdn:0.174 lr:7.7e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 06:59:32 ts/train.py:232 step:193K smpl:3M ep:15K epch:48.68 loss:0.007 grdn:0.162 lr:7.7e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 07:00:31 ts/train.py:232 step:193K smpl:3M ep:15K epch:48.73 loss:0.008 grdn:0.168 lr:7.7e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 07:01:29 ts/train.py:232 step:193K smpl:3M ep:15K epch:48.78 loss:0.008 grdn:0.170 lr:7.7e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 07:02:28 ts/train.py:232 step:194K smpl:3M ep:15K epch:48.83 loss:0.007 grdn:0.167 lr:7.7e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 07:03:26 ts/train.py:232 step:194K smpl:3M ep:15K epch:48.88 loss:0.007 grdn:0.169 lr:7.6e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 07:04:25 ts/train.py:232 step:194K smpl:3M ep:15K epch:48.93 loss:0.007 grdn:0.162 lr:7.6e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 07:05:23 ts/train.py:232 step:194K smpl:3M ep:15K epch:48.98 loss:0.008 grdn:0.167 lr:7.6e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-19 07:06:21 ts/train.py:232 step:194K smpl:3M ep:15K epch:49.03 loss:0.007 grdn:0.166 lr:7.6e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 07:07:20 ts/train.py:232 step:195K smpl:3M ep:15K epch:49.08 loss:0.008 grdn:0.175 lr:7.6e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 07:08:18 ts/train.py:232 step:195K smpl:3M ep:15K epch:49.13 loss:0.007 grdn:0.162 lr:7.6e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 07:09:17 ts/train.py:232 step:195K smpl:3M ep:15K epch:49.18 loss:0.007 grdn:0.173 lr:7.6e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 07:10:15 ts/train.py:232 step:195K smpl:3M ep:15K epch:49.23 loss:0.008 grdn:0.167 lr:7.6e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 07:11:13 ts/train.py:232 step:195K smpl:3M ep:15K epch:49.29 loss:0.007 grdn:0.157 lr:7.6e-05 updt_s:0.221 data_s:0.066 +INFO 2025-05-19 07:12:14 ts/train.py:232 step:196K smpl:3M ep:15K epch:49.34 loss:0.007 grdn:0.168 lr:7.6e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 07:13:16 ts/train.py:232 step:196K smpl:3M ep:15K epch:49.39 loss:0.008 grdn:0.169 lr:7.6e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 07:14:19 ts/train.py:232 step:196K smpl:3M ep:15K epch:49.44 loss:0.008 grdn:0.178 lr:7.6e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 07:15:22 ts/train.py:232 step:196K smpl:3M ep:15K epch:49.49 loss:0.007 grdn:0.161 lr:7.6e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 07:16:25 ts/train.py:232 step:196K smpl:3M ep:15K epch:49.54 loss:0.008 grdn:0.170 lr:7.6e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 07:17:28 ts/train.py:232 step:197K smpl:3M ep:15K epch:49.59 loss:0.007 grdn:0.161 lr:7.6e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 07:18:30 ts/train.py:232 step:197K smpl:3M ep:15K epch:49.64 loss:0.008 grdn:0.182 lr:7.6e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 07:19:33 ts/train.py:232 step:197K smpl:3M ep:15K epch:49.69 loss:0.008 grdn:0.170 lr:7.6e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 07:20:35 ts/train.py:232 step:197K smpl:3M ep:15K epch:49.74 loss:0.007 grdn:0.152 lr:7.6e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 07:21:37 ts/train.py:232 step:197K smpl:3M ep:15K epch:49.79 loss:0.008 grdn:0.168 lr:7.6e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 07:22:40 ts/train.py:232 step:198K smpl:3M ep:15K epch:49.84 loss:0.007 grdn:0.169 lr:7.6e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 07:23:43 ts/train.py:232 step:198K smpl:3M ep:15K epch:49.89 loss:0.007 grdn:0.159 lr:7.6e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 07:24:47 ts/train.py:232 step:198K smpl:3M ep:15K epch:49.94 loss:0.007 grdn:0.169 lr:7.6e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 07:25:50 ts/train.py:232 step:198K smpl:3M ep:15K epch:49.99 loss:0.007 grdn:0.166 lr:7.6e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 07:26:53 ts/train.py:232 step:198K smpl:3M ep:15K epch:50.04 loss:0.007 grdn:0.159 lr:7.5e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 07:27:56 ts/train.py:232 step:199K smpl:3M ep:15K epch:50.09 loss:0.007 grdn:0.171 lr:7.5e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 07:29:00 ts/train.py:232 step:199K smpl:3M ep:15K epch:50.14 loss:0.007 grdn:0.164 lr:7.5e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 07:30:02 ts/train.py:232 step:199K smpl:3M ep:15K epch:50.19 loss:0.007 grdn:0.160 lr:7.5e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 07:31:05 ts/train.py:232 step:199K smpl:3M ep:15K epch:50.24 loss:0.008 grdn:0.178 lr:7.5e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 07:32:09 ts/train.py:232 step:199K smpl:3M ep:15K epch:50.29 loss:0.007 grdn:0.168 lr:7.5e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 07:33:16 ts/train.py:232 step:200K smpl:3M ep:15K epch:50.34 loss:0.007 grdn:0.159 lr:7.5e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-19 07:34:22 ts/train.py:232 step:200K smpl:3M ep:15K epch:50.39 loss:0.007 grdn:0.155 lr:7.5e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-19 07:35:29 ts/train.py:232 step:200K smpl:3M ep:15K epch:50.45 loss:0.007 grdn:0.173 lr:7.5e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-19 07:35:29 ts/train.py:241 Checkpoint policy after step 200000 +INFO 2025-05-19 07:36:41 ts/train.py:232 step:200K smpl:3M ep:15K epch:50.50 loss:0.007 grdn:0.161 lr:7.5e-05 updt_s:0.221 data_s:0.113 +INFO 2025-05-19 07:37:49 ts/train.py:232 step:200K smpl:3M ep:15K epch:50.55 loss:0.007 grdn:0.154 lr:7.5e-05 updt_s:0.221 data_s:0.119 +INFO 2025-05-19 07:38:58 ts/train.py:232 step:201K smpl:3M ep:15K epch:50.60 loss:0.007 grdn:0.160 lr:7.5e-05 updt_s:0.221 data_s:0.118 +INFO 2025-05-19 07:39:57 ts/train.py:232 step:201K smpl:3M ep:15K epch:50.65 loss:0.008 grdn:0.167 lr:7.5e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 07:40:59 ts/train.py:232 step:201K smpl:3M ep:15K epch:50.70 loss:0.007 grdn:0.170 lr:7.5e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 07:41:59 ts/train.py:232 step:201K smpl:3M ep:15K epch:50.75 loss:0.007 grdn:0.162 lr:7.5e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 07:42:59 ts/train.py:232 step:201K smpl:3M ep:15K epch:50.80 loss:0.007 grdn:0.167 lr:7.5e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 07:44:01 ts/train.py:232 step:202K smpl:3M ep:15K epch:50.85 loss:0.007 grdn:0.161 lr:7.5e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 07:45:05 ts/train.py:232 step:202K smpl:3M ep:15K epch:50.90 loss:0.008 grdn:0.174 lr:7.5e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 07:46:09 ts/train.py:232 step:202K smpl:3M ep:15K epch:50.95 loss:0.007 grdn:0.158 lr:7.5e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 07:47:11 ts/train.py:232 step:202K smpl:3M ep:15K epch:51.00 loss:0.007 grdn:0.165 lr:7.5e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 07:48:13 ts/train.py:232 step:202K smpl:3M ep:15K epch:51.05 loss:0.008 grdn:0.176 lr:7.5e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 07:49:14 ts/train.py:232 step:203K smpl:3M ep:15K epch:51.10 loss:0.007 grdn:0.180 lr:7.5e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 07:50:15 ts/train.py:232 step:203K smpl:3M ep:15K epch:51.15 loss:0.006 grdn:0.157 lr:7.4e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 07:51:16 ts/train.py:232 step:203K smpl:3M ep:15K epch:51.20 loss:0.007 grdn:0.164 lr:7.4e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 07:52:14 ts/train.py:232 step:203K smpl:3M ep:15K epch:51.25 loss:0.007 grdn:0.166 lr:7.4e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 07:53:20 ts/train.py:232 step:203K smpl:3M ep:15K epch:51.30 loss:0.008 grdn:0.165 lr:7.4e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 07:54:25 ts/train.py:232 step:204K smpl:3M ep:15K epch:51.35 loss:0.007 grdn:0.166 lr:7.4e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 07:55:28 ts/train.py:232 step:204K smpl:3M ep:15K epch:51.40 loss:0.007 grdn:0.174 lr:7.4e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 07:56:32 ts/train.py:232 step:204K smpl:3M ep:15K epch:51.45 loss:0.007 grdn:0.170 lr:7.4e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 07:57:36 ts/train.py:232 step:204K smpl:3M ep:15K epch:51.50 loss:0.007 grdn:0.164 lr:7.4e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 07:58:40 ts/train.py:232 step:204K smpl:3M ep:15K epch:51.56 loss:0.007 grdn:0.166 lr:7.4e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 07:59:38 ts/train.py:232 step:205K smpl:3M ep:15K epch:51.61 loss:0.007 grdn:0.162 lr:7.4e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 08:00:37 ts/train.py:232 step:205K smpl:3M ep:15K epch:51.66 loss:0.007 grdn:0.164 lr:7.4e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 08:01:36 ts/train.py:232 step:205K smpl:3M ep:16K epch:51.71 loss:0.007 grdn:0.168 lr:7.4e-05 updt_s:0.222 data_s:0.072 +INFO 2025-05-19 08:02:35 ts/train.py:232 step:205K smpl:3M ep:16K epch:51.76 loss:0.008 grdn:0.167 lr:7.4e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 08:03:34 ts/train.py:232 step:205K smpl:3M ep:16K epch:51.81 loss:0.007 grdn:0.165 lr:7.4e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 08:04:34 ts/train.py:232 step:206K smpl:3M ep:16K epch:51.86 loss:0.007 grdn:0.164 lr:7.4e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 08:05:34 ts/train.py:232 step:206K smpl:3M ep:16K epch:51.91 loss:0.007 grdn:0.167 lr:7.4e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 08:06:34 ts/train.py:232 step:206K smpl:3M ep:16K epch:51.96 loss:0.007 grdn:0.167 lr:7.4e-05 updt_s:0.222 data_s:0.075 +INFO 2025-05-19 08:07:34 ts/train.py:232 step:206K smpl:3M ep:16K epch:52.01 loss:0.007 grdn:0.162 lr:7.4e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-19 08:08:34 ts/train.py:232 step:206K smpl:3M ep:16K epch:52.06 loss:0.007 grdn:0.164 lr:7.4e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 08:09:32 ts/train.py:232 step:207K smpl:3M ep:16K epch:52.11 loss:0.008 grdn:0.172 lr:7.4e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 08:10:31 ts/train.py:232 step:207K smpl:3M ep:16K epch:52.16 loss:0.007 grdn:0.161 lr:7.4e-05 updt_s:0.222 data_s:0.075 +INFO 2025-05-19 08:11:31 ts/train.py:232 step:207K smpl:3M ep:16K epch:52.21 loss:0.007 grdn:0.168 lr:7.3e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-19 08:12:35 ts/train.py:232 step:207K smpl:3M ep:16K epch:52.26 loss:0.007 grdn:0.158 lr:7.3e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 08:13:39 ts/train.py:232 step:207K smpl:3M ep:16K epch:52.31 loss:0.007 grdn:0.153 lr:7.3e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 08:14:41 ts/train.py:232 step:208K smpl:3M ep:16K epch:52.36 loss:0.007 grdn:0.165 lr:7.3e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 08:15:41 ts/train.py:232 step:208K smpl:3M ep:16K epch:52.41 loss:0.007 grdn:0.168 lr:7.3e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 08:16:42 ts/train.py:232 step:208K smpl:3M ep:16K epch:52.46 loss:0.007 grdn:0.162 lr:7.3e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 08:17:42 ts/train.py:232 step:208K smpl:3M ep:16K epch:52.51 loss:0.007 grdn:0.158 lr:7.3e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 08:18:42 ts/train.py:232 step:208K smpl:3M ep:16K epch:52.56 loss:0.007 grdn:0.160 lr:7.3e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 08:19:44 ts/train.py:232 step:209K smpl:3M ep:16K epch:52.61 loss:0.007 grdn:0.171 lr:7.3e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 08:20:48 ts/train.py:232 step:209K smpl:3M ep:16K epch:52.66 loss:0.008 grdn:0.170 lr:7.3e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 08:21:49 ts/train.py:232 step:209K smpl:3M ep:16K epch:52.72 loss:0.007 grdn:0.161 lr:7.3e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 08:22:52 ts/train.py:232 step:209K smpl:3M ep:16K epch:52.77 loss:0.007 grdn:0.161 lr:7.3e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 08:23:52 ts/train.py:232 step:209K smpl:3M ep:16K epch:52.82 loss:0.007 grdn:0.164 lr:7.3e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 08:24:54 ts/train.py:232 step:210K smpl:3M ep:16K epch:52.87 loss:0.007 grdn:0.164 lr:7.3e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 08:25:54 ts/train.py:232 step:210K smpl:3M ep:16K epch:52.92 loss:0.007 grdn:0.173 lr:7.3e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 08:26:56 ts/train.py:232 step:210K smpl:3M ep:16K epch:52.97 loss:0.008 grdn:0.171 lr:7.3e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 08:27:57 ts/train.py:232 step:210K smpl:3M ep:16K epch:53.02 loss:0.007 grdn:0.158 lr:7.3e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 08:28:59 ts/train.py:232 step:210K smpl:3M ep:16K epch:53.07 loss:0.007 grdn:0.161 lr:7.3e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 08:30:00 ts/train.py:232 step:211K smpl:3M ep:16K epch:53.12 loss:0.006 grdn:0.156 lr:7.3e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 08:31:01 ts/train.py:232 step:211K smpl:3M ep:16K epch:53.17 loss:0.007 grdn:0.165 lr:7.3e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-19 08:32:01 ts/train.py:232 step:211K smpl:3M ep:16K epch:53.22 loss:0.008 grdn:0.178 lr:7.3e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 08:32:58 ts/train.py:232 step:211K smpl:3M ep:16K epch:53.27 loss:0.007 grdn:0.157 lr:7.3e-05 updt_s:0.221 data_s:0.059 +INFO 2025-05-19 08:33:54 ts/train.py:232 step:211K smpl:3M ep:16K epch:53.32 loss:0.008 grdn:0.180 lr:7.2e-05 updt_s:0.221 data_s:0.058 +INFO 2025-05-19 08:34:49 ts/train.py:232 step:212K smpl:3M ep:16K epch:53.37 loss:0.007 grdn:0.165 lr:7.2e-05 updt_s:0.221 data_s:0.056 +INFO 2025-05-19 08:35:44 ts/train.py:232 step:212K smpl:3M ep:16K epch:53.42 loss:0.007 grdn:0.164 lr:7.2e-05 updt_s:0.222 data_s:0.050 +INFO 2025-05-19 08:36:40 ts/train.py:232 step:212K smpl:3M ep:16K epch:53.47 loss:0.007 grdn:0.162 lr:7.2e-05 updt_s:0.221 data_s:0.059 +INFO 2025-05-19 08:37:36 ts/train.py:232 step:212K smpl:3M ep:16K epch:53.52 loss:0.007 grdn:0.167 lr:7.2e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-19 08:38:33 ts/train.py:232 step:212K smpl:3M ep:16K epch:53.57 loss:0.006 grdn:0.159 lr:7.2e-05 updt_s:0.221 data_s:0.065 +INFO 2025-05-19 08:39:31 ts/train.py:232 step:213K smpl:3M ep:16K epch:53.62 loss:0.008 grdn:0.181 lr:7.2e-05 updt_s:0.221 data_s:0.065 +INFO 2025-05-19 08:40:28 ts/train.py:232 step:213K smpl:3M ep:16K epch:53.67 loss:0.008 grdn:0.173 lr:7.2e-05 updt_s:0.221 data_s:0.064 +INFO 2025-05-19 08:41:25 ts/train.py:232 step:213K smpl:3M ep:16K epch:53.72 loss:0.007 grdn:0.169 lr:7.2e-05 updt_s:0.221 data_s:0.065 +INFO 2025-05-19 08:42:23 ts/train.py:232 step:213K smpl:3M ep:16K epch:53.77 loss:0.007 grdn:0.166 lr:7.2e-05 updt_s:0.221 data_s:0.066 +INFO 2025-05-19 08:43:20 ts/train.py:232 step:213K smpl:3M ep:16K epch:53.83 loss:0.007 grdn:0.168 lr:7.2e-05 updt_s:0.221 data_s:0.064 +INFO 2025-05-19 08:44:18 ts/train.py:232 step:214K smpl:3M ep:16K epch:53.88 loss:0.007 grdn:0.169 lr:7.2e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 08:45:15 ts/train.py:232 step:214K smpl:3M ep:16K epch:53.93 loss:0.007 grdn:0.163 lr:7.2e-05 updt_s:0.221 data_s:0.063 +INFO 2025-05-19 08:46:13 ts/train.py:232 step:214K smpl:3M ep:16K epch:53.98 loss:0.007 grdn:0.170 lr:7.2e-05 updt_s:0.221 data_s:0.066 +INFO 2025-05-19 08:47:09 ts/train.py:232 step:214K smpl:3M ep:16K epch:54.03 loss:0.007 grdn:0.163 lr:7.2e-05 updt_s:0.221 data_s:0.059 +INFO 2025-05-19 08:48:07 ts/train.py:232 step:214K smpl:3M ep:16K epch:54.08 loss:0.007 grdn:0.169 lr:7.2e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 08:49:06 ts/train.py:232 step:215K smpl:3M ep:16K epch:54.13 loss:0.007 grdn:0.176 lr:7.2e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 08:50:09 ts/train.py:232 step:215K smpl:3M ep:16K epch:54.18 loss:0.007 grdn:0.164 lr:7.2e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-19 08:51:14 ts/train.py:232 step:215K smpl:3M ep:16K epch:54.23 loss:0.007 grdn:0.164 lr:7.2e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 08:52:19 ts/train.py:232 step:215K smpl:3M ep:16K epch:54.28 loss:0.008 grdn:0.170 lr:7.2e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 08:53:23 ts/train.py:232 step:215K smpl:3M ep:16K epch:54.33 loss:0.006 grdn:0.154 lr:7.2e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 08:54:27 ts/train.py:232 step:216K smpl:3M ep:16K epch:54.38 loss:0.006 grdn:0.157 lr:7.1e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 08:55:31 ts/train.py:232 step:216K smpl:3M ep:16K epch:54.43 loss:0.007 grdn:0.169 lr:7.1e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 08:56:34 ts/train.py:232 step:216K smpl:3M ep:16K epch:54.48 loss:0.007 grdn:0.165 lr:7.1e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 08:57:33 ts/train.py:232 step:216K smpl:3M ep:16K epch:54.53 loss:0.007 grdn:0.164 lr:7.1e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 08:58:32 ts/train.py:232 step:216K smpl:3M ep:16K epch:54.58 loss:0.007 grdn:0.167 lr:7.1e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 08:59:33 ts/train.py:232 step:217K smpl:3M ep:16K epch:54.63 loss:0.007 grdn:0.173 lr:7.1e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 09:00:33 ts/train.py:232 step:217K smpl:3M ep:16K epch:54.68 loss:0.007 grdn:0.161 lr:7.1e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 09:01:33 ts/train.py:232 step:217K smpl:3M ep:16K epch:54.73 loss:0.007 grdn:0.158 lr:7.1e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 09:02:33 ts/train.py:232 step:217K smpl:3M ep:16K epch:54.78 loss:0.007 grdn:0.165 lr:7.1e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 09:03:33 ts/train.py:232 step:217K smpl:3M ep:16K epch:54.83 loss:0.008 grdn:0.173 lr:7.1e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 09:04:31 ts/train.py:232 step:218K smpl:3M ep:16K epch:54.88 loss:0.006 grdn:0.145 lr:7.1e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 09:05:30 ts/train.py:232 step:218K smpl:3M ep:16K epch:54.93 loss:0.007 grdn:0.170 lr:7.1e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 09:06:31 ts/train.py:232 step:218K smpl:3M ep:16K epch:54.99 loss:0.007 grdn:0.149 lr:7.1e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 09:07:31 ts/train.py:232 step:218K smpl:3M ep:17K epch:55.04 loss:0.006 grdn:0.152 lr:7.1e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 09:08:31 ts/train.py:232 step:218K smpl:3M ep:17K epch:55.09 loss:0.006 grdn:0.154 lr:7.1e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 09:09:33 ts/train.py:232 step:219K smpl:3M ep:17K epch:55.14 loss:0.007 grdn:0.163 lr:7.1e-05 updt_s:0.220 data_s:0.093 +INFO 2025-05-19 09:10:36 ts/train.py:232 step:219K smpl:4M ep:17K epch:55.19 loss:0.007 grdn:0.168 lr:7.1e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 09:11:39 ts/train.py:232 step:219K smpl:4M ep:17K epch:55.24 loss:0.007 grdn:0.176 lr:7.1e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 09:12:43 ts/train.py:232 step:219K smpl:4M ep:17K epch:55.29 loss:0.007 grdn:0.167 lr:7.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 09:13:47 ts/train.py:232 step:219K smpl:4M ep:17K epch:55.34 loss:0.007 grdn:0.161 lr:7.1e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 09:14:51 ts/train.py:232 step:220K smpl:4M ep:17K epch:55.39 loss:0.007 grdn:0.164 lr:7.1e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 09:15:55 ts/train.py:232 step:220K smpl:4M ep:17K epch:55.44 loss:0.007 grdn:0.163 lr:7.0e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 09:16:55 ts/train.py:232 step:220K smpl:4M ep:17K epch:55.49 loss:0.006 grdn:0.158 lr:7.0e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 09:16:55 ts/train.py:241 Checkpoint policy after step 220000 +INFO 2025-05-19 09:18:00 ts/train.py:232 step:220K smpl:4M ep:17K epch:55.54 loss:0.006 grdn:0.156 lr:7.0e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 09:19:03 ts/train.py:232 step:220K smpl:4M ep:17K epch:55.59 loss:0.007 grdn:0.160 lr:7.0e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 09:20:05 ts/train.py:232 step:221K smpl:4M ep:17K epch:55.64 loss:0.007 grdn:0.157 lr:7.0e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 09:21:07 ts/train.py:232 step:221K smpl:4M ep:17K epch:55.69 loss:0.007 grdn:0.165 lr:7.0e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 09:22:10 ts/train.py:232 step:221K smpl:4M ep:17K epch:55.74 loss:0.007 grdn:0.165 lr:7.0e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 09:23:11 ts/train.py:232 step:221K smpl:4M ep:17K epch:55.79 loss:0.007 grdn:0.170 lr:7.0e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 09:24:13 ts/train.py:232 step:221K smpl:4M ep:17K epch:55.84 loss:0.007 grdn:0.163 lr:7.0e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 09:25:14 ts/train.py:232 step:222K smpl:4M ep:17K epch:55.89 loss:0.007 grdn:0.161 lr:7.0e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 09:26:15 ts/train.py:232 step:222K smpl:4M ep:17K epch:55.94 loss:0.007 grdn:0.159 lr:7.0e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 09:27:17 ts/train.py:232 step:222K smpl:4M ep:17K epch:55.99 loss:0.007 grdn:0.171 lr:7.0e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 09:28:19 ts/train.py:232 step:222K smpl:4M ep:17K epch:56.04 loss:0.007 grdn:0.164 lr:7.0e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 09:29:24 ts/train.py:232 step:222K smpl:4M ep:17K epch:56.10 loss:0.007 grdn:0.171 lr:7.0e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 09:30:27 ts/train.py:232 step:223K smpl:4M ep:17K epch:56.15 loss:0.006 grdn:0.156 lr:7.0e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 09:31:30 ts/train.py:232 step:223K smpl:4M ep:17K epch:56.20 loss:0.006 grdn:0.156 lr:7.0e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 09:32:32 ts/train.py:232 step:223K smpl:4M ep:17K epch:56.25 loss:0.007 grdn:0.165 lr:7.0e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 09:33:35 ts/train.py:232 step:223K smpl:4M ep:17K epch:56.30 loss:0.007 grdn:0.165 lr:7.0e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 09:34:39 ts/train.py:232 step:223K smpl:4M ep:17K epch:56.35 loss:0.007 grdn:0.165 lr:7.0e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 09:35:42 ts/train.py:232 step:224K smpl:4M ep:17K epch:56.40 loss:0.006 grdn:0.163 lr:7.0e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 09:36:42 ts/train.py:232 step:224K smpl:4M ep:17K epch:56.45 loss:0.007 grdn:0.169 lr:7.0e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 09:37:43 ts/train.py:232 step:224K smpl:4M ep:17K epch:56.50 loss:0.007 grdn:0.171 lr:6.9e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 09:38:42 ts/train.py:232 step:224K smpl:4M ep:17K epch:56.55 loss:0.006 grdn:0.164 lr:6.9e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 09:39:42 ts/train.py:232 step:224K smpl:4M ep:17K epch:56.60 loss:0.007 grdn:0.165 lr:6.9e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 09:40:42 ts/train.py:232 step:225K smpl:4M ep:17K epch:56.65 loss:0.006 grdn:0.154 lr:6.9e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 09:41:41 ts/train.py:232 step:225K smpl:4M ep:17K epch:56.70 loss:0.006 grdn:0.160 lr:6.9e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 09:42:41 ts/train.py:232 step:225K smpl:4M ep:17K epch:56.75 loss:0.007 grdn:0.158 lr:6.9e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 09:43:41 ts/train.py:232 step:225K smpl:4M ep:17K epch:56.80 loss:0.007 grdn:0.172 lr:6.9e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 09:44:40 ts/train.py:232 step:225K smpl:4M ep:17K epch:56.85 loss:0.006 grdn:0.153 lr:6.9e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 09:45:40 ts/train.py:232 step:226K smpl:4M ep:17K epch:56.90 loss:0.007 grdn:0.163 lr:6.9e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 09:46:40 ts/train.py:232 step:226K smpl:4M ep:17K epch:56.95 loss:0.006 grdn:0.154 lr:6.9e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 09:47:39 ts/train.py:232 step:226K smpl:4M ep:17K epch:57.00 loss:0.007 grdn:0.162 lr:6.9e-05 updt_s:0.222 data_s:0.076 +INFO 2025-05-19 09:48:38 ts/train.py:232 step:226K smpl:4M ep:17K epch:57.05 loss:0.006 grdn:0.159 lr:6.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 09:49:43 ts/train.py:232 step:226K smpl:4M ep:17K epch:57.10 loss:0.006 grdn:0.161 lr:6.9e-05 updt_s:0.220 data_s:0.106 +INFO 2025-05-19 09:50:47 ts/train.py:232 step:227K smpl:4M ep:17K epch:57.15 loss:0.007 grdn:0.173 lr:6.9e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 09:51:51 ts/train.py:232 step:227K smpl:4M ep:17K epch:57.21 loss:0.007 grdn:0.160 lr:6.9e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 09:52:55 ts/train.py:232 step:227K smpl:4M ep:17K epch:57.26 loss:0.007 grdn:0.174 lr:6.9e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 09:54:00 ts/train.py:232 step:227K smpl:4M ep:17K epch:57.31 loss:0.006 grdn:0.155 lr:6.9e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 09:55:04 ts/train.py:232 step:227K smpl:4M ep:17K epch:57.36 loss:0.007 grdn:0.169 lr:6.9e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 09:56:08 ts/train.py:232 step:228K smpl:4M ep:17K epch:57.41 loss:0.007 grdn:0.164 lr:6.9e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 09:57:11 ts/train.py:232 step:228K smpl:4M ep:17K epch:57.46 loss:0.007 grdn:0.161 lr:6.9e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 09:58:15 ts/train.py:232 step:228K smpl:4M ep:17K epch:57.51 loss:0.007 grdn:0.163 lr:6.9e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 09:59:19 ts/train.py:232 step:228K smpl:4M ep:17K epch:57.56 loss:0.008 grdn:0.178 lr:6.8e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 10:00:22 ts/train.py:232 step:228K smpl:4M ep:17K epch:57.61 loss:0.006 grdn:0.161 lr:6.8e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 10:01:27 ts/train.py:232 step:229K smpl:4M ep:17K epch:57.66 loss:0.007 grdn:0.172 lr:6.8e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 10:02:31 ts/train.py:232 step:229K smpl:4M ep:17K epch:57.71 loss:0.007 grdn:0.163 lr:6.8e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 10:03:35 ts/train.py:232 step:229K smpl:4M ep:17K epch:57.76 loss:0.007 grdn:0.161 lr:6.8e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 10:04:39 ts/train.py:232 step:229K smpl:4M ep:17K epch:57.81 loss:0.007 grdn:0.173 lr:6.8e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 10:05:44 ts/train.py:232 step:229K smpl:4M ep:17K epch:57.86 loss:0.007 grdn:0.165 lr:6.8e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 10:06:48 ts/train.py:232 step:230K smpl:4M ep:17K epch:57.91 loss:0.007 grdn:0.171 lr:6.8e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 10:07:53 ts/train.py:232 step:230K smpl:4M ep:17K epch:57.96 loss:0.007 grdn:0.163 lr:6.8e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 10:08:57 ts/train.py:232 step:230K smpl:4M ep:17K epch:58.01 loss:0.006 grdn:0.160 lr:6.8e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 10:10:02 ts/train.py:232 step:230K smpl:4M ep:17K epch:58.06 loss:0.005 grdn:0.147 lr:6.8e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 10:11:07 ts/train.py:232 step:230K smpl:4M ep:17K epch:58.11 loss:0.007 grdn:0.172 lr:6.8e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 10:12:12 ts/train.py:232 step:231K smpl:4M ep:17K epch:58.16 loss:0.007 grdn:0.166 lr:6.8e-05 updt_s:0.222 data_s:0.104 +INFO 2025-05-19 10:13:17 ts/train.py:232 step:231K smpl:4M ep:17K epch:58.21 loss:0.007 grdn:0.161 lr:6.8e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 10:14:22 ts/train.py:232 step:231K smpl:4M ep:17K epch:58.26 loss:0.006 grdn:0.158 lr:6.8e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-19 10:15:27 ts/train.py:232 step:231K smpl:4M ep:17K epch:58.31 loss:0.007 grdn:0.165 lr:6.8e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 10:16:30 ts/train.py:232 step:231K smpl:4M ep:18K epch:58.37 loss:0.006 grdn:0.149 lr:6.8e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 10:17:32 ts/train.py:232 step:232K smpl:4M ep:18K epch:58.42 loss:0.007 grdn:0.159 lr:6.8e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 10:18:32 ts/train.py:232 step:232K smpl:4M ep:18K epch:58.47 loss:0.006 grdn:0.162 lr:6.8e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 10:19:35 ts/train.py:232 step:232K smpl:4M ep:18K epch:58.52 loss:0.007 grdn:0.174 lr:6.8e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 10:20:35 ts/train.py:232 step:232K smpl:4M ep:18K epch:58.57 loss:0.007 grdn:0.166 lr:6.7e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 10:21:37 ts/train.py:232 step:232K smpl:4M ep:18K epch:58.62 loss:0.006 grdn:0.155 lr:6.7e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 10:22:37 ts/train.py:232 step:233K smpl:4M ep:18K epch:58.67 loss:0.006 grdn:0.153 lr:6.7e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 10:23:39 ts/train.py:232 step:233K smpl:4M ep:18K epch:58.72 loss:0.007 grdn:0.172 lr:6.7e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 10:24:40 ts/train.py:232 step:233K smpl:4M ep:18K epch:58.77 loss:0.006 grdn:0.151 lr:6.7e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 10:25:40 ts/train.py:232 step:233K smpl:4M ep:18K epch:58.82 loss:0.007 grdn:0.173 lr:6.7e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 10:26:41 ts/train.py:232 step:233K smpl:4M ep:18K epch:58.87 loss:0.007 grdn:0.176 lr:6.7e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 10:27:42 ts/train.py:232 step:234K smpl:4M ep:18K epch:58.92 loss:0.007 grdn:0.165 lr:6.7e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 10:28:43 ts/train.py:232 step:234K smpl:4M ep:18K epch:58.97 loss:0.007 grdn:0.164 lr:6.7e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 10:29:50 ts/train.py:232 step:234K smpl:4M ep:18K epch:59.02 loss:0.006 grdn:0.149 lr:6.7e-05 updt_s:0.221 data_s:0.116 +INFO 2025-05-19 10:30:57 ts/train.py:232 step:234K smpl:4M ep:18K epch:59.07 loss:0.007 grdn:0.174 lr:6.7e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-19 10:32:03 ts/train.py:232 step:234K smpl:4M ep:18K epch:59.12 loss:0.007 grdn:0.159 lr:6.7e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 10:33:09 ts/train.py:232 step:235K smpl:4M ep:18K epch:59.17 loss:0.006 grdn:0.164 lr:6.7e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 10:34:15 ts/train.py:232 step:235K smpl:4M ep:18K epch:59.22 loss:0.007 grdn:0.171 lr:6.7e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-19 10:35:21 ts/train.py:232 step:235K smpl:4M ep:18K epch:59.27 loss:0.007 grdn:0.173 lr:6.7e-05 updt_s:0.221 data_s:0.109 +INFO 2025-05-19 10:36:28 ts/train.py:232 step:235K smpl:4M ep:18K epch:59.32 loss:0.007 grdn:0.163 lr:6.7e-05 updt_s:0.221 data_s:0.109 +INFO 2025-05-19 10:37:34 ts/train.py:232 step:235K smpl:4M ep:18K epch:59.37 loss:0.007 grdn:0.161 lr:6.7e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-19 10:38:40 ts/train.py:232 step:236K smpl:4M ep:18K epch:59.42 loss:0.006 grdn:0.155 lr:6.7e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 10:39:45 ts/train.py:232 step:236K smpl:4M ep:18K epch:59.48 loss:0.007 grdn:0.168 lr:6.7e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 10:40:50 ts/train.py:232 step:236K smpl:4M ep:18K epch:59.53 loss:0.006 grdn:0.166 lr:6.7e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 10:41:56 ts/train.py:232 step:236K smpl:4M ep:18K epch:59.58 loss:0.007 grdn:0.169 lr:6.6e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 10:43:02 ts/train.py:232 step:236K smpl:4M ep:18K epch:59.63 loss:0.007 grdn:0.163 lr:6.6e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 10:44:08 ts/train.py:232 step:237K smpl:4M ep:18K epch:59.68 loss:0.006 grdn:0.162 lr:6.6e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 10:45:13 ts/train.py:232 step:237K smpl:4M ep:18K epch:59.73 loss:0.006 grdn:0.153 lr:6.6e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 10:46:19 ts/train.py:232 step:237K smpl:4M ep:18K epch:59.78 loss:0.007 grdn:0.165 lr:6.6e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 10:47:25 ts/train.py:232 step:237K smpl:4M ep:18K epch:59.83 loss:0.007 grdn:0.164 lr:6.6e-05 updt_s:0.221 data_s:0.109 +INFO 2025-05-19 10:48:32 ts/train.py:232 step:237K smpl:4M ep:18K epch:59.88 loss:0.006 grdn:0.156 lr:6.6e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-19 10:49:38 ts/train.py:232 step:238K smpl:4M ep:18K epch:59.93 loss:0.006 grdn:0.162 lr:6.6e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 10:50:45 ts/train.py:232 step:238K smpl:4M ep:18K epch:59.98 loss:0.006 grdn:0.170 lr:6.6e-05 updt_s:0.220 data_s:0.115 +INFO 2025-05-19 10:51:50 ts/train.py:232 step:238K smpl:4M ep:18K epch:60.03 loss:0.006 grdn:0.164 lr:6.6e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-19 10:52:56 ts/train.py:232 step:238K smpl:4M ep:18K epch:60.08 loss:0.006 grdn:0.161 lr:6.6e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 10:54:02 ts/train.py:232 step:238K smpl:4M ep:18K epch:60.13 loss:0.006 grdn:0.157 lr:6.6e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 10:55:07 ts/train.py:232 step:239K smpl:4M ep:18K epch:60.18 loss:0.006 grdn:0.159 lr:6.6e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 10:56:12 ts/train.py:232 step:239K smpl:4M ep:18K epch:60.23 loss:0.007 grdn:0.162 lr:6.6e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 10:57:18 ts/train.py:232 step:239K smpl:4M ep:18K epch:60.28 loss:0.007 grdn:0.169 lr:6.6e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-19 10:58:24 ts/train.py:232 step:239K smpl:4M ep:18K epch:60.33 loss:0.006 grdn:0.152 lr:6.6e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 10:59:30 ts/train.py:232 step:239K smpl:4M ep:18K epch:60.38 loss:0.007 grdn:0.172 lr:6.6e-05 updt_s:0.221 data_s:0.109 +INFO 2025-05-19 11:00:36 ts/train.py:232 step:240K smpl:4M ep:18K epch:60.43 loss:0.006 grdn:0.160 lr:6.6e-05 updt_s:0.222 data_s:0.108 +INFO 2025-05-19 11:01:41 ts/train.py:232 step:240K smpl:4M ep:18K epch:60.48 loss:0.006 grdn:0.157 lr:6.6e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 11:02:46 ts/train.py:232 step:240K smpl:4M ep:18K epch:60.53 loss:0.007 grdn:0.156 lr:6.6e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 11:02:46 ts/train.py:241 Checkpoint policy after step 240000 +INFO 2025-05-19 11:03:55 ts/train.py:232 step:240K smpl:4M ep:18K epch:60.58 loss:0.007 grdn:0.172 lr:6.6e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 11:05:00 ts/train.py:232 step:240K smpl:4M ep:18K epch:60.64 loss:0.007 grdn:0.161 lr:6.5e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 11:06:06 ts/train.py:232 step:241K smpl:4M ep:18K epch:60.69 loss:0.006 grdn:0.158 lr:6.5e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 11:07:10 ts/train.py:232 step:241K smpl:4M ep:18K epch:60.74 loss:0.007 grdn:0.164 lr:6.5e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 11:08:17 ts/train.py:232 step:241K smpl:4M ep:18K epch:60.79 loss:0.006 grdn:0.155 lr:6.5e-05 updt_s:0.221 data_s:0.109 +INFO 2025-05-19 11:09:21 ts/train.py:232 step:241K smpl:4M ep:18K epch:60.84 loss:0.006 grdn:0.168 lr:6.5e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 11:10:27 ts/train.py:232 step:241K smpl:4M ep:18K epch:60.89 loss:0.006 grdn:0.162 lr:6.5e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 11:11:34 ts/train.py:232 step:242K smpl:4M ep:18K epch:60.94 loss:0.007 grdn:0.167 lr:6.5e-05 updt_s:0.221 data_s:0.113 +INFO 2025-05-19 11:12:37 ts/train.py:232 step:242K smpl:4M ep:18K epch:60.99 loss:0.006 grdn:0.146 lr:6.5e-05 updt_s:0.222 data_s:0.094 +INFO 2025-05-19 11:13:42 ts/train.py:232 step:242K smpl:4M ep:18K epch:61.04 loss:0.007 grdn:0.165 lr:6.5e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 11:14:47 ts/train.py:232 step:242K smpl:4M ep:18K epch:61.09 loss:0.007 grdn:0.170 lr:6.5e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 11:15:52 ts/train.py:232 step:242K smpl:4M ep:18K epch:61.14 loss:0.007 grdn:0.164 lr:6.5e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 11:16:57 ts/train.py:232 step:243K smpl:4M ep:18K epch:61.19 loss:0.007 grdn:0.173 lr:6.5e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 11:18:02 ts/train.py:232 step:243K smpl:4M ep:18K epch:61.24 loss:0.006 grdn:0.160 lr:6.5e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-19 11:19:03 ts/train.py:232 step:243K smpl:4M ep:18K epch:61.29 loss:0.007 grdn:0.177 lr:6.5e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 11:20:04 ts/train.py:232 step:243K smpl:4M ep:18K epch:61.34 loss:0.007 grdn:0.165 lr:6.5e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 11:21:04 ts/train.py:232 step:243K smpl:4M ep:18K epch:61.39 loss:0.007 grdn:0.160 lr:6.5e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 11:22:06 ts/train.py:232 step:244K smpl:4M ep:18K epch:61.44 loss:0.007 grdn:0.168 lr:6.5e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 11:23:06 ts/train.py:232 step:244K smpl:4M ep:18K epch:61.49 loss:0.007 grdn:0.163 lr:6.5e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 11:24:08 ts/train.py:232 step:244K smpl:4M ep:18K epch:61.54 loss:0.006 grdn:0.151 lr:6.5e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 11:25:09 ts/train.py:232 step:244K smpl:4M ep:18K epch:61.59 loss:0.006 grdn:0.162 lr:6.5e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 11:26:10 ts/train.py:232 step:244K smpl:4M ep:18K epch:61.64 loss:0.007 grdn:0.169 lr:6.4e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 11:27:10 ts/train.py:232 step:245K smpl:4M ep:19K epch:61.69 loss:0.006 grdn:0.156 lr:6.4e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 11:28:11 ts/train.py:232 step:245K smpl:4M ep:19K epch:61.75 loss:0.007 grdn:0.171 lr:6.4e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 11:29:11 ts/train.py:232 step:245K smpl:4M ep:19K epch:61.80 loss:0.006 grdn:0.167 lr:6.4e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 11:30:12 ts/train.py:232 step:245K smpl:4M ep:19K epch:61.85 loss:0.007 grdn:0.170 lr:6.4e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 11:31:16 ts/train.py:232 step:245K smpl:4M ep:19K epch:61.90 loss:0.007 grdn:0.162 lr:6.4e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 11:32:22 ts/train.py:232 step:246K smpl:4M ep:19K epch:61.95 loss:0.006 grdn:0.167 lr:6.4e-05 updt_s:0.221 data_s:0.110 +INFO 2025-05-19 11:33:27 ts/train.py:232 step:246K smpl:4M ep:19K epch:62.00 loss:0.006 grdn:0.168 lr:6.4e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 11:34:32 ts/train.py:232 step:246K smpl:4M ep:19K epch:62.05 loss:0.006 grdn:0.154 lr:6.4e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 11:35:38 ts/train.py:232 step:246K smpl:4M ep:19K epch:62.10 loss:0.007 grdn:0.167 lr:6.4e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 11:36:43 ts/train.py:232 step:246K smpl:4M ep:19K epch:62.15 loss:0.006 grdn:0.159 lr:6.4e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 11:37:48 ts/train.py:232 step:247K smpl:4M ep:19K epch:62.20 loss:0.007 grdn:0.164 lr:6.4e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 11:38:52 ts/train.py:232 step:247K smpl:4M ep:19K epch:62.25 loss:0.006 grdn:0.169 lr:6.4e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 11:39:57 ts/train.py:232 step:247K smpl:4M ep:19K epch:62.30 loss:0.006 grdn:0.157 lr:6.4e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 11:41:01 ts/train.py:232 step:247K smpl:4M ep:19K epch:62.35 loss:0.006 grdn:0.156 lr:6.4e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 11:42:05 ts/train.py:232 step:247K smpl:4M ep:19K epch:62.40 loss:0.006 grdn:0.157 lr:6.4e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 11:43:09 ts/train.py:232 step:248K smpl:4M ep:19K epch:62.45 loss:0.006 grdn:0.166 lr:6.4e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 11:44:13 ts/train.py:232 step:248K smpl:4M ep:19K epch:62.50 loss:0.007 grdn:0.169 lr:6.4e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 11:45:15 ts/train.py:232 step:248K smpl:4M ep:19K epch:62.55 loss:0.006 grdn:0.156 lr:6.4e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 11:46:19 ts/train.py:232 step:248K smpl:4M ep:19K epch:62.60 loss:0.007 grdn:0.166 lr:6.3e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 11:47:22 ts/train.py:232 step:248K smpl:4M ep:19K epch:62.65 loss:0.006 grdn:0.149 lr:6.3e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 11:48:25 ts/train.py:232 step:249K smpl:4M ep:19K epch:62.70 loss:0.006 grdn:0.165 lr:6.3e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 11:49:27 ts/train.py:232 step:249K smpl:4M ep:19K epch:62.75 loss:0.006 grdn:0.153 lr:6.3e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-19 11:50:32 ts/train.py:232 step:249K smpl:4M ep:19K epch:62.80 loss:0.006 grdn:0.167 lr:6.3e-05 updt_s:0.222 data_s:0.101 +INFO 2025-05-19 11:51:36 ts/train.py:232 step:249K smpl:4M ep:19K epch:62.85 loss:0.006 grdn:0.156 lr:6.3e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 11:52:41 ts/train.py:232 step:249K smpl:4M ep:19K epch:62.91 loss:0.006 grdn:0.167 lr:6.3e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 11:53:45 ts/train.py:232 step:250K smpl:4M ep:19K epch:62.96 loss:0.006 grdn:0.170 lr:6.3e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 11:54:49 ts/train.py:232 step:250K smpl:4M ep:19K epch:63.01 loss:0.006 grdn:0.165 lr:6.3e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 11:55:53 ts/train.py:232 step:250K smpl:4M ep:19K epch:63.06 loss:0.006 grdn:0.161 lr:6.3e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 11:56:55 ts/train.py:232 step:250K smpl:4M ep:19K epch:63.11 loss:0.006 grdn:0.164 lr:6.3e-05 updt_s:0.222 data_s:0.091 +INFO 2025-05-19 11:57:59 ts/train.py:232 step:250K smpl:4M ep:19K epch:63.16 loss:0.006 grdn:0.157 lr:6.3e-05 updt_s:0.223 data_s:0.096 +INFO 2025-05-19 11:59:00 ts/train.py:232 step:251K smpl:4M ep:19K epch:63.21 loss:0.006 grdn:0.162 lr:6.3e-05 updt_s:0.223 data_s:0.080 +INFO 2025-05-19 12:00:00 ts/train.py:232 step:251K smpl:4M ep:19K epch:63.26 loss:0.006 grdn:0.160 lr:6.3e-05 updt_s:0.223 data_s:0.076 +INFO 2025-05-19 12:00:59 ts/train.py:232 step:251K smpl:4M ep:19K epch:63.31 loss:0.006 grdn:0.158 lr:6.3e-05 updt_s:0.223 data_s:0.071 +INFO 2025-05-19 12:01:58 ts/train.py:232 step:251K smpl:4M ep:19K epch:63.36 loss:0.006 grdn:0.161 lr:6.3e-05 updt_s:0.222 data_s:0.070 +INFO 2025-05-19 12:02:59 ts/train.py:232 step:251K smpl:4M ep:19K epch:63.41 loss:0.006 grdn:0.172 lr:6.3e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-19 12:03:59 ts/train.py:232 step:252K smpl:4M ep:19K epch:63.46 loss:0.007 grdn:0.178 lr:6.3e-05 updt_s:0.222 data_s:0.074 +INFO 2025-05-19 12:05:00 ts/train.py:232 step:252K smpl:4M ep:19K epch:63.51 loss:0.006 grdn:0.163 lr:6.3e-05 updt_s:0.222 data_s:0.086 +INFO 2025-05-19 12:06:01 ts/train.py:232 step:252K smpl:4M ep:19K epch:63.56 loss:0.006 grdn:0.150 lr:6.3e-05 updt_s:0.222 data_s:0.078 +INFO 2025-05-19 12:07:01 ts/train.py:232 step:252K smpl:4M ep:19K epch:63.61 loss:0.006 grdn:0.162 lr:6.2e-05 updt_s:0.222 data_s:0.078 +INFO 2025-05-19 12:08:00 ts/train.py:232 step:252K smpl:4M ep:19K epch:63.66 loss:0.005 grdn:0.151 lr:6.2e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 12:08:58 ts/train.py:232 step:253K smpl:4M ep:19K epch:63.71 loss:0.007 grdn:0.172 lr:6.2e-05 updt_s:0.222 data_s:0.068 +INFO 2025-05-19 12:09:58 ts/train.py:232 step:253K smpl:4M ep:19K epch:63.76 loss:0.006 grdn:0.161 lr:6.2e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 12:10:59 ts/train.py:232 step:253K smpl:4M ep:19K epch:63.81 loss:0.007 grdn:0.162 lr:6.2e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 12:12:04 ts/train.py:232 step:253K smpl:4M ep:19K epch:63.86 loss:0.006 grdn:0.168 lr:6.2e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 12:13:09 ts/train.py:232 step:253K smpl:4M ep:19K epch:63.91 loss:0.007 grdn:0.167 lr:6.2e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 12:14:14 ts/train.py:232 step:254K smpl:4M ep:19K epch:63.96 loss:0.006 grdn:0.159 lr:6.2e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-19 12:15:18 ts/train.py:232 step:254K smpl:4M ep:19K epch:64.02 loss:0.006 grdn:0.153 lr:6.2e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 12:16:23 ts/train.py:232 step:254K smpl:4M ep:19K epch:64.07 loss:0.005 grdn:0.154 lr:6.2e-05 updt_s:0.222 data_s:0.102 +INFO 2025-05-19 12:17:28 ts/train.py:232 step:254K smpl:4M ep:19K epch:64.12 loss:0.006 grdn:0.167 lr:6.2e-05 updt_s:0.222 data_s:0.099 +INFO 2025-05-19 12:18:31 ts/train.py:232 step:254K smpl:4M ep:19K epch:64.17 loss:0.006 grdn:0.166 lr:6.2e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-19 12:19:32 ts/train.py:232 step:255K smpl:4M ep:19K epch:64.22 loss:0.007 grdn:0.164 lr:6.2e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 12:20:33 ts/train.py:232 step:255K smpl:4M ep:19K epch:64.27 loss:0.006 grdn:0.162 lr:6.2e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 12:21:33 ts/train.py:232 step:255K smpl:4M ep:19K epch:64.32 loss:0.006 grdn:0.162 lr:6.2e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 12:22:32 ts/train.py:232 step:255K smpl:4M ep:19K epch:64.37 loss:0.006 grdn:0.148 lr:6.2e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 12:23:32 ts/train.py:232 step:255K smpl:4M ep:19K epch:64.42 loss:0.006 grdn:0.156 lr:6.2e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 12:24:31 ts/train.py:232 step:256K smpl:4M ep:19K epch:64.47 loss:0.006 grdn:0.153 lr:6.2e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 12:25:30 ts/train.py:232 step:256K smpl:4M ep:19K epch:64.52 loss:0.006 grdn:0.157 lr:6.2e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 12:26:30 ts/train.py:232 step:256K smpl:4M ep:19K epch:64.57 loss:0.006 grdn:0.157 lr:6.2e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 12:27:28 ts/train.py:232 step:256K smpl:4M ep:19K epch:64.62 loss:0.006 grdn:0.157 lr:6.1e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 12:28:27 ts/train.py:232 step:256K smpl:4M ep:19K epch:64.67 loss:0.006 grdn:0.161 lr:6.1e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 12:29:26 ts/train.py:232 step:257K smpl:4M ep:19K epch:64.72 loss:0.006 grdn:0.156 lr:6.1e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 12:30:25 ts/train.py:232 step:257K smpl:4M ep:19K epch:64.77 loss:0.006 grdn:0.161 lr:6.1e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 12:31:31 ts/train.py:232 step:257K smpl:4M ep:19K epch:64.82 loss:0.006 grdn:0.150 lr:6.1e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 12:32:36 ts/train.py:232 step:257K smpl:4M ep:19K epch:64.87 loss:0.006 grdn:0.155 lr:6.1e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 12:33:42 ts/train.py:232 step:257K smpl:4M ep:19K epch:64.92 loss:0.006 grdn:0.165 lr:6.1e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 12:34:47 ts/train.py:232 step:258K smpl:4M ep:19K epch:64.97 loss:0.007 grdn:0.167 lr:6.1e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 12:35:53 ts/train.py:232 step:258K smpl:4M ep:20K epch:65.02 loss:0.006 grdn:0.149 lr:6.1e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 12:36:58 ts/train.py:232 step:258K smpl:4M ep:20K epch:65.07 loss:0.006 grdn:0.161 lr:6.1e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 12:38:01 ts/train.py:232 step:258K smpl:4M ep:20K epch:65.12 loss:0.007 grdn:0.166 lr:6.1e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-19 12:39:04 ts/train.py:232 step:258K smpl:4M ep:20K epch:65.18 loss:0.006 grdn:0.164 lr:6.1e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 12:40:08 ts/train.py:232 step:259K smpl:4M ep:20K epch:65.23 loss:0.006 grdn:0.162 lr:6.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 12:41:12 ts/train.py:232 step:259K smpl:4M ep:20K epch:65.28 loss:0.006 grdn:0.159 lr:6.1e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 12:42:15 ts/train.py:232 step:259K smpl:4M ep:20K epch:65.33 loss:0.006 grdn:0.152 lr:6.1e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 12:43:19 ts/train.py:232 step:259K smpl:4M ep:20K epch:65.38 loss:0.006 grdn:0.159 lr:6.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 12:44:22 ts/train.py:232 step:259K smpl:4M ep:20K epch:65.43 loss:0.006 grdn:0.158 lr:6.1e-05 updt_s:0.222 data_s:0.096 +INFO 2025-05-19 12:45:27 ts/train.py:232 step:260K smpl:4M ep:20K epch:65.48 loss:0.006 grdn:0.158 lr:6.1e-05 updt_s:0.222 data_s:0.102 +INFO 2025-05-19 12:46:30 ts/train.py:232 step:260K smpl:4M ep:20K epch:65.53 loss:0.007 grdn:0.176 lr:6.1e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 12:47:34 ts/train.py:232 step:260K smpl:4M ep:20K epch:65.58 loss:0.006 grdn:0.160 lr:6.0e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 12:47:34 ts/train.py:241 Checkpoint policy after step 260000 +INFO 2025-05-19 12:48:52 ts/train.py:232 step:260K smpl:4M ep:20K epch:65.63 loss:0.006 grdn:0.157 lr:6.0e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 12:49:55 ts/train.py:232 step:260K smpl:4M ep:20K epch:65.68 loss:0.006 grdn:0.163 lr:6.0e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 12:50:58 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.73 loss:0.006 grdn:0.161 lr:6.0e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 12:52:03 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.78 loss:0.007 grdn:0.173 lr:6.0e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 12:53:07 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.83 loss:0.007 grdn:0.166 lr:6.0e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 12:54:10 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.88 loss:0.006 grdn:0.158 lr:6.0e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 12:55:14 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.93 loss:0.006 grdn:0.168 lr:6.0e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 12:56:19 ts/train.py:232 step:262K smpl:4M ep:20K epch:65.98 loss:0.006 grdn:0.154 lr:6.0e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 12:57:23 ts/train.py:232 step:262K smpl:4M ep:20K epch:66.03 loss:0.006 grdn:0.161 lr:6.0e-05 updt_s:0.222 data_s:0.100 +INFO 2025-05-19 12:58:28 ts/train.py:232 step:262K smpl:4M ep:20K epch:66.08 loss:0.006 grdn:0.167 lr:6.0e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 12:59:33 ts/train.py:232 step:262K smpl:4M ep:20K epch:66.13 loss:0.005 grdn:0.154 lr:6.0e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 13:00:37 ts/train.py:232 step:262K smpl:4M ep:20K epch:66.18 loss:0.006 grdn:0.168 lr:6.0e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 13:01:43 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.23 loss:0.006 grdn:0.163 lr:6.0e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 13:02:48 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.29 loss:0.006 grdn:0.169 lr:6.0e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 13:03:54 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.34 loss:0.006 grdn:0.168 lr:6.0e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 13:04:58 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.39 loss:0.006 grdn:0.166 lr:6.0e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 13:06:03 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.44 loss:0.006 grdn:0.162 lr:6.0e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 13:07:08 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.49 loss:0.007 grdn:0.166 lr:6.0e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 13:08:14 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.54 loss:0.006 grdn:0.160 lr:6.0e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 13:09:26 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.59 loss:0.006 grdn:0.159 lr:5.9e-05 updt_s:0.221 data_s:0.139 +INFO 2025-05-19 13:10:37 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.64 loss:0.005 grdn:0.147 lr:5.9e-05 updt_s:0.221 data_s:0.134 +INFO 2025-05-19 13:11:47 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.69 loss:0.006 grdn:0.157 lr:5.9e-05 updt_s:0.221 data_s:0.130 +INFO 2025-05-19 13:13:03 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.74 loss:0.006 grdn:0.152 lr:5.9e-05 updt_s:0.221 data_s:0.157 +INFO 2025-05-19 13:14:10 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.79 loss:0.006 grdn:0.158 lr:5.9e-05 updt_s:0.221 data_s:0.115 +INFO 2025-05-19 13:15:19 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.84 loss:0.006 grdn:0.165 lr:5.9e-05 updt_s:0.221 data_s:0.120 +INFO 2025-05-19 13:16:30 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.89 loss:0.006 grdn:0.159 lr:5.9e-05 updt_s:0.221 data_s:0.131 +INFO 2025-05-19 13:17:39 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.94 loss:0.006 grdn:0.147 lr:5.9e-05 updt_s:0.221 data_s:0.127 +INFO 2025-05-19 13:18:49 ts/train.py:232 step:266K smpl:4M ep:20K epch:66.99 loss:0.006 grdn:0.162 lr:5.9e-05 updt_s:0.221 data_s:0.126 +INFO 2025-05-19 13:19:58 ts/train.py:232 step:266K smpl:4M ep:20K epch:67.04 loss:0.006 grdn:0.163 lr:5.9e-05 updt_s:0.221 data_s:0.124 +INFO 2025-05-19 13:21:05 ts/train.py:232 step:266K smpl:4M ep:20K epch:67.09 loss:0.006 grdn:0.154 lr:5.9e-05 updt_s:0.221 data_s:0.117 +INFO 2025-05-19 13:22:14 ts/train.py:232 step:266K smpl:4M ep:20K epch:67.14 loss:0.006 grdn:0.166 lr:5.9e-05 updt_s:0.221 data_s:0.122 +INFO 2025-05-19 13:23:23 ts/train.py:232 step:266K smpl:4M ep:20K epch:67.19 loss:0.005 grdn:0.153 lr:5.9e-05 updt_s:0.221 data_s:0.120 +INFO 2025-05-19 13:24:34 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.24 loss:0.006 grdn:0.156 lr:5.9e-05 updt_s:0.221 data_s:0.136 +INFO 2025-05-19 13:25:44 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.29 loss:0.006 grdn:0.164 lr:5.9e-05 updt_s:0.221 data_s:0.127 +INFO 2025-05-19 13:26:55 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.34 loss:0.006 grdn:0.165 lr:5.9e-05 updt_s:0.221 data_s:0.132 +INFO 2025-05-19 13:28:06 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.39 loss:0.007 grdn:0.174 lr:5.9e-05 updt_s:0.221 data_s:0.132 +INFO 2025-05-19 13:29:14 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.45 loss:0.005 grdn:0.153 lr:5.9e-05 updt_s:0.221 data_s:0.122 +INFO 2025-05-19 13:30:23 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.50 loss:0.006 grdn:0.159 lr:5.9e-05 updt_s:0.221 data_s:0.121 +INFO 2025-05-19 13:31:32 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.55 loss:0.006 grdn:0.170 lr:5.8e-05 updt_s:0.221 data_s:0.123 +INFO 2025-05-19 13:32:40 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.60 loss:0.007 grdn:0.168 lr:5.8e-05 updt_s:0.221 data_s:0.119 +INFO 2025-05-19 13:33:50 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.65 loss:0.006 grdn:0.161 lr:5.8e-05 updt_s:0.221 data_s:0.129 +INFO 2025-05-19 13:35:03 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.70 loss:0.006 grdn:0.154 lr:5.8e-05 updt_s:0.221 data_s:0.145 +INFO 2025-05-19 13:36:13 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.75 loss:0.006 grdn:0.165 lr:5.8e-05 updt_s:0.223 data_s:0.122 +INFO 2025-05-19 13:37:19 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.80 loss:0.005 grdn:0.153 lr:5.8e-05 updt_s:0.222 data_s:0.109 +INFO 2025-05-19 13:38:25 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.85 loss:0.006 grdn:0.172 lr:5.8e-05 updt_s:0.221 data_s:0.109 +INFO 2025-05-19 13:39:31 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.90 loss:0.006 grdn:0.165 lr:5.8e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 13:40:37 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.95 loss:0.006 grdn:0.156 lr:5.8e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 13:41:42 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.00 loss:0.006 grdn:0.162 lr:5.8e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 13:42:47 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.05 loss:0.006 grdn:0.169 lr:5.8e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 13:43:54 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.10 loss:0.006 grdn:0.162 lr:5.8e-05 updt_s:0.222 data_s:0.115 +INFO 2025-05-19 13:45:00 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.15 loss:0.007 grdn:0.168 lr:5.8e-05 updt_s:0.222 data_s:0.105 +INFO 2025-05-19 13:46:05 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.20 loss:0.006 grdn:0.162 lr:5.8e-05 updt_s:0.222 data_s:0.101 +INFO 2025-05-19 13:47:10 ts/train.py:232 step:271K smpl:4M ep:20K epch:68.25 loss:0.005 grdn:0.154 lr:5.8e-05 updt_s:0.222 data_s:0.106 +INFO 2025-05-19 13:48:18 ts/train.py:232 step:271K smpl:4M ep:20K epch:68.30 loss:0.006 grdn:0.157 lr:5.8e-05 updt_s:0.221 data_s:0.116 +INFO 2025-05-19 13:49:25 ts/train.py:232 step:271K smpl:4M ep:21K epch:68.35 loss:0.006 grdn:0.168 lr:5.8e-05 updt_s:0.223 data_s:0.109 +INFO 2025-05-19 13:50:30 ts/train.py:232 step:271K smpl:4M ep:21K epch:68.40 loss:0.006 grdn:0.171 lr:5.8e-05 updt_s:0.222 data_s:0.106 +INFO 2025-05-19 13:51:37 ts/train.py:232 step:271K smpl:4M ep:21K epch:68.45 loss:0.005 grdn:0.149 lr:5.8e-05 updt_s:0.222 data_s:0.108 +INFO 2025-05-19 13:52:43 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.50 loss:0.006 grdn:0.161 lr:5.8e-05 updt_s:0.222 data_s:0.108 +INFO 2025-05-19 13:53:49 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.56 loss:0.005 grdn:0.149 lr:5.7e-05 updt_s:0.222 data_s:0.106 +INFO 2025-05-19 13:54:55 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.61 loss:0.006 grdn:0.165 lr:5.7e-05 updt_s:0.222 data_s:0.111 +INFO 2025-05-19 13:56:04 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.66 loss:0.006 grdn:0.168 lr:5.7e-05 updt_s:0.222 data_s:0.122 +INFO 2025-05-19 13:57:09 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.71 loss:0.006 grdn:0.162 lr:5.7e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 13:58:12 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.76 loss:0.005 grdn:0.153 lr:5.7e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-19 13:59:13 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.81 loss:0.005 grdn:0.149 lr:5.7e-05 updt_s:0.222 data_s:0.084 +INFO 2025-05-19 14:00:15 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.86 loss:0.006 grdn:0.163 lr:5.7e-05 updt_s:0.221 data_s:0.088 +INFO 2025-05-19 14:01:17 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.91 loss:0.005 grdn:0.152 lr:5.7e-05 updt_s:0.223 data_s:0.085 +INFO 2025-05-19 14:02:20 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.96 loss:0.006 grdn:0.155 lr:5.7e-05 updt_s:0.222 data_s:0.089 +INFO 2025-05-19 14:03:22 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.01 loss:0.006 grdn:0.161 lr:5.7e-05 updt_s:0.222 data_s:0.087 +INFO 2025-05-19 14:04:22 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.06 loss:0.006 grdn:0.165 lr:5.7e-05 updt_s:0.222 data_s:0.081 +INFO 2025-05-19 14:05:23 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.11 loss:0.006 grdn:0.158 lr:5.7e-05 updt_s:0.222 data_s:0.083 +INFO 2025-05-19 14:06:22 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.16 loss:0.006 grdn:0.154 lr:5.7e-05 updt_s:0.222 data_s:0.073 +INFO 2025-05-19 14:07:21 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.21 loss:0.006 grdn:0.150 lr:5.7e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-19 14:08:21 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.26 loss:0.006 grdn:0.165 lr:5.7e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-19 14:09:21 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.31 loss:0.006 grdn:0.160 lr:5.7e-05 updt_s:0.222 data_s:0.077 +INFO 2025-05-19 14:10:21 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.36 loss:0.007 grdn:0.177 lr:5.7e-05 updt_s:0.222 data_s:0.079 +INFO 2025-05-19 14:11:22 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.41 loss:0.005 grdn:0.149 lr:5.7e-05 updt_s:0.222 data_s:0.080 +INFO 2025-05-19 14:12:26 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.46 loss:0.006 grdn:0.157 lr:5.7e-05 updt_s:0.222 data_s:0.098 +INFO 2025-05-19 14:13:27 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.51 loss:0.006 grdn:0.167 lr:5.6e-05 updt_s:0.222 data_s:0.083 +INFO 2025-05-19 14:14:27 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.56 loss:0.006 grdn:0.162 lr:5.6e-05 updt_s:0.222 data_s:0.079 +INFO 2025-05-19 14:15:28 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.61 loss:0.006 grdn:0.161 lr:5.6e-05 updt_s:0.222 data_s:0.082 +INFO 2025-05-19 14:16:38 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.67 loss:0.006 grdn:0.157 lr:5.6e-05 updt_s:0.221 data_s:0.126 +INFO 2025-05-19 14:17:42 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.72 loss:0.006 grdn:0.163 lr:5.6e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 14:18:46 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.77 loss:0.006 grdn:0.165 lr:5.6e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 14:19:52 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.82 loss:0.006 grdn:0.163 lr:5.6e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-19 14:20:57 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.87 loss:0.006 grdn:0.166 lr:5.6e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 14:22:02 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.92 loss:0.006 grdn:0.171 lr:5.6e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 14:23:08 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.97 loss:0.006 grdn:0.162 lr:5.6e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 14:24:12 ts/train.py:232 step:278K smpl:4M ep:21K epch:70.02 loss:0.006 grdn:0.157 lr:5.6e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 14:25:17 ts/train.py:232 step:278K smpl:4M ep:21K epch:70.07 loss:0.006 grdn:0.164 lr:5.6e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 14:26:22 ts/train.py:232 step:278K smpl:4M ep:21K epch:70.12 loss:0.006 grdn:0.164 lr:5.6e-05 updt_s:0.222 data_s:0.102 +Exception in thread Thread-75 (_pin_memory_loop): +Traceback (most recent call last): + File "/opt/conda/envs/lerobot/lib/python3.10/threading.py", line 1016, in _bootstrap_inner + self.run() + File "/opt/conda/envs/lerobot/lib/python3.10/threading.py", line 953, in run + self._target(*self._args, **self._kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 59, in _pin_memory_loop + do_one_step() + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 35, in do_one_step + r = in_queue.get(timeout=MP_STATUS_CHECK_INTERVAL) + File "/opt/conda/envs/lerobot/lib/python3.10/multiprocessing/queues.py", line 122, in get + return _ForkingPickler.loads(res) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/multiprocessing/reductions.py", line 541, in rebuild_storage_fd + fd = df.detach() + File "/opt/conda/envs/lerobot/lib/python3.10/multiprocessing/resource_sharer.py", line 57, in detach + with _resource_sharer.get_connection(self._id) as conn: + File "/opt/conda/envs/lerobot/lib/python3.10/multiprocessing/resource_sharer.py", line 86, in get_connection + c = Client(address, authkey=process.current_process().authkey) + File "/opt/conda/envs/lerobot/lib/python3.10/multiprocessing/connection.py", line 502, in Client + c = SocketClient(address) + File "/opt/conda/envs/lerobot/lib/python3.10/multiprocessing/connection.py", line 630, in SocketClient + s.connect(address) +FileNotFoundError: [Errno 2] No such file or directory +Traceback (most recent call last): + File "/NAS/LSY/lerobot/lerobot/scripts/train.py", line 288, in + train() + File "/NAS/LSY/lerobot/lerobot/configs/parser.py", line 227, in wrapper_inner + response = fn(cfg, *args, **kwargs) + File "/NAS/LSY/lerobot/lerobot/scripts/train.py", line 212, in train + train_tracker, output_dict = update_policy( + File "/NAS/LSY/lerobot/lerobot/scripts/train.py", line 71, in update_policy + loss, output_dict = policy.forward(batch) + File "/NAS/LSY/lerobot/lerobot/common/policies/diffusion/modeling_diffusion.py", line 305, in forward + loss = self.diffusion.compute_loss(batch) + File "/NAS/LSY/lerobot/lerobot/common/policies/diffusion/modeling_diffusion.py", line 574, in compute_loss + global_cond = self._prepare_global_conditioning(batch) # (B, global_cond_dim) + File "/NAS/LSY/lerobot/lerobot/common/policies/diffusion/modeling_diffusion.py", line 465, in _prepare_global_conditioning + img_features = self.rgb_encoder( + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/NAS/LSY/lerobot/lerobot/common/policies/diffusion/modeling_diffusion.py", line 777, in forward + x = torch.flatten(self.pool(self.backbone(x)), start_dim=1) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/container.py", line 250, in forward + input = module(input) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/container.py", line 250, in forward + input = module(input) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torchvision/models/resnet.py", line 146, in forward + out = self.conv1(x) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl + return forward_call(*args, **kwargs) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 554, in forward + return self._conv_forward(input, self.weight, self.bias) + File "/opt/conda/envs/lerobot/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 549, in _conv_forward + return F.conv2d( +KeyboardInterrupt diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/requirements.txt b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..68f3c0c612674471eb5bd86cd21f3a339e5e7f24 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/requirements.txt @@ -0,0 +1,146 @@ +pyyaml-include==1.4.1 +pillow==11.2.1 +six==1.17.0 +nvidia-cusparse-cu12==12.3.1.170 +packaging==24.2 +torch==2.6.0 +async-timeout==5.0.1 +wcwidth==0.2.13 +nvidia-nvjitlink-cu12==12.4.127 +pip==25.0.1 +requests==2.32.3 +draccus==0.10.0 +python-dateutil==2.9.0.post0 +numba==0.61.2 +antlr4-python3-runtime==4.9.3 +pyzmq==26.4.0 +termcolor==3.0.1 +pyserial==3.5 +pynput==1.8.1 +pfzy==0.3.4 +deepdiff==8.4.2 +h5py==3.13.0 +imageio==2.37.0 +typing-inspect==0.9.0 +imageio-ffmpeg==0.6.0 +multidict==6.4.3 +fasteners==0.19 +nvidia-cufft-cu12==11.2.1.3 +nvidia-cusolver-cu12==11.6.1.9 +triton==3.2.0 +beautifulsoup4==4.13.4 +annotated-types==0.7.0 +llvmlite==0.44.0 +TorchCodec==0.2.1 +hf_transfer==0.1.9 +cffi==1.17.1 +zarr==2.18.3 +opencv-python-headless==4.11.0.86 +tqdm==4.67.1 +protobuf==5.29.4 +av==14.3.0 +click==8.1.8 +pyarrow==19.0.1 +Flask==3.1.0 +zipp==3.21.0 +importlib_metadata==8.6.1 +urllib3==2.4.0 +huggingface-hub==0.30.2 +gitdb==4.0.12 +nvidia-nccl-cu12==2.21.5 +pydantic_core==2.33.1 +MarkupSafe==3.0.2 +blinker==1.9.0 +aiohappyeyeballs==2.6.1 +einops==0.8.1 +feetech-servo-sdk==1.0.0 +omegaconf==2.3.0 +nvidia-cuda-nvrtc-cu12==12.4.127 +gdown==5.2.0 +pycparser==2.22 +pandas==2.2.3 +orderly-set==5.4.0 +certifi==2025.1.31 +torchvision==0.21.0 +sentry-sdk==2.26.1 +dill==0.3.8 +mpmath==1.3.0 +python-xlib==0.33 +mergedeep==1.3.4 +Werkzeug==3.1.3 +prompt_toolkit==3.0.51 +PyYAML==6.0.2 +multiprocess==0.70.16 +toml==0.10.2 +wheel==0.45.1 +platformdirs==4.3.7 +numcodecs==0.13.1 +evdev==1.9.1 +pydantic==2.11.3 +fsspec==2024.12.0 +xxhash==3.5.0 +setproctitle==1.3.5 +propcache==0.3.1 +nvidia-cuda-cupti-cu12==12.4.127 +GitPython==3.1.44 +cloudpickle==3.1.1 +frozenlist==1.6.0 +filelock==3.18.0 +idna==3.10 +lerobot==0.1.0 +Farama-Notifications==0.0.4 +charset-normalizer==3.4.1 +wandb==0.19.9 +tzdata==2025.2 +nvidia-cublas-cu12==12.4.5.8 +pytz==2025.2 +smmap==5.0.2 +networkx==3.4.2 +numpy==2.2.4 +gymnasium==0.29.1 +aiohttp==3.11.16 +psutil==7.0.0 +attrs==25.3.0 +inquirerpy==0.3.4 +nvidia-curand-cu12==10.3.5.147 +nvidia-cudnn-cu12==9.1.0.70 +itsdangerous==2.2.0 +asciitree==0.3.3 +rerun-sdk==0.22.1 +mypy-extensions==1.0.0 +regex==2024.11.6 +aiosignal==1.3.2 +docker-pycreds==0.4.0 +sympy==1.13.1 +PySocks==1.7.1 +yarl==1.20.0 +nvidia-cuda-runtime-cu12==12.4.127 +soupsieve==2.6 +Jinja2==3.1.6 +nvidia-nvtx-cu12==12.4.127 +safetensors==0.5.3 +datasets==3.5.0 +typing_extensions==4.13.2 +diffusers==0.33.1 +pymunk==6.11.1 +nvidia-cusparselt-cu12==0.6.2 +typing-inspection==0.4.0 +setuptools==78.1.0 +cmake==4.0.0 +jsonlines==4.0.0 +zipp==3.19.2 +packaging==24.2 +importlib_metadata==8.0.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +backports.tarfile==1.2.0 +autocommand==2.2.2 +inflect==7.3.1 +typeguard==4.3.0 +more-itertools==10.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +tomli==2.0.1 +jaraco.collections==5.1.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/wandb-metadata.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4ae2ffbbfcc11ebea214695500dee9a8df8c4d27 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/wandb-metadata.json @@ -0,0 +1,118 @@ +{ + "os": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.17", + "startedAt": "2025-05-18T14:52:55.338138Z", + "args": [ + "--dataset.repo_id=shylee/so100_pengripG", + "--policy.type=diffusion", + "--policy.device=cuda", + "--dataset.root=/SSD/LSY/lerobot_dataset/pengripG", + "--wandb.enable=true", + "--save_freq=20000", + "--steps=600000", + "--policy.noise_scheduler_type=DDIM", + "--policy.num_inference_steps=10", + "--batch_size=16", + "--policy.down_dims=[256,512,1024]", + "--policy.vision_backbone=resnet50", + "--policy.pretrained_backbone_weights=ResNet50_Weights.IMAGENET1K_V1", + "--policy.use_group_norm=false", + "--policy.pre_resize_shape=[224,224]", + "--policy.crop_shape=[224,224]", + "--policy.crop_is_random=false", + "--policy.freeze_vision_backbone=false", + "--policy.horizon=16", + "--policy.n_action_steps=8", + "--policy.optimizer_lr=0.0001", + "--policy.use_ac=false", + "--policy.ac_threshold=0.99", + "--policy.use_pcg=false", + "--policy.pcg_weight=0.1", + "--policy.use_bid=false", + "--policy.bid_batch_size=15", + "--policy.bid_mode_size=3", + "--policy.bid_temporal_coherence=0.5", + "--dataset.image_transforms.enable=false", + "--log_freq=200", + "--output_dir=/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "--job_name=DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain" + ], + "program": "/NAS/LSY/lerobot/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "git": { + "remote": "https://github.com/huggingface/lerobot.git", + "commit": "a445d9c9da6bea99a8972daa4fe1fdd053d711d2" + }, + "root": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain", + "host": "91cf0fc7c553", + "executable": "/opt/conda/envs/lerobot/bin/python", + "codePathLocal": "lerobot/scripts/train.py", + "cpu_count": 48, + "cpu_count_logical": 96, + "gpu": "NVIDIA GeForce RTX 3090", + "gpu_count": 8, + "disk": { + "/": { + "total": "1820335460352", + "used": "1095481737216" + } + }, + "memory": { + "total": "253372637184" + }, + "cpu": { + "count": 48, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/wandb-summary.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b74081e47f9f308554e0f06303624e56f9e16e7f --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/update_s":0.2216261836467311,"train/steps":278000,"_step":278000,"train/samples":4448000,"train/lr":5.584258665349344e-05,"train/loss":0.00612235098000383,"train/episodes":21035.70584062426,"train/epochs":70.11901946874754,"train/grad_norm":0.16436642093583942,"_wandb":{"runtime":84872},"train/dataloading_s":0.101693581022555,"_runtime":84872.768239785,"_timestamp":1.7476647823906717e+09} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-core.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..ac18ad3bcacd0980f6c383119424be22d4aee1fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-05-18T14:52:55.147249479Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp5e58ptwh/port-18114.txt","pid":18114,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-05-18T14:52:55.149071963Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":18114} +{"time":"2025-05-18T14:52:55.149066236Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":33239,"Zone":""}} +{"time":"2025-05-18T14:52:55.330323472Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:47176"} +{"time":"2025-05-18T14:52:55.338919419Z","level":"INFO","msg":"handleInformInit: received","streamId":"fekph8dz","id":"127.0.0.1:47176"} +{"time":"2025-05-18T14:52:55.670506568Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"fekph8dz","id":"127.0.0.1:47176"} +{"time":"2025-05-19T14:27:28.10588862Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:47176"} +{"time":"2025-05-19T14:27:28.106258078Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:47176"} +{"time":"2025-05-19T14:27:28.106299284Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-05-19T14:27:28.106382953Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:47176"} +{"time":"2025-05-19T14:27:30.582386788Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:47176"} +{"time":"2025-05-19T14:27:30.582457817Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:47176"} +{"time":"2025-05-19T14:27:30.582490432Z","level":"INFO","msg":"server is closed"} diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-internal.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..dac689ddf9885760f57acfe786773ab591878c31 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-05-18T14:52:55.339058052Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-core.log"} +{"time":"2025-05-18T14:52:55.67042234Z","level":"INFO","msg":"created new stream","id":"fekph8dz"} +{"time":"2025-05-18T14:52:55.670496162Z","level":"INFO","msg":"stream: started","id":"fekph8dz"} +{"time":"2025-05-18T14:52:55.670554828Z","level":"INFO","msg":"sender: started","stream_id":"fekph8dz"} +{"time":"2025-05-18T14:52:55.670556224Z","level":"INFO","msg":"writer: Do: started","stream_id":"fekph8dz"} +{"time":"2025-05-18T14:52:55.670618731Z","level":"INFO","msg":"handler: started","stream_id":"fekph8dz"} +{"time":"2025-05-18T14:52:56.221542859Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-05-19T14:27:28.106339931Z","level":"INFO","msg":"stream: closing","id":"fekph8dz"} +{"time":"2025-05-19T14:27:28.106399575Z","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-05-19T14:27:28.106543378Z","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-05-19T14:27:29.143033663Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-05-19T14:27:29.418524413Z","level":"INFO","msg":"handler: closed","stream_id":"fekph8dz"} +{"time":"2025-05-19T14:27:29.418600121Z","level":"INFO","msg":"writer: Close: closed","stream_id":"fekph8dz"} +{"time":"2025-05-19T14:27:29.418619047Z","level":"INFO","msg":"sender: closed","stream_id":"fekph8dz"} +{"time":"2025-05-19T14:27:29.418794627Z","level":"INFO","msg":"stream: closed","id":"fekph8dz"} diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0fa511745ddef851e37b373283ff9f07c3755070 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug.log @@ -0,0 +1,23 @@ +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Configure stats pid to 18114 +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug.log +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/logs/debug-internal.log +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:init():781] calling init triggers +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'shylee/so100_pengripG', 'root': '/SSD/LSY/lerobot_dataset/pengripG', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ac': False, 'ac_threshold': 0.99, 'use_pcg': False, 'pcg_weight': 0.1, 'use_bid': False, 'bid_batch_size': 15, 'bid_mode_size': 3, 'bid_temporal_coherence': 0.5, 'weak_model_path': None, 'weak_model_config': None}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain', 'job_name': 'DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 600000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:init():809] starting backend +2025-05-18 14:52:55,332 INFO MainThread:18114 [wandb_init.py:init():813] sending inform_init request +2025-05-18 14:52:55,337 INFO MainThread:18114 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-05-18 14:52:55,337 INFO MainThread:18114 [wandb_init.py:init():823] backend started and connected +2025-05-18 14:52:55,472 INFO MainThread:18114 [wandb_init.py:init():915] updated telemetry +2025-05-18 14:52:55,737 INFO MainThread:18114 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-05-18 14:52:56,212 INFO MainThread:18114 [wandb_init.py:init():1014] starting run threads in backend +2025-05-18 14:52:56,355 INFO MainThread:18114 [wandb_run.py:_console_start():2454] atexit reg +2025-05-18 14:52:56,355 INFO MainThread:18114 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-05-18 14:52:56,355 INFO MainThread:18114 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-05-18 14:52:56,355 INFO MainThread:18114 [wandb_run.py:_redirect():2394] Redirects installed. +2025-05-18 14:52:56,356 INFO MainThread:18114 [wandb_init.py:init():1056] run started, returning control to user process +2025-05-19 14:27:28,103 INFO MsgRouterThr:18114 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/run-fekph8dz.wandb b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/run-fekph8dz.wandb new file mode 100644 index 0000000000000000000000000000000000000000..37b48244ba114915363315ffb1743d06727ecf2d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250518_145255-fekph8dz/run-fekph8dz.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5110836fd17df4762f5a5c0f18c9d423d241222d66f06daefd02dc53a8d7739 +size 19035122