diff --git a/.gitattributes b/.gitattributes index c9b6d752f2d5130de57dd31bef7165b408e38bae..3aef022c4d6882af5db9d23b18a7349a56cde2eb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -53,3 +53,4 @@ DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-202 DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250518_143255-5e56tibb/run-5e56tibb.wandb filter=lfs diff=lfs merge=lfs -text DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_shorttrain/wandb/run-20250519_150138-5e56tibb/run-5e56tibb.wandb filter=lfs diff=lfs merge=lfs -text DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_middletrain/wandb/run-20250519_150246-fekph8dz/run-fekph8dz.wandb filter=lfs diff=lfs merge=lfs -text +DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/run-bqr8y636.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a476e2574935f66af709df77c23f1826966c8637 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d95f5ece3c250f0b73c5a8dcab8dc208bd3ad15d41bcfb73fc4ed8ed7d0be5d +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..086c9a8cb16a368bb0e11fbf67968eb8709748bc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.800949179930113e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fc430df458ad1d4a5f3a28f11c2eaa716d43607c --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08263ee32e78dc147512ed8e06a4dcc7f513da5e638c7f312988b069ebf2c964 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ae4dd25e80f9f8e216f47a3b54dddfe892dcafb8 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30f5dcf9d86ab6d40c5f4424b4efb3d8522ef08cfa4ba99e1b18727deff4bc7a +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c92657c79e611559198b979b20e1fb8b623a8e7f --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 280000, + "verbose": false, + "_step_count": 280001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.800949179930113e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..7dd543c28615294e867b2577a61ab8cc835e631a --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/280000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 280000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d6389eff1940fd8d7a896d976045f20392cd2c18 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a36b8b3c5a37c480033a8585ce7c60a66efd7ecf359ea65d993d8ea5e843194 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..441b7eada7faf4a78ccaa1e14c1d8083b6e9035c --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.505039436856827e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1b8e828068788946a5508acf5b17efdfcdf9a8c3 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52a34fb6c5f0bfa2464b5e12e753f5baaa438789033b7fe30bf7e5287c7413b8 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d7b0a510380f02f5005a5c6cde3ae2e561bbcdb8 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d70ec757ce50e217e55e9f9af4e6bbc64f6359f812489fc68b023607554fc85 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..46f0924834037be23d642823ee826bc729c90d27 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 300000, + "verbose": false, + "_step_count": 300001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.505039436856827e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..db7e948ddab8257b846ad5ef28f8c555a9524598 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/300000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 300000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0a9ae470b1fea1c46a0724026cb0b4af9a7bced2 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27af29a48fd4860aeab76e70a002fe27fa2f168929941fc46eb300a0a6a55970 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..45d8264a7c7800ee1c401534f24e0cecae8dae02 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 7.196911827329292e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..401382c481dd65edcc576146f3ec76aff0b0873c --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c4e7660a3d8566e448c5a9866d9c04ce5000e3496e6dc7d88db46c6adbae472 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..40b7dcb44a10d5a08e27ea784836b66de1ec615b --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43e11a6c813c7229f1de118b102d59cdc29f4a502513ac858bd6e11d4b998a45 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..acfa95e344c33e42271eb5d41425a32bc53d31c6 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 320000, + "verbose": false, + "_step_count": 320001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 7.196911827329292e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..5a84d0fb9b5af235bc8686fcf332bf66b5491d48 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/320000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 320000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..83e2713ffcb5f2b912b8574707733ce7b3625c27 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:775f96b61fda929db323fa54b00410d267f901d0d53dfa7df896b64d661cd14d +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..05e5b4012a27015e6511a72ff7892f35abd30ce7 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.878069186763465e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..88c1bffc49caf996ae9cbca002eb6518453dd89b --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:942763896eb0910b7fa8a563120278b58c6a5f5450b624c1c56f64e285916518 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..629e105b4c5942307ad8fc515d29f497c8586099 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37ccf8dc123ae0aa4f67953719f34767f74aa76c9b1823a172edf3cfd645c5df +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6c0ec36c2ae8b1c1aa91816820aea1a1877b53d2 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 340000, + "verbose": false, + "_step_count": 340001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.878069186763465e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..8a7a31b491f60de60fc850f75bfdfc405cc267e5 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/340000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 340000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d2b9277b95952a114ac6e7971186abbe54e805bc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab30ae7eb30be988093f96fd66145e2769d64a57d312e43c8e00bfe0eaab671a +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..c95ffe05c43624624f92810f8806852a0be08830 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.550066611157049e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b0098f814880790282988408a8d46ebf85479fb5 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c0b78a6d53dea135dcf1cd0f2f2ca939380801759e9b00aa519fa24dd875f36 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c3bdf98772b60d4aa8c4797402872429abe38c63 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68b85d369be8362b5b5e925e2da0a3e71387344dfb3f26f1cda1a9aed81bf66b +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a39d6e5bb2a4966dfcb991d5bc882f8c4dadb29b --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 360000, + "verbose": false, + "_step_count": 360001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.550066611157049e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..bd79b4d9c6aaae91547528200cf9573568cb1ede --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/360000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 360000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f3b746c7ace34de6eb53d380fd56c8dc190ab231 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7866c809900a0b29765a29297bf5a2968f1570997e3b62a8607848a6696859e +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..df95fd55459a9c2a949ff7aa9fb1d4814d2c2d8d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 6.214503872396447e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..39c26115bd6e52c4b732dd96e2e730d6dd9843b5 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc6f7e3930b1eefa2f2b586fa6a4136ea76b18ece12b67bfe5024d6e44835f50 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..463c40990c1db65c05211fce81a91284d2109a2a --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a0de54a7a1612a7bf707e4e2733964d583a5fdcec574a2d38c80ff8ac41bfa1 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bd569c09bef52899b50fd08803703663b241a6d2 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 380000, + "verbose": false, + "_step_count": 380001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 6.214503872396447e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..4c3b40642a9a25da584d6587de866bd19b865a0c --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/380000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 380000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b300c36da8294bde81d3afbdf1c7e3a4b9d3fcf5 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:381146ff47104f6c588c40dfa6492014ad7a150ff7171f472e96396efe0b07a6 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..bebb2048239985cf72ed5c9dff5c990770df5543 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 5.873017615665317e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1b3fd178b1d13edb077aa6dcbcae2e6e706c99b9 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b76da2e251b1f832d82c2dfa8d26546878e79da78ca9873f184b79396bcd868d +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8acd6308c8cfc63361bce6cf9c9c7336500dcd21 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e544094ce665c09d91d0436fd8b3ce27bcfb1c8536e59cbbcce139d440907d1 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..863709eca3512cd63ad9178625d50bb559637cc1 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 400000, + "verbose": false, + "_step_count": 400001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.873017615665317e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..f13646a804137593a621abee8f4d116e4ad6127a --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/400000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 400000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dfed3a9ca24ae63e7af8721e89ab86d50db14eb7 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e2c745f2e015409e44beed023d85ae3c77c66523d46254a9627405b977b5a42 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..1d40674d60d7b8af26656472e6e042f9db106ad6 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 5.527273377010439e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7254b0df516531196fdfe63a3ccfd076a0b90495 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8607db061628eb736f076903983a7b19988eadc6a4f3f1f0860f89dc524226 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7814b9f16bdde93446735d8e9fa0ad49d99f5aa3 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e2c79a94bbffe4462dd36784007add6565768ac6d55ba16c5e1c597579c96b3 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..56b95070343527bb3d50e47b311c1b9e0f69978f --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 420000, + "verbose": false, + "_step_count": 420001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.527273377010439e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..bb0ce907623865596f71441ca75ab257971c1b55 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/420000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 420000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4258b268b4f58a6f080a4d6aaa6f4401fd37a717 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8afae0d52571b6f21e78353b1e067abac494b213a9a37c00179a1e2bfa2dfcf +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..861e8b6569ed1c40331c0ab99b9be6e419957593 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 5.1789574599977653e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d52c82bb6be3320c6a0a669db004c378e7282260 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4ed731da737b12f5aedc074455e4ef32514c38c7cca311e5f604c316c96585e +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..df1b0b6e3c6ffbce319a5983a2754793f297fd35 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85f9c6021283fd5f54124c89ba6ede29f688189ad736bd139ae56ea008e2547a +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..79d8d30ef2f517d65383dbee4da45f10b72ac7bf --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 440000, + "verbose": false, + "_step_count": 440001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 5.1789574599977653e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..78c7159b2d576dbc02106f2e6902349139e264ff --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/440000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 440000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..56f586f4531fdceb8308193df03c8782a39d0d79 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:194a468d8a55b39015806c02891ebbf2a15a0450da54f749b8b0f39b0429ee4f +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..496d4e40e54ec0c5d1b25a87133d4d680522b0d4 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 4.8297687110787045e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7abdb9aeaed68d2bcf048eb3269580eea94491f7 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a411c9c48aa17aa4110e5b0b1c75ec36cec7ea63be13568778c0f5d801042e68 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f1a38b2ff9e1a06d618b3fd7fd7d4c3dc55f4d95 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22a67885bea6f317ab71bd20a83c00e8cd2b0370548212da1abd1c29055b2eda +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..42f9eca13f2b9b384a90fa1f3908316bea86887d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 460000, + "verbose": false, + "_step_count": 460001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 4.8297687110787045e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..49da076aca4a54b3293aa4705f46d30616c97217 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/460000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 460000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b20b05e5f56098d1633909db583f1a65dbbd7c27 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b320b2dbd5b17aa5be60fbdeddfc452796a8f64da06e9bef064224de9dfeeca8 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..185ed4617482b40deb15b33a188fe7f97e23f4c9 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 4.4814102337808293e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e5ec7f0592506f2ea22d1d8cb0b09f2d6f4f60fe --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ec9c7f49bd64af8e7248c8520740fa816349b53ee1967eb5601c5285773c2be +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..566cc5df087f861f44d3dfb3337099ca8199538d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f50ed04c738c7041f21fce321eaafa1deee9f48f20a47f6c1c331a35f2e39486 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a0b5034bbc249bb90d3558fd2d0584df557319e4 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 480000, + "verbose": false, + "_step_count": 480001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 4.4814102337808293e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..a54500e6a42a0151bd2a06726edce1d9ecf0a2e2 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/480000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 480000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4b9721c399aabf8cfcb79e4bf55a3e1f5b0495c7 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b47268234b593a70169a4080dd1a140281ff79df9413d761138f54ca8b0a280 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..49a3cb4c132b20b9c4e271d3b018b3a44b6e6908 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 4.135581082135471e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d3c11ceb2b2cf33d045f2f47910dc2ee330e864 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dbff50c51d120e0507ae1335f595c4f10688b8b4e2c8817c99eafac654eb08d +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a36c49e1e647c2e088c653b2e255ba135e34302 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4046cc943be1aef5636959938d29eada9064380409536925e536ce020d5cc77e +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0a8743cf59182163deff01fd8f3f552ea99fb1dc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 500000, + "verbose": false, + "_step_count": 500001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 4.135581082135471e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..9c27c35dc10c42f51533a30dfc30e421493446d6 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/500000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 500000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5be8c5ef10da6a9bcd2617e59620e3c84ec02720 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030ea1a85e03602dd506f3aca5bd42ef53dc9be47f4bbe37c4e7d28cd5cf6944 +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..f263ce14b6720ca682071458440f9bfa43fec1cb --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 3.7939679738559815e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ce42ff7c7f1d093f428cfc7c844568017872325c --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46b93520bb6e45e44401022609650399703cd0420f846cd65a2271424138cc8d +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..04b55f501fcb48e9a7fb4fd1456b1274529ba3c1 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9560d794b4aabb15f1dcda5afc2fac9445e74fb7095a5aabb1e4416df2b40f01 +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0899120adf1930ae1a2577536f8171f26457332e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 520000, + "verbose": false, + "_step_count": 520001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 3.7939679738559815e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..96bade7e61bdcd7ea9e84ec8ed085edc4c23f99d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/520000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 520000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/pretrained_model/config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/pretrained_model/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c67a964ea3e70dc8909af56b3ab05bd9d44b31fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/pretrained_model/config.json @@ -0,0 +1,107 @@ +{ + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/pretrained_model/model.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/pretrained_model/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..26f0690d11f5fce10e6303d7ac58b612880f346d --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/pretrained_model/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4752b2da6b003f12dbc27a534eb0519c8f490448359413f65a38528b08f31bfa +size 369241872 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/pretrained_model/train_config.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/pretrained_model/train_config.json new file mode 100644 index 0000000000000000000000000000000000000000..d32daa6fe25603c79a80100928655185f17b526e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/pretrained_model/train_config.json @@ -0,0 +1,217 @@ +{ + "dataset": { + "repo_id": "shylee/so100_pengripG", + "root": "/SSD/LSY/lerobot_dataset/pengripG", + "episodes": null, + "image_transforms": { + "enable": false, + "max_num_transforms": 3, + "random_order": false, + "tfs": { + "brightness": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "brightness": [ + 0.8, + 1.2 + ] + } + }, + "contrast": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "contrast": [ + 0.8, + 1.2 + ] + } + }, + "saturation": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "saturation": [ + 0.5, + 1.5 + ] + } + }, + "hue": { + "weight": 1.0, + "type": "ColorJitter", + "kwargs": { + "hue": [ + -0.05, + 0.05 + ] + } + }, + "sharpness": { + "weight": 1.0, + "type": "SharpnessJitter", + "kwargs": { + "sharpness": [ + 0.5, + 1.5 + ] + } + } + } + }, + "revision": null, + "use_imagenet_stats": true, + "video_backend": "torchcodec" + }, + "env": null, + "policy": { + "type": "diffusion", + "n_obs_steps": 2, + "normalization_mapping": { + "VISUAL": "MEAN_STD", + "STATE": "MIN_MAX", + "ACTION": "MIN_MAX" + }, + "input_features": { + "observation.state": { + "type": "STATE", + "shape": [ + 6 + ] + }, + "observation.images.FrontCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.TopCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + }, + "observation.images.WristCam": { + "type": "VISUAL", + "shape": [ + 3, + 480, + 640 + ] + } + }, + "output_features": { + "action": { + "type": "ACTION", + "shape": [ + 6 + ] + } + }, + "device": "cuda", + "use_amp": false, + "horizon": 16, + "n_action_steps": 8, + "drop_n_last_frames": 7, + "vision_backbone": "resnet50", + "crop_shape": [ + 224, + 224 + ], + "crop_is_random": false, + "pretrained_backbone_weights": "ResNet50_Weights.IMAGENET1K_V1", + "use_group_norm": false, + "spatial_softmax_num_keypoints": 32, + "use_separate_rgb_encoder_per_camera": false, + "down_dims": [ + 256, + 512, + 1024 + ], + "kernel_size": 5, + "n_groups": 8, + "diffusion_step_embed_dim": 128, + "use_film_scale_modulation": true, + "noise_scheduler_type": "DDIM", + "num_train_timesteps": 100, + "beta_schedule": "squaredcos_cap_v2", + "beta_start": 0.0001, + "beta_end": 0.02, + "prediction_type": "epsilon", + "clip_sample": true, + "clip_sample_range": 1.0, + "num_inference_steps": 10, + "do_mask_loss_for_padding": false, + "optimizer_lr": 0.0001, + "optimizer_betas": [ + 0.95, + 0.999 + ], + "optimizer_eps": 1e-08, + "optimizer_weight_decay": 1e-06, + "scheduler_name": "cosine", + "scheduler_warmup_steps": 500, + "pre_resize_shape": [ + 224, + 224 + ], + "freeze_vision_backbone": false, + "use_ac": false, + "ac_threshold": 0.99, + "use_pcg": false, + "pcg_weight": 0.1, + "use_bid": false, + "bid_batch_size": 15, + "bid_mode_size": 3, + "bid_temporal_coherence": 0.5, + "weak_model_path": null, + "weak_model_config": null + }, + "output_dir": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "job_name": "DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "resume": true, + "seed": 1000, + "num_workers": 4, + "batch_size": 16, + "steps": 900000, + "eval_freq": 20000, + "log_freq": 200, + "save_checkpoint": true, + "save_freq": 20000, + "use_policy_training_preset": true, + "optimizer": { + "type": "adam", + "lr": 0.0001, + "weight_decay": 1e-06, + "grad_clip_norm": 10.0, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08 + }, + "scheduler": { + "type": "diffuser", + "num_warmup_steps": 500, + "name": "cosine" + }, + "eval": { + "n_episodes": 50, + "batch_size": 50, + "use_async_envs": false + }, + "wandb": { + "enable": true, + "disable_artifact": false, + "project": "lerobot", + "entity": null, + "notes": null, + "run_id": "bqr8y636", + "mode": null + } +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/optimizer_param_groups.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/optimizer_param_groups.json new file mode 100644 index 0000000000000000000000000000000000000000..88b4b9ccc1b52457955627c808b873afd081d43e --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/optimizer_param_groups.json @@ -0,0 +1,331 @@ +[ + { + "lr": 3.4582370636841024e-05, + "betas": [ + 0.95, + 0.999 + ], + "eps": 1e-08, + "weight_decay": 1e-06, + "amsgrad": false, + "maximize": false, + "foreach": null, + "capturable": false, + "differentiable": false, + "fused": null, + "initial_lr": 0.0001, + "params": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310 + ] + } +] \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/optimizer_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/optimizer_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..da5f91507fe2b0a84a9992b5184f226b2b8125b1 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/optimizer_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e63647aeae3d3bb533e95831c0048f2e486ffb0395340324ff202556f4e0e476 +size 738026076 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/rng_state.safetensors b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/rng_state.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b8824ada2034912f54988894277cc18000d5ec35 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/rng_state.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c5eb37e63641bb4a244052014bd68d6eee4623ec6e697a8a04b059e260c54fe +size 15708 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/scheduler_state.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/scheduler_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6ba6b261244956225f1e9c33461cb499b6610fff --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/scheduler_state.json @@ -0,0 +1,15 @@ +{ + "base_lrs": [ + 0.0001 + ], + "last_epoch": 540000, + "verbose": false, + "_step_count": 540001, + "_get_lr_called_within_step": false, + "_last_lr": [ + 3.4582370636841024e-05 + ], + "lr_lambdas": [ + null + ] +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/training_step.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/training_step.json new file mode 100644 index 0000000000000000000000000000000000000000..3c2f6e37b4db10fefdc1e301eb1523863fc3a230 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/540000/training_state/training_step.json @@ -0,0 +1,3 @@ +{ + "step": 540000 +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug-internal.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug-internal.log index 419f81963ea1cf74fb916b88a9c0c35defdd88b5..5132a729786bf802307a03b958ef8b718784c2d8 100644 --- a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug-internal.log +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug-internal.log @@ -1,16 +1,332 @@ -{"time":"2025-05-18T14:54:05.26111007Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250518_145405-bqr8y636/logs/debug-core.log"} -{"time":"2025-05-18T14:54:05.591761165Z","level":"INFO","msg":"created new stream","id":"bqr8y636"} -{"time":"2025-05-18T14:54:05.591831564Z","level":"INFO","msg":"stream: started","id":"bqr8y636"} -{"time":"2025-05-18T14:54:05.591873678Z","level":"INFO","msg":"writer: Do: started","stream_id":"bqr8y636"} -{"time":"2025-05-18T14:54:05.591956927Z","level":"INFO","msg":"handler: started","stream_id":"bqr8y636"} -{"time":"2025-05-18T14:54:05.592220435Z","level":"INFO","msg":"sender: started","stream_id":"bqr8y636"} -{"time":"2025-05-18T14:54:05.935180894Z","level":"INFO","msg":"Starting system monitor"} -{"time":"2025-05-19T00:52:06.776263087Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/marchmelo0923-postech/lerobot/bqr8y636/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} -{"time":"2025-05-19T14:27:35.7208776Z","level":"INFO","msg":"stream: closing","id":"bqr8y636"} -{"time":"2025-05-19T14:27:35.720986412Z","level":"INFO","msg":"Stopping system monitor"} -{"time":"2025-05-19T14:27:35.721176099Z","level":"INFO","msg":"Stopped system monitor"} -{"time":"2025-05-19T14:27:36.540912828Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} -{"time":"2025-05-19T14:27:36.804201475Z","level":"INFO","msg":"handler: closed","stream_id":"bqr8y636"} -{"time":"2025-05-19T14:27:36.804229901Z","level":"INFO","msg":"writer: Close: closed","stream_id":"bqr8y636"} -{"time":"2025-05-19T14:27:36.804255951Z","level":"INFO","msg":"sender: closed","stream_id":"bqr8y636"} -{"time":"2025-05-19T14:27:36.804384598Z","level":"INFO","msg":"stream: closed","id":"bqr8y636"} +{"time":"2025-05-19T15:05:23.415308382Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-core.log"} +{"time":"2025-05-19T15:05:24.501288642Z","level":"INFO","msg":"created new stream","id":"bqr8y636"} +{"time":"2025-05-19T15:05:24.501356248Z","level":"INFO","msg":"stream: started","id":"bqr8y636"} +{"time":"2025-05-19T15:05:24.501413936Z","level":"INFO","msg":"writer: Do: started","stream_id":"bqr8y636"} +{"time":"2025-05-19T15:05:24.501432863Z","level":"INFO","msg":"handler: started","stream_id":"bqr8y636"} +{"time":"2025-05-19T15:05:24.501480145Z","level":"INFO","msg":"sender: started","stream_id":"bqr8y636"} +{"time":"2025-05-19T15:05:27.724845141Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-05-19T15:06:36.770264445Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770384152Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770395047Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770404545Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770413974Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770423472Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770431993Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770441421Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770449034Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:07:34.530202527Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530296603Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530307917Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530316438Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530326006Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530335434Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530344374Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530352895Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530362323Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:08:32.290189386Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290241836Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.29025322Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.29026223Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.29027075Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290278293Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290299106Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290308115Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290316566Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:09:29.426161463Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426248484Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426257075Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426265106Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.42627216Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426279703Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426287316Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426295278Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426302401Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:10:26.818317636Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818400397Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818411781Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818421699Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818430289Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818438321Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818448238Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818456759Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818467095Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:11:24.990168864Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990258819Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990269225Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990278165Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990285288Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990292831Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990300444Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990308964Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990317904Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:12:23.562134693Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562210401Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562220318Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562228839Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562237359Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562246369Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562253911Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562262432Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562271022Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:13:21.202152787Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.20220866Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202219904Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202228425Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202236456Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202244558Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202253986Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202262018Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202271516Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:14:19.398158706Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398228268Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398239652Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.39824908Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398258998Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398268007Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398276039Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398285956Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398294477Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:15:17.182353839Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.182497711Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.182647589Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.182779588Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.182927231Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.18310323Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.18322573Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.183392789Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.183490706Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:16:15.190403151Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190677555Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190761783Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190775053Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190783992Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190840773Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190926956Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.191029552Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.191127539Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:17:12.25419108Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254285645Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254297098Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254306946Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254315048Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254325384Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254334393Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254343822Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254352831Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:18:10.326147235Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326205901Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326215819Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326225806Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326235304Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326243336Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326266034Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326275952Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326283564Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:19:07.598555156Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.59867626Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598688551Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598699866Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598710342Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598720329Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598733529Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598744354Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598758113Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:20:05.758124648Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758190857Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758201822Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758209854Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758219771Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758229269Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758238698Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758247149Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758256158Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:21:03.298179167Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298248239Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298260043Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298269471Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.29827848Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298286512Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298294544Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298303064Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298310677Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:22:01.658324187Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658405622Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658415958Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658424968Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658433488Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658442498Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658451018Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658459469Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658466593Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:59.490190645Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490269146Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490277178Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490284232Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490291356Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490298898Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490306022Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490312238Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490317895Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:23:57.338177152Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338214028Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338225412Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.33823491Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338244269Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338254186Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338262288Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338271716Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338280726Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:25:00.678187256Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678278538Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678289503Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678315065Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678324494Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678333922Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678342932Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678350963Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678358995Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:59.510224168Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.51031119Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510321107Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510329209Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510336752Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.51034625Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510354212Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510362313Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510370834Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:26:58.638271252Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638333619Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638345073Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638354991Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638365327Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638376781Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638387187Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638399409Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638408908Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:27:58.754161763Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754256886Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754270156Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754280562Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754289571Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754299Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754324981Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754334479Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754344466Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:28:58.886349234Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886479837Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886493595Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886504001Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886515385Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886526211Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886537106Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886547023Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886557011Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:29:58.614261493Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614329588Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614342369Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614354172Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614363601Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614373099Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614382527Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614392026Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614402432Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:30:59.878202327Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878285088Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878294517Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878303526Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878311558Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878319589Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.87832811Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.87833712Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878360307Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:32:00.950145719Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950218633Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950229528Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950240005Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950249433Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950259281Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950266893Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950276811Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.95028582Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:33:03.554554311Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554631415Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554642799Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554652717Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554663123Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554672133Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554682539Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554692945Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554702374Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:34:04.774330814Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774515263Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774529022Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774536635Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.7745432Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774552698Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774561708Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.77456925Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774577282Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:35:07.758300095Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758414145Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.75844725Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758456678Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758465688Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758474208Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.75848224Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758490761Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758498792Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:36:10.370165061Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370204801Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370215696Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370225124Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370234134Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370244051Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370252572Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370261511Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370271988Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:37:11.914254057Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914356234Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914368107Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914378024Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914387942Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914396462Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914407357Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914415878Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914425865Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:38:14.480437609Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.480539437Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.480594262Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.480806158Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.4808232Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.480915948Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.481007719Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.481098582Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.481204111Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:39:15.225596693Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.225690768Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.225947712Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226019578Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226030055Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226059388Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226135095Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226243907Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226326668Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:40:16.907359295Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907479421Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907536621Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907647388Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.90773874Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907825272Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907921792Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.908025436Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.908112947Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:41:17.796835319Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.796921991Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.796964035Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797079552Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797166504Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797258344Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797337823Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797455574Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797553072Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-20T05:23:13.729960737Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug.log index 80706369aa72abc718dde2731e60a40fd363f105..7ffb1fee70422a673daf2fd9fd37b522e8c997fc 100644 --- a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug.log +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/debug.log @@ -1,23 +1,23 @@ -2025-05-18 14:54:05,251 INFO MainThread:19256 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_setup.py:_flush():67] Configure stats pid to 19256 -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_setup.py:_flush():67] Loading settings from environment variables -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250518_145405-bqr8y636/logs/debug.log -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250518_145405-bqr8y636/logs/debug-internal.log -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_init.py:init():781] calling init triggers -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_init.py:init():786] wandb.init called with sweep_config: {} -config: {'dataset': {'repo_id': 'shylee/so100_pengripG', 'root': '/SSD/LSY/lerobot_dataset/pengripG', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ac': False, 'ac_threshold': 0.99, 'use_pcg': False, 'pcg_weight': 0.1, 'use_bid': False, 'bid_batch_size': 15, 'bid_mode_size': 3, 'bid_temporal_coherence': 0.5, 'weak_model_path': None, 'weak_model_config': None}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain', 'job_name': 'DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 900000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}} -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_init.py:init():809] starting backend -2025-05-18 14:54:05,252 INFO MainThread:19256 [wandb_init.py:init():813] sending inform_init request -2025-05-18 14:54:05,259 INFO MainThread:19256 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2025-05-18 14:54:05,259 INFO MainThread:19256 [wandb_init.py:init():823] backend started and connected -2025-05-18 14:54:05,424 INFO MainThread:19256 [wandb_init.py:init():915] updated telemetry -2025-05-18 14:54:05,482 INFO MainThread:19256 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout -2025-05-18 14:54:05,931 INFO MainThread:19256 [wandb_init.py:init():1014] starting run threads in backend -2025-05-18 14:54:06,042 INFO MainThread:19256 [wandb_run.py:_console_start():2454] atexit reg -2025-05-18 14:54:06,043 INFO MainThread:19256 [wandb_run.py:_redirect():2306] redirect: wrap_raw -2025-05-18 14:54:06,043 INFO MainThread:19256 [wandb_run.py:_redirect():2371] Wrapping output streams. -2025-05-18 14:54:06,043 INFO MainThread:19256 [wandb_run.py:_redirect():2394] Redirects installed. -2025-05-18 14:54:06,044 INFO MainThread:19256 [wandb_init.py:init():1056] run started, returning control to user process -2025-05-19 14:27:35,719 INFO MsgRouterThr:19256 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles. +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Configure stats pid to 830750 +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug.log +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-internal.log +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:init():781] calling init triggers +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'shylee/so100_pengripG', 'root': '/SSD/LSY/lerobot_dataset/pengripG', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {'observation.state': {'type': , 'shape': [6]}, 'observation.images.FrontCam': {'type': , 'shape': [3, 480, 640]}, 'observation.images.TopCam': {'type': , 'shape': [3, 480, 640]}, 'observation.images.WristCam': {'type': , 'shape': [3, 480, 640]}}, 'output_features': {'action': {'type': , 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ac': False, 'ac_threshold': 0.99, 'use_pcg': False, 'pcg_weight': 0.1, 'use_bid': False, 'bid_batch_size': 15, 'bid_mode_size': 3, 'bid_temporal_coherence': 0.5, 'weak_model_path': None, 'weak_model_config': None}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain', 'job_name': 'DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain', 'resume': True, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 900000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'bqr8y636', 'mode': None}, '_wandb': {}} +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:init():809] starting backend +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:init():813] sending inform_init request +2025-05-19 15:05:23,413 INFO MainThread:830750 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-05-19 15:05:23,413 INFO MainThread:830750 [wandb_init.py:init():823] backend started and connected +2025-05-19 15:05:23,418 INFO MainThread:830750 [wandb_init.py:init():915] updated telemetry +2025-05-19 15:05:26,076 INFO MainThread:830750 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-05-19 15:05:27,428 INFO MainThread:830750 [wandb_init.py:init():1009] run resumed +2025-05-19 15:05:27,431 INFO MainThread:830750 [wandb_init.py:init():1014] starting run threads in backend +2025-05-19 15:05:27,834 INFO MainThread:830750 [wandb_run.py:_console_start():2454] atexit reg +2025-05-19 15:05:27,834 INFO MainThread:830750 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-05-19 15:05:27,834 INFO MainThread:830750 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-05-19 15:05:27,834 INFO MainThread:830750 [wandb_run.py:_redirect():2394] Redirects installed. +2025-05-19 15:05:27,836 INFO MainThread:830750 [wandb_init.py:init():1056] run started, returning control to user process diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/files/output.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7056b2885ca11b9a98ac69e082575c55e744a125 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/files/output.log @@ -0,0 +1,1460 @@ +Logs will be synced with wandb. +INFO 2025-05-19 15:05:27 ndb_utils.py:96 Track this run --> https://wandb.ai/marchmelo0923-postech/lerobot/runs/bqr8y636 +INFO 2025-05-19 15:05:28 ts/train.py:127 Creating dataset +Resolving data files: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████| 300/300 [00:00<00:00, 189615.91it/s] +INFO 2025-05-19 15:05:29 ts/train.py:138 Creating policy +Loading weights from local directory +INFO 2025-05-19 15:05:31 ts/train.py:144 Creating optimizer and scheduler +INFO 2025-05-19 15:05:31 ts/train.py:156 Output dir: /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain +INFO 2025-05-19 15:05:31 ts/train.py:159 cfg.steps=900000 (900K) +INFO 2025-05-19 15:05:31 ts/train.py:160 dataset.num_frames=63435 (63K) +INFO 2025-05-19 15:05:31 ts/train.py:161 dataset.num_episodes=300 +INFO 2025-05-19 15:05:31 ts/train.py:162 num_learnable_params=92242982 (92M) +INFO 2025-05-19 15:05:31 ts/train.py:163 num_total_params=92243036 (92M) +INFO 2025-05-19 15:05:31 ts/train.py:202 Start offline training on a fixed dataset +INFO 2025-05-19 15:06:36 ts/train.py:232 step:260K smpl:4M ep:20K epch:65.63 loss:0.007 grdn:0.159 lr:8.1e-05 updt_s:0.244 data_s:0.081 +INFO 2025-05-19 15:07:34 ts/train.py:232 step:260K smpl:4M ep:20K epch:65.68 loss:0.007 grdn:0.161 lr:8.1e-05 updt_s:0.221 data_s:0.068 +INFO 2025-05-19 15:08:32 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.73 loss:0.007 grdn:0.160 lr:8.1e-05 updt_s:0.221 data_s:0.067 +INFO 2025-05-19 15:09:29 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.78 loss:0.007 grdn:0.170 lr:8.1e-05 updt_s:0.221 data_s:0.064 +INFO 2025-05-19 15:10:26 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.83 loss:0.008 grdn:0.166 lr:8.1e-05 updt_s:0.221 data_s:0.065 +INFO 2025-05-19 15:11:24 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.88 loss:0.007 grdn:0.167 lr:8.1e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 15:12:23 ts/train.py:232 step:261K smpl:4M ep:20K epch:65.93 loss:0.007 grdn:0.165 lr:8.1e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 15:13:21 ts/train.py:232 step:262K smpl:4M ep:20K epch:65.98 loss:0.006 grdn:0.155 lr:8.1e-05 updt_s:0.221 data_s:0.066 +INFO 2025-05-19 15:14:19 ts/train.py:232 step:262K smpl:4M ep:20K epch:66.03 loss:0.007 grdn:0.164 lr:8.1e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 15:15:17 ts/train.py:232 step:262K smpl:4M ep:20K epch:66.08 loss:0.007 grdn:0.158 lr:8.1e-05 updt_s:0.222 data_s:0.067 +INFO 2025-05-19 15:16:15 ts/train.py:232 step:262K smpl:4M ep:20K epch:66.13 loss:0.006 grdn:0.159 lr:8.1e-05 updt_s:0.221 data_s:0.068 +INFO 2025-05-19 15:17:12 ts/train.py:232 step:262K smpl:4M ep:20K epch:66.18 loss:0.007 grdn:0.165 lr:8.1e-05 updt_s:0.221 data_s:0.063 +INFO 2025-05-19 15:18:10 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.23 loss:0.007 grdn:0.161 lr:8.0e-05 updt_s:0.221 data_s:0.068 +INFO 2025-05-19 15:19:07 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.29 loss:0.007 grdn:0.169 lr:8.0e-05 updt_s:0.222 data_s:0.064 +INFO 2025-05-19 15:20:05 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.34 loss:0.007 grdn:0.166 lr:8.0e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 15:21:03 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.39 loss:0.007 grdn:0.159 lr:8.0e-05 updt_s:0.221 data_s:0.066 +INFO 2025-05-19 15:22:01 ts/train.py:232 step:263K smpl:4M ep:20K epch:66.44 loss:0.006 grdn:0.156 lr:8.0e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 15:22:59 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.49 loss:0.007 grdn:0.167 lr:8.0e-05 updt_s:0.221 data_s:0.067 +INFO 2025-05-19 15:23:57 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.54 loss:0.006 grdn:0.159 lr:8.0e-05 updt_s:0.221 data_s:0.068 +INFO 2025-05-19 15:25:00 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.59 loss:0.007 grdn:0.168 lr:8.0e-05 updt_s:0.228 data_s:0.088 +INFO 2025-05-19 15:25:59 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.64 loss:0.006 grdn:0.154 lr:8.0e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 15:26:58 ts/train.py:232 step:264K smpl:4M ep:20K epch:66.69 loss:0.007 grdn:0.149 lr:8.0e-05 updt_s:0.222 data_s:0.073 +INFO 2025-05-19 15:27:58 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.74 loss:0.006 grdn:0.154 lr:8.0e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 15:28:58 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.79 loss:0.006 grdn:0.158 lr:8.0e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 15:29:58 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.84 loss:0.007 grdn:0.168 lr:8.0e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 15:30:59 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.89 loss:0.007 grdn:0.161 lr:8.0e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 15:32:00 ts/train.py:232 step:265K smpl:4M ep:20K epch:66.94 loss:0.006 grdn:0.151 lr:8.0e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 15:33:03 ts/train.py:232 step:266K smpl:4M ep:20K epch:66.99 loss:0.007 grdn:0.167 lr:8.0e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 15:34:04 ts/train.py:232 step:266K smpl:4M ep:20K epch:67.04 loss:0.007 grdn:0.160 lr:8.0e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 15:35:07 ts/train.py:232 step:266K smpl:4M ep:20K epch:67.09 loss:0.007 grdn:0.158 lr:8.0e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 15:36:10 ts/train.py:232 step:266K smpl:4M ep:20K epch:67.14 loss:0.007 grdn:0.162 lr:8.0e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 15:37:11 ts/train.py:232 step:266K smpl:4M ep:20K epch:67.19 loss:0.006 grdn:0.150 lr:8.0e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 15:38:14 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.24 loss:0.007 grdn:0.161 lr:8.0e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 15:39:15 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.29 loss:0.007 grdn:0.163 lr:8.0e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 15:40:16 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.34 loss:0.007 grdn:0.166 lr:8.0e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 15:41:17 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.39 loss:0.008 grdn:0.176 lr:8.0e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 15:42:17 ts/train.py:232 step:267K smpl:4M ep:20K epch:67.45 loss:0.006 grdn:0.153 lr:8.0e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 15:43:16 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.50 loss:0.007 grdn:0.170 lr:8.0e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 15:44:22 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.55 loss:0.007 grdn:0.165 lr:8.0e-05 updt_s:0.220 data_s:0.107 +INFO 2025-05-19 15:45:27 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.60 loss:0.007 grdn:0.168 lr:8.0e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 15:46:31 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.65 loss:0.006 grdn:0.158 lr:8.0e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 15:47:36 ts/train.py:232 step:268K smpl:4M ep:20K epch:67.70 loss:0.007 grdn:0.159 lr:8.0e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 15:48:42 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.75 loss:0.007 grdn:0.163 lr:8.0e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-19 15:49:46 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.80 loss:0.007 grdn:0.160 lr:8.0e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 15:50:48 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.85 loss:0.007 grdn:0.164 lr:8.0e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-19 15:51:47 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.90 loss:0.006 grdn:0.152 lr:8.0e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 15:52:46 ts/train.py:232 step:269K smpl:4M ep:20K epch:67.95 loss:0.006 grdn:0.152 lr:8.0e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 15:53:45 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.00 loss:0.007 grdn:0.156 lr:8.0e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 15:54:44 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.05 loss:0.007 grdn:0.165 lr:7.9e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 15:55:43 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.10 loss:0.006 grdn:0.154 lr:7.9e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 15:56:41 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.15 loss:0.007 grdn:0.169 lr:7.9e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 15:57:41 ts/train.py:232 step:270K smpl:4M ep:20K epch:68.20 loss:0.007 grdn:0.159 lr:7.9e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 15:58:40 ts/train.py:232 step:271K smpl:4M ep:20K epch:68.25 loss:0.006 grdn:0.154 lr:7.9e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 15:59:39 ts/train.py:232 step:271K smpl:4M ep:20K epch:68.30 loss:0.006 grdn:0.152 lr:7.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 16:00:38 ts/train.py:232 step:271K smpl:4M ep:21K epch:68.35 loss:0.007 grdn:0.162 lr:7.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 16:01:38 ts/train.py:232 step:271K smpl:4M ep:21K epch:68.40 loss:0.007 grdn:0.167 lr:7.9e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 16:02:38 ts/train.py:232 step:271K smpl:4M ep:21K epch:68.45 loss:0.006 grdn:0.153 lr:7.9e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 16:03:43 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.50 loss:0.007 grdn:0.168 lr:7.9e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 16:04:50 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.56 loss:0.006 grdn:0.149 lr:7.9e-05 updt_s:0.221 data_s:0.115 +INFO 2025-05-19 16:05:57 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.61 loss:0.007 grdn:0.160 lr:7.9e-05 updt_s:0.221 data_s:0.113 +INFO 2025-05-19 16:07:04 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.66 loss:0.007 grdn:0.158 lr:7.9e-05 updt_s:0.221 data_s:0.115 +INFO 2025-05-19 16:08:12 ts/train.py:232 step:272K smpl:4M ep:21K epch:68.71 loss:0.007 grdn:0.157 lr:7.9e-05 updt_s:0.221 data_s:0.115 +INFO 2025-05-19 16:09:19 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.76 loss:0.007 grdn:0.162 lr:7.9e-05 updt_s:0.221 data_s:0.115 +INFO 2025-05-19 16:10:24 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.81 loss:0.006 grdn:0.156 lr:7.9e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 16:11:25 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.86 loss:0.007 grdn:0.159 lr:7.9e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 16:12:26 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.91 loss:0.006 grdn:0.153 lr:7.9e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 16:13:26 ts/train.py:232 step:273K smpl:4M ep:21K epch:68.96 loss:0.007 grdn:0.164 lr:7.9e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 16:14:27 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.01 loss:0.007 grdn:0.162 lr:7.9e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 16:15:28 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.06 loss:0.007 grdn:0.164 lr:7.9e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 16:16:29 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.11 loss:0.007 grdn:0.157 lr:7.9e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 16:17:29 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.16 loss:0.006 grdn:0.149 lr:7.9e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-19 16:18:32 ts/train.py:232 step:274K smpl:4M ep:21K epch:69.21 loss:0.006 grdn:0.157 lr:7.9e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-19 16:19:32 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.26 loss:0.007 grdn:0.160 lr:7.9e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 16:20:33 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.31 loss:0.007 grdn:0.159 lr:7.9e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 16:21:33 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.36 loss:0.008 grdn:0.171 lr:7.9e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 16:22:36 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.41 loss:0.006 grdn:0.152 lr:7.9e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 16:23:38 ts/train.py:232 step:275K smpl:4M ep:21K epch:69.46 loss:0.007 grdn:0.159 lr:7.9e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-19 16:24:39 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.51 loss:0.007 grdn:0.167 lr:7.9e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 16:25:39 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.56 loss:0.007 grdn:0.159 lr:7.9e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 16:26:39 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.61 loss:0.007 grdn:0.162 lr:7.9e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 16:27:38 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.67 loss:0.007 grdn:0.163 lr:7.9e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 16:28:37 ts/train.py:232 step:276K smpl:4M ep:21K epch:69.72 loss:0.007 grdn:0.163 lr:7.9e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 16:29:36 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.77 loss:0.007 grdn:0.161 lr:7.9e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 16:30:31 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.82 loss:0.007 grdn:0.169 lr:7.8e-05 updt_s:0.221 data_s:0.056 +INFO 2025-05-19 16:31:27 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.87 loss:0.007 grdn:0.160 lr:7.8e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-19 16:32:23 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.92 loss:0.007 grdn:0.162 lr:7.8e-05 updt_s:0.221 data_s:0.058 +INFO 2025-05-19 16:33:19 ts/train.py:232 step:277K smpl:4M ep:21K epch:69.97 loss:0.007 grdn:0.160 lr:7.8e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-19 16:34:15 ts/train.py:232 step:278K smpl:4M ep:21K epch:70.02 loss:0.007 grdn:0.161 lr:7.8e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-19 16:35:10 ts/train.py:232 step:278K smpl:4M ep:21K epch:70.07 loss:0.007 grdn:0.160 lr:7.8e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-19 16:36:06 ts/train.py:232 step:278K smpl:4M ep:21K epch:70.12 loss:0.007 grdn:0.161 lr:7.8e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-19 16:37:02 ts/train.py:232 step:278K smpl:4M ep:21K epch:70.17 loss:0.006 grdn:0.147 lr:7.8e-05 updt_s:0.221 data_s:0.058 +INFO 2025-05-19 16:37:58 ts/train.py:232 step:278K smpl:4M ep:21K epch:70.22 loss:0.007 grdn:0.166 lr:7.8e-05 updt_s:0.221 data_s:0.056 +INFO 2025-05-19 16:38:53 ts/train.py:232 step:279K smpl:4M ep:21K epch:70.27 loss:0.007 grdn:0.156 lr:7.8e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-19 16:39:49 ts/train.py:232 step:279K smpl:4M ep:21K epch:70.32 loss:0.007 grdn:0.164 lr:7.8e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-19 16:40:45 ts/train.py:232 step:279K smpl:4M ep:21K epch:70.37 loss:0.007 grdn:0.160 lr:7.8e-05 updt_s:0.221 data_s:0.058 +INFO 2025-05-19 16:41:43 ts/train.py:232 step:279K smpl:4M ep:21K epch:70.42 loss:0.007 grdn:0.159 lr:7.8e-05 updt_s:0.220 data_s:0.071 +INFO 2025-05-19 16:42:43 ts/train.py:232 step:279K smpl:4M ep:21K epch:70.47 loss:0.007 grdn:0.173 lr:7.8e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-19 16:43:42 ts/train.py:232 step:280K smpl:4M ep:21K epch:70.52 loss:0.007 grdn:0.166 lr:7.8e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 16:44:41 ts/train.py:232 step:280K smpl:4M ep:21K epch:70.57 loss:0.006 grdn:0.155 lr:7.8e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 16:45:41 ts/train.py:232 step:280K smpl:4M ep:21K epch:70.62 loss:0.006 grdn:0.150 lr:7.8e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 16:45:41 ts/train.py:241 Checkpoint policy after step 280000 +INFO 2025-05-19 16:46:44 ts/train.py:232 step:280K smpl:4M ep:21K epch:70.67 loss:0.006 grdn:0.160 lr:7.8e-05 updt_s:0.221 data_s:0.069 +INFO 2025-05-19 16:47:44 ts/train.py:232 step:280K smpl:4M ep:21K epch:70.72 loss:0.006 grdn:0.157 lr:7.8e-05 updt_s:0.222 data_s:0.080 +INFO 2025-05-19 16:48:44 ts/train.py:232 step:281K smpl:4M ep:21K epch:70.77 loss:0.006 grdn:0.153 lr:7.8e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 16:49:44 ts/train.py:232 step:281K smpl:4M ep:21K epch:70.83 loss:0.007 grdn:0.161 lr:7.8e-05 updt_s:0.221 data_s:0.082 +INFO 2025-05-19 16:50:43 ts/train.py:232 step:281K smpl:4M ep:21K epch:70.88 loss:0.007 grdn:0.164 lr:7.8e-05 updt_s:0.221 data_s:0.071 +INFO 2025-05-19 16:51:42 ts/train.py:232 step:281K smpl:4M ep:21K epch:70.93 loss:0.007 grdn:0.160 lr:7.8e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 16:52:41 ts/train.py:232 step:281K smpl:5M ep:21K epch:70.98 loss:0.006 grdn:0.145 lr:7.8e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 16:53:41 ts/train.py:232 step:282K smpl:5M ep:21K epch:71.03 loss:0.006 grdn:0.154 lr:7.8e-05 updt_s:0.221 data_s:0.077 +INFO 2025-05-19 16:54:39 ts/train.py:232 step:282K smpl:5M ep:21K epch:71.08 loss:0.007 grdn:0.149 lr:7.8e-05 updt_s:0.221 data_s:0.066 +INFO 2025-05-19 16:55:38 ts/train.py:232 step:282K smpl:5M ep:21K epch:71.13 loss:0.007 grdn:0.159 lr:7.8e-05 updt_s:0.221 data_s:0.074 +INFO 2025-05-19 16:56:38 ts/train.py:232 step:282K smpl:5M ep:21K epch:71.18 loss:0.008 grdn:0.174 lr:7.8e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-19 16:57:38 ts/train.py:232 step:282K smpl:5M ep:21K epch:71.23 loss:0.007 grdn:0.160 lr:7.8e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-19 16:58:36 ts/train.py:232 step:283K smpl:5M ep:21K epch:71.28 loss:0.006 grdn:0.157 lr:7.8e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 16:59:34 ts/train.py:232 step:283K smpl:5M ep:21K epch:71.33 loss:0.006 grdn:0.149 lr:7.8e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 17:00:33 ts/train.py:232 step:283K smpl:5M ep:21K epch:71.38 loss:0.006 grdn:0.156 lr:7.8e-05 updt_s:0.222 data_s:0.071 +INFO 2025-05-19 17:01:41 ts/train.py:232 step:283K smpl:5M ep:21K epch:71.43 loss:0.006 grdn:0.155 lr:7.8e-05 updt_s:0.221 data_s:0.117 +INFO 2025-05-19 17:02:46 ts/train.py:232 step:283K smpl:5M ep:21K epch:71.48 loss:0.006 grdn:0.148 lr:7.8e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-19 17:03:51 ts/train.py:232 step:284K smpl:5M ep:21K epch:71.53 loss:0.006 grdn:0.152 lr:7.8e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 17:04:55 ts/train.py:232 step:284K smpl:5M ep:21K epch:71.58 loss:0.006 grdn:0.160 lr:7.7e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 17:06:00 ts/train.py:232 step:284K smpl:5M ep:21K epch:71.63 loss:0.007 grdn:0.165 lr:7.7e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-19 17:07:05 ts/train.py:232 step:284K smpl:5M ep:22K epch:71.68 loss:0.006 grdn:0.155 lr:7.7e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 17:07:56 ts/train.py:232 step:284K smpl:5M ep:22K epch:71.73 loss:0.007 grdn:0.161 lr:7.7e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-19 17:08:46 ts/train.py:232 step:285K smpl:5M ep:22K epch:71.78 loss:0.007 grdn:0.159 lr:7.7e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-19 17:09:37 ts/train.py:232 step:285K smpl:5M ep:22K epch:71.83 loss:0.007 grdn:0.164 lr:7.7e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-19 17:10:28 ts/train.py:232 step:285K smpl:5M ep:22K epch:71.88 loss:0.006 grdn:0.147 lr:7.7e-05 updt_s:0.222 data_s:0.033 +INFO 2025-05-19 17:11:18 ts/train.py:232 step:285K smpl:5M ep:22K epch:71.94 loss:0.007 grdn:0.159 lr:7.7e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-19 17:12:08 ts/train.py:232 step:285K smpl:5M ep:22K epch:71.99 loss:0.007 grdn:0.170 lr:7.7e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-19 17:12:58 ts/train.py:232 step:286K smpl:5M ep:22K epch:72.04 loss:0.007 grdn:0.156 lr:7.7e-05 updt_s:0.222 data_s:0.027 +INFO 2025-05-19 17:13:48 ts/train.py:232 step:286K smpl:5M ep:22K epch:72.09 loss:0.006 grdn:0.148 lr:7.7e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-19 17:14:38 ts/train.py:232 step:286K smpl:5M ep:22K epch:72.14 loss:0.006 grdn:0.153 lr:7.7e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-19 17:15:29 ts/train.py:232 step:286K smpl:5M ep:22K epch:72.19 loss:0.006 grdn:0.155 lr:7.7e-05 updt_s:0.222 data_s:0.031 +INFO 2025-05-19 17:16:18 ts/train.py:232 step:286K smpl:5M ep:22K epch:72.24 loss:0.007 grdn:0.158 lr:7.7e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-19 17:17:07 ts/train.py:232 step:287K smpl:5M ep:22K epch:72.29 loss:0.007 grdn:0.166 lr:7.7e-05 updt_s:0.222 data_s:0.023 +INFO 2025-05-19 17:17:56 ts/train.py:232 step:287K smpl:5M ep:22K epch:72.34 loss:0.007 grdn:0.166 lr:7.7e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-19 17:18:59 ts/train.py:232 step:287K smpl:5M ep:22K epch:72.39 loss:0.007 grdn:0.169 lr:7.7e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 17:20:03 ts/train.py:232 step:287K smpl:5M ep:22K epch:72.44 loss:0.006 grdn:0.148 lr:7.7e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 17:21:07 ts/train.py:232 step:287K smpl:5M ep:22K epch:72.49 loss:0.007 grdn:0.157 lr:7.7e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 17:22:11 ts/train.py:232 step:288K smpl:5M ep:22K epch:72.54 loss:0.007 grdn:0.157 lr:7.7e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 17:23:15 ts/train.py:232 step:288K smpl:5M ep:22K epch:72.59 loss:0.007 grdn:0.164 lr:7.7e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 17:24:19 ts/train.py:232 step:288K smpl:5M ep:22K epch:72.64 loss:0.006 grdn:0.153 lr:7.7e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 17:25:12 ts/train.py:232 step:288K smpl:5M ep:22K epch:72.69 loss:0.007 grdn:0.158 lr:7.7e-05 updt_s:0.221 data_s:0.045 +INFO 2025-05-19 17:26:01 ts/train.py:232 step:288K smpl:5M ep:22K epch:72.74 loss:0.007 grdn:0.156 lr:7.7e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-19 17:26:50 ts/train.py:232 step:289K smpl:5M ep:22K epch:72.79 loss:0.007 grdn:0.159 lr:7.7e-05 updt_s:0.221 data_s:0.021 +INFO 2025-05-19 17:27:38 ts/train.py:232 step:289K smpl:5M ep:22K epch:72.84 loss:0.007 grdn:0.166 lr:7.7e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-19 17:28:27 ts/train.py:232 step:289K smpl:5M ep:22K epch:72.89 loss:0.007 grdn:0.170 lr:7.7e-05 updt_s:0.222 data_s:0.020 +INFO 2025-05-19 17:29:16 ts/train.py:232 step:289K smpl:5M ep:22K epch:72.94 loss:0.007 grdn:0.167 lr:7.7e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 17:30:05 ts/train.py:232 step:289K smpl:5M ep:22K epch:72.99 loss:0.006 grdn:0.156 lr:7.7e-05 updt_s:0.222 data_s:0.020 +INFO 2025-05-19 17:30:53 ts/train.py:232 step:290K smpl:5M ep:22K epch:73.04 loss:0.006 grdn:0.163 lr:7.7e-05 updt_s:0.222 data_s:0.020 +INFO 2025-05-19 17:31:41 ts/train.py:232 step:290K smpl:5M ep:22K epch:73.10 loss:0.007 grdn:0.173 lr:7.7e-05 updt_s:0.222 data_s:0.017 +INFO 2025-05-19 17:32:29 ts/train.py:232 step:290K smpl:5M ep:22K epch:73.15 loss:0.006 grdn:0.160 lr:7.7e-05 updt_s:0.222 data_s:0.017 +INFO 2025-05-19 17:33:17 ts/train.py:232 step:290K smpl:5M ep:22K epch:73.20 loss:0.007 grdn:0.172 lr:7.7e-05 updt_s:0.222 data_s:0.018 +INFO 2025-05-19 17:34:06 ts/train.py:232 step:290K smpl:5M ep:22K epch:73.25 loss:0.007 grdn:0.162 lr:7.7e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-19 17:34:55 ts/train.py:232 step:291K smpl:5M ep:22K epch:73.30 loss:0.007 grdn:0.161 lr:7.6e-05 updt_s:0.222 data_s:0.022 +INFO 2025-05-19 17:35:56 ts/train.py:232 step:291K smpl:5M ep:22K epch:73.35 loss:0.007 grdn:0.165 lr:7.6e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 17:36:59 ts/train.py:232 step:291K smpl:5M ep:22K epch:73.40 loss:0.006 grdn:0.147 lr:7.6e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 17:38:03 ts/train.py:232 step:291K smpl:5M ep:22K epch:73.45 loss:0.006 grdn:0.151 lr:7.6e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 17:39:06 ts/train.py:232 step:291K smpl:5M ep:22K epch:73.50 loss:0.007 grdn:0.167 lr:7.6e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 17:40:10 ts/train.py:232 step:292K smpl:5M ep:22K epch:73.55 loss:0.007 grdn:0.165 lr:7.6e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 17:41:13 ts/train.py:232 step:292K smpl:5M ep:22K epch:73.60 loss:0.007 grdn:0.168 lr:7.6e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 17:42:09 ts/train.py:232 step:292K smpl:5M ep:22K epch:73.65 loss:0.007 grdn:0.171 lr:7.6e-05 updt_s:0.221 data_s:0.060 +INFO 2025-05-19 17:43:00 ts/train.py:232 step:292K smpl:5M ep:22K epch:73.70 loss:0.006 grdn:0.149 lr:7.6e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 17:43:50 ts/train.py:232 step:292K smpl:5M ep:22K epch:73.75 loss:0.007 grdn:0.158 lr:7.6e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 17:44:40 ts/train.py:232 step:293K smpl:5M ep:22K epch:73.80 loss:0.006 grdn:0.159 lr:7.6e-05 updt_s:0.222 data_s:0.027 +INFO 2025-05-19 17:45:30 ts/train.py:232 step:293K smpl:5M ep:22K epch:73.85 loss:0.007 grdn:0.161 lr:7.6e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-19 17:46:20 ts/train.py:232 step:293K smpl:5M ep:22K epch:73.90 loss:0.006 grdn:0.155 lr:7.6e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-19 17:47:09 ts/train.py:232 step:293K smpl:5M ep:22K epch:73.95 loss:0.006 grdn:0.149 lr:7.6e-05 updt_s:0.222 data_s:0.024 +INFO 2025-05-19 17:47:59 ts/train.py:232 step:293K smpl:5M ep:22K epch:74.00 loss:0.007 grdn:0.166 lr:7.6e-05 updt_s:0.222 data_s:0.027 +INFO 2025-05-19 17:48:48 ts/train.py:232 step:294K smpl:5M ep:22K epch:74.05 loss:0.007 grdn:0.160 lr:7.6e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-19 17:49:40 ts/train.py:232 step:294K smpl:5M ep:22K epch:74.10 loss:0.006 grdn:0.153 lr:7.6e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-19 17:50:31 ts/train.py:232 step:294K smpl:5M ep:22K epch:74.15 loss:0.007 grdn:0.164 lr:7.6e-05 updt_s:0.222 data_s:0.032 +INFO 2025-05-19 17:51:20 ts/train.py:232 step:294K smpl:5M ep:22K epch:74.21 loss:0.006 grdn:0.153 lr:7.6e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 17:52:12 ts/train.py:232 step:294K smpl:5M ep:22K epch:74.26 loss:0.006 grdn:0.158 lr:7.6e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-19 17:53:11 ts/train.py:232 step:295K smpl:5M ep:22K epch:74.31 loss:0.007 grdn:0.161 lr:7.6e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-19 17:54:14 ts/train.py:232 step:295K smpl:5M ep:22K epch:74.36 loss:0.006 grdn:0.160 lr:7.6e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-19 17:55:17 ts/train.py:232 step:295K smpl:5M ep:22K epch:74.41 loss:0.007 grdn:0.164 lr:7.6e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 17:56:20 ts/train.py:232 step:295K smpl:5M ep:22K epch:74.46 loss:0.006 grdn:0.154 lr:7.6e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 17:57:25 ts/train.py:232 step:295K smpl:5M ep:22K epch:74.51 loss:0.006 grdn:0.161 lr:7.6e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 17:58:27 ts/train.py:232 step:296K smpl:5M ep:22K epch:74.56 loss:0.007 grdn:0.158 lr:7.6e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 17:59:26 ts/train.py:232 step:296K smpl:5M ep:22K epch:74.61 loss:0.006 grdn:0.158 lr:7.6e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 18:00:15 ts/train.py:232 step:296K smpl:5M ep:22K epch:74.66 loss:0.007 grdn:0.157 lr:7.6e-05 updt_s:0.221 data_s:0.020 +INFO 2025-05-19 18:01:03 ts/train.py:232 step:296K smpl:5M ep:22K epch:74.71 loss:0.006 grdn:0.160 lr:7.6e-05 updt_s:0.222 data_s:0.018 +INFO 2025-05-19 18:01:51 ts/train.py:232 step:296K smpl:5M ep:22K epch:74.76 loss:0.007 grdn:0.169 lr:7.6e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-19 18:02:39 ts/train.py:232 step:297K smpl:5M ep:22K epch:74.81 loss:0.007 grdn:0.164 lr:7.6e-05 updt_s:0.222 data_s:0.018 +INFO 2025-05-19 18:03:29 ts/train.py:232 step:297K smpl:5M ep:22K epch:74.86 loss:0.007 grdn:0.167 lr:7.6e-05 updt_s:0.222 data_s:0.023 +INFO 2025-05-19 18:04:17 ts/train.py:232 step:297K smpl:5M ep:22K epch:74.91 loss:0.007 grdn:0.171 lr:7.6e-05 updt_s:0.222 data_s:0.022 +INFO 2025-05-19 18:05:06 ts/train.py:232 step:297K smpl:5M ep:22K epch:74.96 loss:0.007 grdn:0.161 lr:7.5e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-19 18:05:54 ts/train.py:232 step:297K smpl:5M ep:23K epch:75.01 loss:0.006 grdn:0.152 lr:7.5e-05 updt_s:0.221 data_s:0.019 +INFO 2025-05-19 18:06:43 ts/train.py:232 step:298K smpl:5M ep:23K epch:75.06 loss:0.006 grdn:0.157 lr:7.5e-05 updt_s:0.222 data_s:0.020 +INFO 2025-05-19 18:07:31 ts/train.py:232 step:298K smpl:5M ep:23K epch:75.11 loss:0.007 grdn:0.166 lr:7.5e-05 updt_s:0.221 data_s:0.020 +INFO 2025-05-19 18:08:20 ts/train.py:232 step:298K smpl:5M ep:23K epch:75.16 loss:0.006 grdn:0.151 lr:7.5e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-19 18:09:09 ts/train.py:232 step:298K smpl:5M ep:23K epch:75.21 loss:0.006 grdn:0.158 lr:7.5e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-19 18:10:04 ts/train.py:232 step:298K smpl:5M ep:23K epch:75.26 loss:0.007 grdn:0.166 lr:7.5e-05 updt_s:0.221 data_s:0.053 +INFO 2025-05-19 18:11:08 ts/train.py:232 step:299K smpl:5M ep:23K epch:75.31 loss:0.006 grdn:0.152 lr:7.5e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 18:12:11 ts/train.py:232 step:299K smpl:5M ep:23K epch:75.37 loss:0.007 grdn:0.171 lr:7.5e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 18:13:14 ts/train.py:232 step:299K smpl:5M ep:23K epch:75.42 loss:0.006 grdn:0.161 lr:7.5e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 18:14:16 ts/train.py:232 step:299K smpl:5M ep:23K epch:75.47 loss:0.008 grdn:0.180 lr:7.5e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 18:15:20 ts/train.py:232 step:299K smpl:5M ep:23K epch:75.52 loss:0.006 grdn:0.159 lr:7.5e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 18:16:21 ts/train.py:232 step:300K smpl:5M ep:23K epch:75.57 loss:0.007 grdn:0.162 lr:7.5e-05 updt_s:0.220 data_s:0.083 +INFO 2025-05-19 18:17:09 ts/train.py:232 step:300K smpl:5M ep:23K epch:75.62 loss:0.006 grdn:0.156 lr:7.5e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 18:17:58 ts/train.py:232 step:300K smpl:5M ep:23K epch:75.67 loss:0.007 grdn:0.158 lr:7.5e-05 updt_s:0.222 data_s:0.020 +INFO 2025-05-19 18:17:58 ts/train.py:241 Checkpoint policy after step 300000 +INFO 2025-05-19 18:18:50 ts/train.py:232 step:300K smpl:5M ep:23K epch:75.72 loss:0.006 grdn:0.159 lr:7.5e-05 updt_s:0.221 data_s:0.018 +INFO 2025-05-19 18:19:39 ts/train.py:232 step:300K smpl:5M ep:23K epch:75.77 loss:0.006 grdn:0.151 lr:7.5e-05 updt_s:0.221 data_s:0.019 +INFO 2025-05-19 18:20:26 ts/train.py:232 step:301K smpl:5M ep:23K epch:75.82 loss:0.007 grdn:0.163 lr:7.5e-05 updt_s:0.221 data_s:0.018 +INFO 2025-05-19 18:21:15 ts/train.py:232 step:301K smpl:5M ep:23K epch:75.87 loss:0.006 grdn:0.151 lr:7.5e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 18:22:04 ts/train.py:232 step:301K smpl:5M ep:23K epch:75.92 loss:0.007 grdn:0.166 lr:7.5e-05 updt_s:0.221 data_s:0.020 +INFO 2025-05-19 18:22:52 ts/train.py:232 step:301K smpl:5M ep:23K epch:75.97 loss:0.007 grdn:0.162 lr:7.5e-05 updt_s:0.221 data_s:0.019 +INFO 2025-05-19 18:23:40 ts/train.py:232 step:301K smpl:5M ep:23K epch:76.02 loss:0.006 grdn:0.164 lr:7.5e-05 updt_s:0.221 data_s:0.021 +INFO 2025-05-19 18:24:28 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.07 loss:0.006 grdn:0.147 lr:7.5e-05 updt_s:0.221 data_s:0.017 +INFO 2025-05-19 18:25:17 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.12 loss:0.007 grdn:0.167 lr:7.5e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-19 18:26:05 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.17 loss:0.006 grdn:0.156 lr:7.5e-05 updt_s:0.221 data_s:0.020 +INFO 2025-05-19 18:26:58 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.22 loss:0.006 grdn:0.159 lr:7.5e-05 updt_s:0.220 data_s:0.043 +INFO 2025-05-19 18:28:02 ts/train.py:232 step:302K smpl:5M ep:23K epch:76.27 loss:0.006 grdn:0.163 lr:7.5e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-19 18:29:06 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.32 loss:0.006 grdn:0.158 lr:7.5e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-19 18:30:08 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.37 loss:0.007 grdn:0.167 lr:7.5e-05 updt_s:0.220 data_s:0.092 +INFO 2025-05-19 18:31:12 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.42 loss:0.006 grdn:0.151 lr:7.5e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-19 18:32:16 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.48 loss:0.006 grdn:0.158 lr:7.5e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-19 18:33:19 ts/train.py:232 step:303K smpl:5M ep:23K epch:76.53 loss:0.007 grdn:0.167 lr:7.5e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-19 18:34:07 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.58 loss:0.007 grdn:0.167 lr:7.5e-05 updt_s:0.221 data_s:0.018 +INFO 2025-05-19 18:34:57 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.63 loss:0.007 grdn:0.171 lr:7.4e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-19 18:35:45 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.68 loss:0.006 grdn:0.156 lr:7.4e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 18:36:34 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.73 loss:0.006 grdn:0.158 lr:7.4e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-19 18:37:23 ts/train.py:232 step:304K smpl:5M ep:23K epch:76.78 loss:0.006 grdn:0.163 lr:7.4e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 18:38:12 ts/train.py:232 step:305K smpl:5M ep:23K epch:76.83 loss:0.006 grdn:0.153 lr:7.4e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 18:39:01 ts/train.py:232 step:305K smpl:5M ep:23K epch:76.88 loss:0.006 grdn:0.151 lr:7.4e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-19 18:39:50 ts/train.py:232 step:305K smpl:5M ep:23K epch:76.93 loss:0.006 grdn:0.151 lr:7.4e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 18:40:39 ts/train.py:232 step:305K smpl:5M ep:23K epch:76.98 loss:0.006 grdn:0.158 lr:7.4e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-19 18:41:29 ts/train.py:232 step:305K smpl:5M ep:23K epch:77.03 loss:0.006 grdn:0.157 lr:7.4e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-19 18:42:18 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.08 loss:0.007 grdn:0.173 lr:7.4e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-19 18:43:07 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.13 loss:0.007 grdn:0.164 lr:7.4e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-19 18:43:56 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.18 loss:0.007 grdn:0.171 lr:7.4e-05 updt_s:0.221 data_s:0.021 +INFO 2025-05-19 18:45:01 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.23 loss:0.006 grdn:0.150 lr:7.4e-05 updt_s:0.220 data_s:0.106 +INFO 2025-05-19 18:46:04 ts/train.py:232 step:306K smpl:5M ep:23K epch:77.28 loss:0.006 grdn:0.152 lr:7.4e-05 updt_s:0.220 data_s:0.095 +INFO 2025-05-19 18:47:07 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.33 loss:0.006 grdn:0.152 lr:7.4e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-19 18:48:09 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.38 loss:0.007 grdn:0.171 lr:7.4e-05 updt_s:0.220 data_s:0.089 +INFO 2025-05-19 18:49:13 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.43 loss:0.007 grdn:0.160 lr:7.4e-05 updt_s:0.220 data_s:0.095 +INFO 2025-05-19 18:50:15 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.48 loss:0.006 grdn:0.160 lr:7.4e-05 updt_s:0.220 data_s:0.092 +INFO 2025-05-19 18:51:07 ts/train.py:232 step:307K smpl:5M ep:23K epch:77.53 loss:0.006 grdn:0.153 lr:7.4e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-19 18:51:57 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.58 loss:0.006 grdn:0.156 lr:7.4e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-19 18:52:46 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.64 loss:0.007 grdn:0.172 lr:7.4e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-19 18:53:37 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.69 loss:0.006 grdn:0.159 lr:7.4e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-19 18:54:28 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.74 loss:0.006 grdn:0.155 lr:7.4e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-19 18:55:18 ts/train.py:232 step:308K smpl:5M ep:23K epch:77.79 loss:0.007 grdn:0.159 lr:7.4e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 18:56:08 ts/train.py:232 step:309K smpl:5M ep:23K epch:77.84 loss:0.006 grdn:0.151 lr:7.4e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-19 18:56:58 ts/train.py:232 step:309K smpl:5M ep:23K epch:77.89 loss:0.007 grdn:0.166 lr:7.4e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-19 18:57:49 ts/train.py:232 step:309K smpl:5M ep:23K epch:77.94 loss:0.007 grdn:0.167 lr:7.4e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 18:58:39 ts/train.py:232 step:309K smpl:5M ep:23K epch:77.99 loss:0.007 grdn:0.169 lr:7.4e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 18:59:28 ts/train.py:232 step:309K smpl:5M ep:23K epch:78.04 loss:0.007 grdn:0.159 lr:7.4e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 19:00:18 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.09 loss:0.006 grdn:0.153 lr:7.4e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-19 19:01:09 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.14 loss:0.006 grdn:0.157 lr:7.4e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 19:02:13 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.19 loss:0.006 grdn:0.153 lr:7.4e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-19 19:03:16 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.24 loss:0.007 grdn:0.164 lr:7.4e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 19:04:19 ts/train.py:232 step:310K smpl:5M ep:23K epch:78.29 loss:0.007 grdn:0.162 lr:7.3e-05 updt_s:0.220 data_s:0.095 +INFO 2025-05-19 19:05:21 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.34 loss:0.006 grdn:0.154 lr:7.3e-05 updt_s:0.220 data_s:0.092 +INFO 2025-05-19 19:06:24 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.39 loss:0.007 grdn:0.169 lr:7.3e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-19 19:07:28 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.44 loss:0.006 grdn:0.159 lr:7.3e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-19 19:08:22 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.49 loss:0.006 grdn:0.159 lr:7.3e-05 updt_s:0.221 data_s:0.051 +INFO 2025-05-19 19:09:12 ts/train.py:232 step:311K smpl:5M ep:24K epch:78.54 loss:0.006 grdn:0.161 lr:7.3e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-19 19:10:02 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.59 loss:0.006 grdn:0.162 lr:7.3e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-19 19:10:53 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.64 loss:0.007 grdn:0.165 lr:7.3e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-19 19:11:44 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.69 loss:0.006 grdn:0.153 lr:7.3e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 19:12:35 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.75 loss:0.006 grdn:0.146 lr:7.3e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 19:13:26 ts/train.py:232 step:312K smpl:5M ep:24K epch:78.80 loss:0.006 grdn:0.149 lr:7.3e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-19 19:14:18 ts/train.py:232 step:313K smpl:5M ep:24K epch:78.85 loss:0.006 grdn:0.160 lr:7.3e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-19 19:15:10 ts/train.py:232 step:313K smpl:5M ep:24K epch:78.90 loss:0.007 grdn:0.169 lr:7.3e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-19 19:16:02 ts/train.py:232 step:313K smpl:5M ep:24K epch:78.95 loss:0.006 grdn:0.153 lr:7.3e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-19 19:16:54 ts/train.py:232 step:313K smpl:5M ep:24K epch:79.00 loss:0.007 grdn:0.169 lr:7.3e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 19:17:47 ts/train.py:232 step:313K smpl:5M ep:24K epch:79.05 loss:0.007 grdn:0.173 lr:7.3e-05 updt_s:0.221 data_s:0.045 +INFO 2025-05-19 19:18:38 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.10 loss:0.006 grdn:0.162 lr:7.3e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 19:19:38 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.15 loss:0.007 grdn:0.172 lr:7.3e-05 updt_s:0.220 data_s:0.080 +INFO 2025-05-19 19:20:41 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.20 loss:0.006 grdn:0.153 lr:7.3e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-19 19:21:45 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.25 loss:0.006 grdn:0.147 lr:7.3e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-19 19:22:48 ts/train.py:232 step:314K smpl:5M ep:24K epch:79.30 loss:0.006 grdn:0.151 lr:7.3e-05 updt_s:0.220 data_s:0.092 +INFO 2025-05-19 19:23:51 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.35 loss:0.006 grdn:0.164 lr:7.3e-05 updt_s:0.220 data_s:0.093 +INFO 2025-05-19 19:24:53 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.40 loss:0.006 grdn:0.166 lr:7.3e-05 updt_s:0.220 data_s:0.093 +INFO 2025-05-19 19:25:50 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.45 loss:0.006 grdn:0.158 lr:7.3e-05 updt_s:0.221 data_s:0.062 +INFO 2025-05-19 19:26:38 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.50 loss:0.006 grdn:0.151 lr:7.3e-05 updt_s:0.221 data_s:0.020 +INFO 2025-05-19 19:27:27 ts/train.py:232 step:315K smpl:5M ep:24K epch:79.55 loss:0.006 grdn:0.157 lr:7.3e-05 updt_s:0.221 data_s:0.021 +INFO 2025-05-19 19:28:16 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.60 loss:0.007 grdn:0.161 lr:7.3e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-19 19:29:06 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.65 loss:0.006 grdn:0.153 lr:7.3e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-19 19:29:55 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.70 loss:0.006 grdn:0.152 lr:7.3e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-19 19:30:43 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.75 loss:0.006 grdn:0.157 lr:7.3e-05 updt_s:0.221 data_s:0.020 +INFO 2025-05-19 19:31:33 ts/train.py:232 step:316K smpl:5M ep:24K epch:79.80 loss:0.006 grdn:0.157 lr:7.3e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-19 19:32:22 ts/train.py:232 step:317K smpl:5M ep:24K epch:79.85 loss:0.007 grdn:0.165 lr:7.3e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-19 19:33:10 ts/train.py:232 step:317K smpl:5M ep:24K epch:79.91 loss:0.006 grdn:0.154 lr:7.2e-05 updt_s:0.221 data_s:0.020 +INFO 2025-05-19 19:33:59 ts/train.py:232 step:317K smpl:5M ep:24K epch:79.96 loss:0.006 grdn:0.160 lr:7.2e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-19 19:34:48 ts/train.py:232 step:317K smpl:5M ep:24K epch:80.01 loss:0.006 grdn:0.161 lr:7.2e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-19 19:35:37 ts/train.py:232 step:317K smpl:5M ep:24K epch:80.06 loss:0.006 grdn:0.160 lr:7.2e-05 updt_s:0.221 data_s:0.021 +INFO 2025-05-19 19:36:35 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.11 loss:0.006 grdn:0.157 lr:7.2e-05 updt_s:0.220 data_s:0.069 +INFO 2025-05-19 19:37:39 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.16 loss:0.006 grdn:0.160 lr:7.2e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-19 19:38:43 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.21 loss:0.006 grdn:0.150 lr:7.2e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-19 19:39:46 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.26 loss:0.006 grdn:0.151 lr:7.2e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-19 19:40:49 ts/train.py:232 step:318K smpl:5M ep:24K epch:80.31 loss:0.007 grdn:0.163 lr:7.2e-05 updt_s:0.220 data_s:0.093 +INFO 2025-05-19 19:41:52 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.36 loss:0.006 grdn:0.160 lr:7.2e-05 updt_s:0.220 data_s:0.092 +INFO 2025-05-19 19:42:51 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.41 loss:0.006 grdn:0.151 lr:7.2e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 19:43:41 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.46 loss:0.006 grdn:0.153 lr:7.2e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 19:44:31 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.51 loss:0.006 grdn:0.159 lr:7.2e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 19:45:22 ts/train.py:232 step:319K smpl:5M ep:24K epch:80.56 loss:0.006 grdn:0.162 lr:7.2e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 19:46:14 ts/train.py:232 step:320K smpl:5M ep:24K epch:80.61 loss:0.006 grdn:0.160 lr:7.2e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-19 19:47:04 ts/train.py:232 step:320K smpl:5M ep:24K epch:80.66 loss:0.007 grdn:0.177 lr:7.2e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 19:47:55 ts/train.py:232 step:320K smpl:5M ep:24K epch:80.71 loss:0.007 grdn:0.165 lr:7.2e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 19:47:55 ts/train.py:241 Checkpoint policy after step 320000 +INFO 2025-05-19 19:48:51 ts/train.py:232 step:320K smpl:5M ep:24K epch:80.76 loss:0.005 grdn:0.142 lr:7.2e-05 updt_s:0.222 data_s:0.031 +INFO 2025-05-19 19:49:42 ts/train.py:232 step:320K smpl:5M ep:24K epch:80.81 loss:0.006 grdn:0.154 lr:7.2e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 19:50:34 ts/train.py:232 step:321K smpl:5M ep:24K epch:80.86 loss:0.006 grdn:0.155 lr:7.2e-05 updt_s:0.222 data_s:0.038 +INFO 2025-05-19 19:51:26 ts/train.py:232 step:321K smpl:5M ep:24K epch:80.91 loss:0.006 grdn:0.153 lr:7.2e-05 updt_s:0.222 data_s:0.040 +INFO 2025-05-19 19:52:16 ts/train.py:232 step:321K smpl:5M ep:24K epch:80.96 loss:0.006 grdn:0.153 lr:7.2e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-19 19:53:07 ts/train.py:232 step:321K smpl:5M ep:24K epch:81.02 loss:0.006 grdn:0.163 lr:7.2e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 19:54:05 ts/train.py:232 step:321K smpl:5M ep:24K epch:81.07 loss:0.006 grdn:0.158 lr:7.2e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-19 19:55:09 ts/train.py:232 step:322K smpl:5M ep:24K epch:81.12 loss:0.005 grdn:0.147 lr:7.2e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-19 19:56:13 ts/train.py:232 step:322K smpl:5M ep:24K epch:81.17 loss:0.006 grdn:0.153 lr:7.2e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 19:57:17 ts/train.py:232 step:322K smpl:5M ep:24K epch:81.22 loss:0.007 grdn:0.174 lr:7.2e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 19:58:20 ts/train.py:232 step:322K smpl:5M ep:24K epch:81.27 loss:0.006 grdn:0.161 lr:7.2e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 19:59:23 ts/train.py:232 step:322K smpl:5M ep:24K epch:81.32 loss:0.006 grdn:0.157 lr:7.2e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 20:00:24 ts/train.py:232 step:323K smpl:5M ep:24K epch:81.37 loss:0.006 grdn:0.162 lr:7.2e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-19 20:01:13 ts/train.py:232 step:323K smpl:5M ep:24K epch:81.42 loss:0.007 grdn:0.159 lr:7.2e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-19 20:02:04 ts/train.py:232 step:323K smpl:5M ep:24K epch:81.47 loss:0.006 grdn:0.158 lr:7.2e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 20:02:54 ts/train.py:232 step:323K smpl:5M ep:24K epch:81.52 loss:0.006 grdn:0.154 lr:7.1e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 20:03:43 ts/train.py:232 step:323K smpl:5M ep:24K epch:81.57 loss:0.007 grdn:0.168 lr:7.1e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-19 20:04:32 ts/train.py:232 step:324K smpl:5M ep:24K epch:81.62 loss:0.006 grdn:0.159 lr:7.1e-05 updt_s:0.222 data_s:0.023 +INFO 2025-05-19 20:05:21 ts/train.py:232 step:324K smpl:5M ep:25K epch:81.67 loss:0.006 grdn:0.162 lr:7.1e-05 updt_s:0.222 data_s:0.023 +INFO 2025-05-19 20:06:10 ts/train.py:232 step:324K smpl:5M ep:25K epch:81.72 loss:0.006 grdn:0.162 lr:7.1e-05 updt_s:0.222 data_s:0.024 +INFO 2025-05-19 20:07:00 ts/train.py:232 step:324K smpl:5M ep:25K epch:81.77 loss:0.006 grdn:0.155 lr:7.1e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 20:07:50 ts/train.py:232 step:324K smpl:5M ep:25K epch:81.82 loss:0.006 grdn:0.163 lr:7.1e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-19 20:08:40 ts/train.py:232 step:325K smpl:5M ep:25K epch:81.87 loss:0.006 grdn:0.163 lr:7.1e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 20:09:30 ts/train.py:232 step:325K smpl:5M ep:25K epch:81.92 loss:0.006 grdn:0.157 lr:7.1e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-19 20:10:19 ts/train.py:232 step:325K smpl:5M ep:25K epch:81.97 loss:0.006 grdn:0.148 lr:7.1e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-19 20:11:13 ts/train.py:232 step:325K smpl:5M ep:25K epch:82.02 loss:0.006 grdn:0.161 lr:7.1e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-19 20:12:18 ts/train.py:232 step:325K smpl:5M ep:25K epch:82.07 loss:0.006 grdn:0.153 lr:7.1e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 20:13:20 ts/train.py:232 step:326K smpl:5M ep:25K epch:82.13 loss:0.006 grdn:0.162 lr:7.1e-05 updt_s:0.220 data_s:0.092 +INFO 2025-05-19 20:14:24 ts/train.py:232 step:326K smpl:5M ep:25K epch:82.18 loss:0.005 grdn:0.146 lr:7.1e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 20:15:28 ts/train.py:232 step:326K smpl:5M ep:25K epch:82.23 loss:0.006 grdn:0.160 lr:7.1e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-19 20:16:31 ts/train.py:232 step:326K smpl:5M ep:25K epch:82.28 loss:0.007 grdn:0.169 lr:7.1e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 20:17:34 ts/train.py:232 step:326K smpl:5M ep:25K epch:82.33 loss:0.007 grdn:0.160 lr:7.1e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 20:18:24 ts/train.py:232 step:327K smpl:5M ep:25K epch:82.38 loss:0.007 grdn:0.166 lr:7.1e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-19 20:19:13 ts/train.py:232 step:327K smpl:5M ep:25K epch:82.43 loss:0.006 grdn:0.158 lr:7.1e-05 updt_s:0.222 data_s:0.022 +INFO 2025-05-19 20:20:02 ts/train.py:232 step:327K smpl:5M ep:25K epch:82.48 loss:0.007 grdn:0.157 lr:7.1e-05 updt_s:0.222 data_s:0.024 +INFO 2025-05-19 20:20:51 ts/train.py:232 step:327K smpl:5M ep:25K epch:82.53 loss:0.005 grdn:0.152 lr:7.1e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 20:21:41 ts/train.py:232 step:327K smpl:5M ep:25K epch:82.58 loss:0.006 grdn:0.149 lr:7.1e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 20:22:30 ts/train.py:232 step:328K smpl:5M ep:25K epch:82.63 loss:0.006 grdn:0.153 lr:7.1e-05 updt_s:0.222 data_s:0.022 +INFO 2025-05-19 20:23:19 ts/train.py:232 step:328K smpl:5M ep:25K epch:82.68 loss:0.006 grdn:0.153 lr:7.1e-05 updt_s:0.222 data_s:0.024 +INFO 2025-05-19 20:24:09 ts/train.py:232 step:328K smpl:5M ep:25K epch:82.73 loss:0.006 grdn:0.151 lr:7.1e-05 updt_s:0.222 data_s:0.022 +INFO 2025-05-19 20:24:59 ts/train.py:232 step:328K smpl:5M ep:25K epch:82.78 loss:0.006 grdn:0.163 lr:7.1e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-19 20:25:48 ts/train.py:232 step:328K smpl:5M ep:25K epch:82.83 loss:0.006 grdn:0.152 lr:7.1e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 20:26:38 ts/train.py:232 step:329K smpl:5M ep:25K epch:82.88 loss:0.007 grdn:0.162 lr:7.1e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 20:27:28 ts/train.py:232 step:329K smpl:5M ep:25K epch:82.93 loss:0.006 grdn:0.154 lr:7.1e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-19 20:28:18 ts/train.py:232 step:329K smpl:5M ep:25K epch:82.98 loss:0.006 grdn:0.162 lr:7.1e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-19 20:29:24 ts/train.py:232 step:329K smpl:5M ep:25K epch:83.03 loss:0.007 grdn:0.167 lr:7.1e-05 updt_s:0.221 data_s:0.109 +INFO 2025-05-19 20:30:28 ts/train.py:232 step:329K smpl:5M ep:25K epch:83.08 loss:0.006 grdn:0.160 lr:7.0e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 20:31:31 ts/train.py:232 step:330K smpl:5M ep:25K epch:83.13 loss:0.006 grdn:0.157 lr:7.0e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-19 20:32:34 ts/train.py:232 step:330K smpl:5M ep:25K epch:83.18 loss:0.007 grdn:0.172 lr:7.0e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 20:33:38 ts/train.py:232 step:330K smpl:5M ep:25K epch:83.23 loss:0.006 grdn:0.155 lr:7.0e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 20:34:41 ts/train.py:232 step:330K smpl:5M ep:25K epch:83.29 loss:0.006 grdn:0.156 lr:7.0e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 20:35:33 ts/train.py:232 step:330K smpl:5M ep:25K epch:83.34 loss:0.006 grdn:0.156 lr:7.0e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-19 20:36:22 ts/train.py:232 step:331K smpl:5M ep:25K epch:83.39 loss:0.006 grdn:0.160 lr:7.0e-05 updt_s:0.222 data_s:0.024 +INFO 2025-05-19 20:37:11 ts/train.py:232 step:331K smpl:5M ep:25K epch:83.44 loss:0.006 grdn:0.152 lr:7.0e-05 updt_s:0.223 data_s:0.021 +INFO 2025-05-19 20:38:00 ts/train.py:232 step:331K smpl:5M ep:25K epch:83.49 loss:0.007 grdn:0.166 lr:7.0e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-19 20:38:50 ts/train.py:232 step:331K smpl:5M ep:25K epch:83.54 loss:0.007 grdn:0.166 lr:7.0e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 20:39:41 ts/train.py:232 step:331K smpl:5M ep:25K epch:83.59 loss:0.007 grdn:0.170 lr:7.0e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 20:40:30 ts/train.py:232 step:332K smpl:5M ep:25K epch:83.64 loss:0.006 grdn:0.144 lr:7.0e-05 updt_s:0.222 data_s:0.022 +INFO 2025-05-19 20:41:19 ts/train.py:232 step:332K smpl:5M ep:25K epch:83.69 loss:0.006 grdn:0.159 lr:7.0e-05 updt_s:0.222 data_s:0.024 +INFO 2025-05-19 20:42:08 ts/train.py:232 step:332K smpl:5M ep:25K epch:83.74 loss:0.007 grdn:0.168 lr:7.0e-05 updt_s:0.222 data_s:0.024 +INFO 2025-05-19 20:42:57 ts/train.py:232 step:332K smpl:5M ep:25K epch:83.79 loss:0.006 grdn:0.153 lr:7.0e-05 updt_s:0.222 data_s:0.018 +INFO 2025-05-19 20:43:45 ts/train.py:232 step:332K smpl:5M ep:25K epch:83.84 loss:0.006 grdn:0.155 lr:7.0e-05 updt_s:0.222 data_s:0.019 +INFO 2025-05-19 20:44:34 ts/train.py:232 step:333K smpl:5M ep:25K epch:83.89 loss:0.006 grdn:0.159 lr:7.0e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-19 20:45:24 ts/train.py:232 step:333K smpl:5M ep:25K epch:83.94 loss:0.006 grdn:0.159 lr:7.0e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-19 20:46:28 ts/train.py:232 step:333K smpl:5M ep:25K epch:83.99 loss:0.006 grdn:0.168 lr:7.0e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 20:47:32 ts/train.py:232 step:333K smpl:5M ep:25K epch:84.04 loss:0.006 grdn:0.163 lr:7.0e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 20:48:35 ts/train.py:232 step:333K smpl:5M ep:25K epch:84.09 loss:0.007 grdn:0.165 lr:7.0e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 20:49:38 ts/train.py:232 step:334K smpl:5M ep:25K epch:84.14 loss:0.006 grdn:0.152 lr:7.0e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 20:50:41 ts/train.py:232 step:334K smpl:5M ep:25K epch:84.19 loss:0.006 grdn:0.160 lr:7.0e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-19 20:51:44 ts/train.py:232 step:334K smpl:5M ep:25K epch:84.24 loss:0.006 grdn:0.162 lr:7.0e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 20:52:37 ts/train.py:232 step:334K smpl:5M ep:25K epch:84.29 loss:0.006 grdn:0.149 lr:7.0e-05 updt_s:0.222 data_s:0.047 +INFO 2025-05-19 20:53:29 ts/train.py:232 step:334K smpl:5M ep:25K epch:84.34 loss:0.006 grdn:0.149 lr:7.0e-05 updt_s:0.222 data_s:0.033 +INFO 2025-05-19 20:54:20 ts/train.py:232 step:335K smpl:5M ep:25K epch:84.40 loss:0.007 grdn:0.172 lr:7.0e-05 updt_s:0.222 data_s:0.036 +INFO 2025-05-19 20:55:11 ts/train.py:232 step:335K smpl:5M ep:25K epch:84.45 loss:0.005 grdn:0.148 lr:7.0e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 20:56:01 ts/train.py:232 step:335K smpl:5M ep:25K epch:84.50 loss:0.006 grdn:0.160 lr:7.0e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-19 20:56:52 ts/train.py:232 step:335K smpl:5M ep:25K epch:84.55 loss:0.006 grdn:0.150 lr:7.0e-05 updt_s:0.222 data_s:0.034 +INFO 2025-05-19 20:57:44 ts/train.py:232 step:335K smpl:5M ep:25K epch:84.60 loss:0.006 grdn:0.159 lr:7.0e-05 updt_s:0.222 data_s:0.037 +INFO 2025-05-19 20:58:35 ts/train.py:232 step:336K smpl:5M ep:25K epch:84.65 loss:0.007 grdn:0.177 lr:7.0e-05 updt_s:0.222 data_s:0.033 +INFO 2025-05-19 20:59:26 ts/train.py:232 step:336K smpl:5M ep:25K epch:84.70 loss:0.006 grdn:0.157 lr:6.9e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 21:00:16 ts/train.py:232 step:336K smpl:5M ep:25K epch:84.75 loss:0.006 grdn:0.157 lr:6.9e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 21:01:06 ts/train.py:232 step:336K smpl:5M ep:25K epch:84.80 loss:0.006 grdn:0.157 lr:6.9e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-19 21:01:56 ts/train.py:232 step:336K smpl:5M ep:25K epch:84.85 loss:0.006 grdn:0.157 lr:6.9e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-19 21:02:45 ts/train.py:232 step:337K smpl:5M ep:25K epch:84.90 loss:0.006 grdn:0.156 lr:6.9e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-19 21:03:46 ts/train.py:232 step:337K smpl:5M ep:25K epch:84.95 loss:0.006 grdn:0.155 lr:6.9e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-19 21:04:50 ts/train.py:232 step:337K smpl:5M ep:26K epch:85.00 loss:0.007 grdn:0.173 lr:6.9e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 21:05:54 ts/train.py:232 step:337K smpl:5M ep:26K epch:85.05 loss:0.006 grdn:0.154 lr:6.9e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 21:06:57 ts/train.py:232 step:337K smpl:5M ep:26K epch:85.10 loss:0.006 grdn:0.154 lr:6.9e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 21:08:00 ts/train.py:232 step:338K smpl:5M ep:26K epch:85.15 loss:0.006 grdn:0.163 lr:6.9e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-19 21:09:03 ts/train.py:232 step:338K smpl:5M ep:26K epch:85.20 loss:0.006 grdn:0.161 lr:6.9e-05 updt_s:0.222 data_s:0.093 +INFO 2025-05-19 21:10:00 ts/train.py:232 step:338K smpl:5M ep:26K epch:85.25 loss:0.006 grdn:0.162 lr:6.9e-05 updt_s:0.222 data_s:0.065 +INFO 2025-05-19 21:10:52 ts/train.py:232 step:338K smpl:5M ep:26K epch:85.30 loss:0.006 grdn:0.161 lr:6.9e-05 updt_s:0.222 data_s:0.036 +INFO 2025-05-19 21:11:43 ts/train.py:232 step:338K smpl:5M ep:26K epch:85.35 loss:0.006 grdn:0.153 lr:6.9e-05 updt_s:0.222 data_s:0.033 +INFO 2025-05-19 21:12:35 ts/train.py:232 step:339K smpl:5M ep:26K epch:85.40 loss:0.006 grdn:0.155 lr:6.9e-05 updt_s:0.222 data_s:0.038 +INFO 2025-05-19 21:13:26 ts/train.py:232 step:339K smpl:5M ep:26K epch:85.45 loss:0.006 grdn:0.160 lr:6.9e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 21:14:17 ts/train.py:232 step:339K smpl:5M ep:26K epch:85.50 loss:0.006 grdn:0.153 lr:6.9e-05 updt_s:0.222 data_s:0.032 +INFO 2025-05-19 21:15:07 ts/train.py:232 step:339K smpl:5M ep:26K epch:85.56 loss:0.006 grdn:0.150 lr:6.9e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-19 21:15:57 ts/train.py:232 step:339K smpl:5M ep:26K epch:85.61 loss:0.006 grdn:0.156 lr:6.9e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-19 21:16:50 ts/train.py:232 step:340K smpl:5M ep:26K epch:85.66 loss:0.007 grdn:0.174 lr:6.9e-05 updt_s:0.222 data_s:0.039 +INFO 2025-05-19 21:17:40 ts/train.py:232 step:340K smpl:5M ep:26K epch:85.71 loss:0.006 grdn:0.152 lr:6.9e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 21:18:31 ts/train.py:232 step:340K smpl:5M ep:26K epch:85.76 loss:0.006 grdn:0.164 lr:6.9e-05 updt_s:0.222 data_s:0.031 +INFO 2025-05-19 21:18:31 ts/train.py:241 Checkpoint policy after step 340000 +INFO 2025-05-19 21:19:27 ts/train.py:232 step:340K smpl:5M ep:26K epch:85.81 loss:0.006 grdn:0.168 lr:6.9e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-19 21:20:18 ts/train.py:232 step:340K smpl:5M ep:26K epch:85.86 loss:0.006 grdn:0.152 lr:6.9e-05 updt_s:0.222 data_s:0.034 +INFO 2025-05-19 21:21:18 ts/train.py:232 step:341K smpl:5M ep:26K epch:85.91 loss:0.006 grdn:0.158 lr:6.9e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-19 21:22:22 ts/train.py:232 step:341K smpl:5M ep:26K epch:85.96 loss:0.006 grdn:0.162 lr:6.9e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 21:23:26 ts/train.py:232 step:341K smpl:5M ep:26K epch:86.01 loss:0.006 grdn:0.167 lr:6.9e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-19 21:24:30 ts/train.py:232 step:341K smpl:5M ep:26K epch:86.06 loss:0.006 grdn:0.159 lr:6.9e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-19 21:25:33 ts/train.py:232 step:341K smpl:5M ep:26K epch:86.11 loss:0.006 grdn:0.149 lr:6.9e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-19 21:26:37 ts/train.py:232 step:342K smpl:5M ep:26K epch:86.16 loss:0.006 grdn:0.160 lr:6.9e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 21:27:36 ts/train.py:232 step:342K smpl:5M ep:26K epch:86.21 loss:0.005 grdn:0.138 lr:6.9e-05 updt_s:0.221 data_s:0.073 +INFO 2025-05-19 21:28:27 ts/train.py:232 step:342K smpl:5M ep:26K epch:86.26 loss:0.007 grdn:0.175 lr:6.8e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-19 21:29:17 ts/train.py:232 step:342K smpl:5M ep:26K epch:86.31 loss:0.006 grdn:0.165 lr:6.8e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-19 21:30:08 ts/train.py:232 step:342K smpl:5M ep:26K epch:86.36 loss:0.005 grdn:0.154 lr:6.8e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-19 21:30:58 ts/train.py:232 step:343K smpl:5M ep:26K epch:86.41 loss:0.006 grdn:0.164 lr:6.8e-05 updt_s:0.222 data_s:0.031 +INFO 2025-05-19 21:31:48 ts/train.py:232 step:343K smpl:5M ep:26K epch:86.46 loss:0.006 grdn:0.157 lr:6.8e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 21:32:39 ts/train.py:232 step:343K smpl:5M ep:26K epch:86.51 loss:0.006 grdn:0.150 lr:6.8e-05 updt_s:0.222 data_s:0.034 +INFO 2025-05-19 21:33:30 ts/train.py:232 step:343K smpl:5M ep:26K epch:86.56 loss:0.006 grdn:0.165 lr:6.8e-05 updt_s:0.222 data_s:0.034 +INFO 2025-05-19 21:34:21 ts/train.py:232 step:343K smpl:5M ep:26K epch:86.61 loss:0.006 grdn:0.160 lr:6.8e-05 updt_s:0.222 data_s:0.031 +INFO 2025-05-19 21:35:12 ts/train.py:232 step:344K smpl:5M ep:26K epch:86.67 loss:0.006 grdn:0.156 lr:6.8e-05 updt_s:0.222 data_s:0.035 +INFO 2025-05-19 21:36:03 ts/train.py:232 step:344K smpl:6M ep:26K epch:86.72 loss:0.005 grdn:0.143 lr:6.8e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-19 21:36:55 ts/train.py:232 step:344K smpl:6M ep:26K epch:86.77 loss:0.006 grdn:0.152 lr:6.8e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-19 21:37:47 ts/train.py:232 step:344K smpl:6M ep:26K epch:86.82 loss:0.006 grdn:0.157 lr:6.8e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-19 21:38:44 ts/train.py:232 step:344K smpl:6M ep:26K epch:86.87 loss:0.005 grdn:0.148 lr:6.8e-05 updt_s:0.222 data_s:0.062 +INFO 2025-05-19 21:39:48 ts/train.py:232 step:345K smpl:6M ep:26K epch:86.92 loss:0.006 grdn:0.160 lr:6.8e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-19 21:40:53 ts/train.py:232 step:345K smpl:6M ep:26K epch:86.97 loss:0.006 grdn:0.150 lr:6.8e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-19 21:41:57 ts/train.py:232 step:345K smpl:6M ep:26K epch:87.02 loss:0.006 grdn:0.163 lr:6.8e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-19 21:43:01 ts/train.py:232 step:345K smpl:6M ep:26K epch:87.07 loss:0.006 grdn:0.153 lr:6.8e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-19 21:44:05 ts/train.py:232 step:345K smpl:6M ep:26K epch:87.12 loss:0.006 grdn:0.165 lr:6.8e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-19 21:45:07 ts/train.py:232 step:346K smpl:6M ep:26K epch:87.17 loss:0.006 grdn:0.151 lr:6.8e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-19 21:45:55 ts/train.py:232 step:346K smpl:6M ep:26K epch:87.22 loss:0.006 grdn:0.157 lr:6.8e-05 updt_s:0.222 data_s:0.019 +INFO 2025-05-19 21:46:44 ts/train.py:232 step:346K smpl:6M ep:26K epch:87.27 loss:0.006 grdn:0.149 lr:6.8e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-19 21:47:32 ts/train.py:232 step:346K smpl:6M ep:26K epch:87.32 loss:0.006 grdn:0.161 lr:6.8e-05 updt_s:0.222 data_s:0.019 +INFO 2025-05-19 21:48:21 ts/train.py:232 step:346K smpl:6M ep:26K epch:87.37 loss:0.006 grdn:0.157 lr:6.8e-05 updt_s:0.222 data_s:0.022 +INFO 2025-05-19 21:49:11 ts/train.py:232 step:347K smpl:6M ep:26K epch:87.42 loss:0.006 grdn:0.161 lr:6.8e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-19 21:50:01 ts/train.py:232 step:347K smpl:6M ep:26K epch:87.47 loss:0.006 grdn:0.152 lr:6.8e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-19 21:50:50 ts/train.py:232 step:347K smpl:6M ep:26K epch:87.52 loss:0.005 grdn:0.152 lr:6.8e-05 updt_s:0.222 data_s:0.022 +INFO 2025-05-19 21:51:38 ts/train.py:232 step:347K smpl:6M ep:26K epch:87.57 loss:0.006 grdn:0.159 lr:6.8e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-19 21:52:27 ts/train.py:232 step:347K smpl:6M ep:26K epch:87.62 loss:0.006 grdn:0.154 lr:6.8e-05 updt_s:0.222 data_s:0.022 +INFO 2025-05-19 21:53:16 ts/train.py:232 step:348K smpl:6M ep:26K epch:87.67 loss:0.006 grdn:0.148 lr:6.8e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-19 21:54:05 ts/train.py:232 step:348K smpl:6M ep:26K epch:87.72 loss:0.007 grdn:0.175 lr:6.8e-05 updt_s:0.222 data_s:0.023 +INFO 2025-05-19 21:54:54 ts/train.py:232 step:348K smpl:6M ep:26K epch:87.77 loss:0.006 grdn:0.149 lr:6.7e-05 updt_s:0.221 data_s:0.021 +INFO 2025-05-19 21:55:46 ts/train.py:232 step:348K smpl:6M ep:26K epch:87.83 loss:0.006 grdn:0.169 lr:6.7e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-19 21:56:51 ts/train.py:232 step:348K smpl:6M ep:26K epch:87.88 loss:0.006 grdn:0.156 lr:6.7e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 21:57:55 ts/train.py:232 step:349K smpl:6M ep:26K epch:87.93 loss:0.006 grdn:0.163 lr:6.7e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 21:59:00 ts/train.py:232 step:349K smpl:6M ep:26K epch:87.98 loss:0.006 grdn:0.162 lr:6.7e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 22:00:04 ts/train.py:232 step:349K smpl:6M ep:26K epch:88.03 loss:0.006 grdn:0.177 lr:6.7e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-19 22:01:08 ts/train.py:232 step:349K smpl:6M ep:26K epch:88.08 loss:0.006 grdn:0.152 lr:6.7e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-19 22:02:12 ts/train.py:232 step:349K smpl:6M ep:26K epch:88.13 loss:0.007 grdn:0.164 lr:6.7e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-19 22:03:03 ts/train.py:232 step:350K smpl:6M ep:26K epch:88.18 loss:0.007 grdn:0.177 lr:6.7e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-19 22:03:56 ts/train.py:232 step:350K smpl:6M ep:26K epch:88.23 loss:0.006 grdn:0.155 lr:6.7e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-19 22:04:49 ts/train.py:232 step:350K smpl:6M ep:26K epch:88.28 loss:0.005 grdn:0.150 lr:6.7e-05 updt_s:0.221 data_s:0.045 +INFO 2025-05-19 22:05:40 ts/train.py:232 step:350K smpl:6M ep:26K epch:88.33 loss:0.006 grdn:0.151 lr:6.7e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-19 22:06:33 ts/train.py:232 step:350K smpl:6M ep:27K epch:88.38 loss:0.006 grdn:0.153 lr:6.7e-05 updt_s:0.222 data_s:0.043 +INFO 2025-05-19 22:07:24 ts/train.py:232 step:351K smpl:6M ep:27K epch:88.43 loss:0.006 grdn:0.152 lr:6.7e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-19 22:08:15 ts/train.py:232 step:351K smpl:6M ep:27K epch:88.48 loss:0.006 grdn:0.153 lr:6.7e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 22:09:07 ts/train.py:232 step:351K smpl:6M ep:27K epch:88.53 loss:0.006 grdn:0.166 lr:6.7e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-19 22:09:58 ts/train.py:232 step:351K smpl:6M ep:27K epch:88.58 loss:0.006 grdn:0.152 lr:6.7e-05 updt_s:0.222 data_s:0.031 +INFO 2025-05-19 22:10:50 ts/train.py:232 step:351K smpl:6M ep:27K epch:88.63 loss:0.006 grdn:0.159 lr:6.7e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-19 22:11:42 ts/train.py:232 step:352K smpl:6M ep:27K epch:88.68 loss:0.006 grdn:0.160 lr:6.7e-05 updt_s:0.222 data_s:0.042 +INFO 2025-05-19 22:12:35 ts/train.py:232 step:352K smpl:6M ep:27K epch:88.73 loss:0.006 grdn:0.157 lr:6.7e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-19 22:13:26 ts/train.py:232 step:352K smpl:6M ep:27K epch:88.78 loss:0.006 grdn:0.160 lr:6.7e-05 updt_s:0.222 data_s:0.034 +INFO 2025-05-19 22:14:32 ts/train.py:232 step:352K smpl:6M ep:27K epch:88.83 loss:0.006 grdn:0.163 lr:6.7e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 22:15:37 ts/train.py:232 step:352K smpl:6M ep:27K epch:88.88 loss:0.005 grdn:0.151 lr:6.7e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 22:16:41 ts/train.py:232 step:353K smpl:6M ep:27K epch:88.94 loss:0.006 grdn:0.171 lr:6.7e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-19 22:17:45 ts/train.py:232 step:353K smpl:6M ep:27K epch:88.99 loss:0.006 grdn:0.153 lr:6.7e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 22:18:50 ts/train.py:232 step:353K smpl:6M ep:27K epch:89.04 loss:0.006 grdn:0.163 lr:6.7e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-19 22:19:55 ts/train.py:232 step:353K smpl:6M ep:27K epch:89.09 loss:0.006 grdn:0.154 lr:6.7e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 22:20:48 ts/train.py:232 step:353K smpl:6M ep:27K epch:89.14 loss:0.005 grdn:0.155 lr:6.7e-05 updt_s:0.222 data_s:0.043 +INFO 2025-05-19 22:21:38 ts/train.py:232 step:354K smpl:6M ep:27K epch:89.19 loss:0.005 grdn:0.151 lr:6.7e-05 updt_s:0.223 data_s:0.026 +INFO 2025-05-19 22:22:28 ts/train.py:232 step:354K smpl:6M ep:27K epch:89.24 loss:0.006 grdn:0.154 lr:6.7e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-19 22:23:17 ts/train.py:232 step:354K smpl:6M ep:27K epch:89.29 loss:0.005 grdn:0.147 lr:6.7e-05 updt_s:0.222 data_s:0.023 +INFO 2025-05-19 22:24:07 ts/train.py:232 step:354K smpl:6M ep:27K epch:89.34 loss:0.005 grdn:0.150 lr:6.6e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-19 22:24:57 ts/train.py:232 step:354K smpl:6M ep:27K epch:89.39 loss:0.006 grdn:0.160 lr:6.6e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-19 22:25:47 ts/train.py:232 step:355K smpl:6M ep:27K epch:89.44 loss:0.006 grdn:0.154 lr:6.6e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-19 22:26:36 ts/train.py:232 step:355K smpl:6M ep:27K epch:89.49 loss:0.006 grdn:0.162 lr:6.6e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-19 22:27:25 ts/train.py:232 step:355K smpl:6M ep:27K epch:89.54 loss:0.005 grdn:0.153 lr:6.6e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-19 22:28:16 ts/train.py:232 step:355K smpl:6M ep:27K epch:89.59 loss:0.006 grdn:0.167 lr:6.6e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 22:29:05 ts/train.py:232 step:355K smpl:6M ep:27K epch:89.64 loss:0.006 grdn:0.150 lr:6.6e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-19 22:29:55 ts/train.py:232 step:356K smpl:6M ep:27K epch:89.69 loss:0.006 grdn:0.168 lr:6.6e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-19 22:30:44 ts/train.py:232 step:356K smpl:6M ep:27K epch:89.74 loss:0.007 grdn:0.176 lr:6.6e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 22:31:48 ts/train.py:232 step:356K smpl:6M ep:27K epch:89.79 loss:0.005 grdn:0.150 lr:6.6e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-19 22:32:52 ts/train.py:232 step:356K smpl:6M ep:27K epch:89.84 loss:0.005 grdn:0.152 lr:6.6e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-19 22:33:56 ts/train.py:232 step:356K smpl:6M ep:27K epch:89.89 loss:0.005 grdn:0.155 lr:6.6e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-19 22:34:59 ts/train.py:232 step:357K smpl:6M ep:27K epch:89.94 loss:0.006 grdn:0.163 lr:6.6e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-19 22:36:03 ts/train.py:232 step:357K smpl:6M ep:27K epch:89.99 loss:0.006 grdn:0.153 lr:6.6e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-19 22:37:06 ts/train.py:232 step:357K smpl:6M ep:27K epch:90.04 loss:0.006 grdn:0.161 lr:6.6e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-19 22:38:02 ts/train.py:232 step:357K smpl:6M ep:27K epch:90.10 loss:0.006 grdn:0.154 lr:6.6e-05 updt_s:0.222 data_s:0.054 +INFO 2025-05-19 22:38:55 ts/train.py:232 step:357K smpl:6M ep:27K epch:90.15 loss:0.006 grdn:0.159 lr:6.6e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-19 22:39:45 ts/train.py:232 step:358K smpl:6M ep:27K epch:90.20 loss:0.006 grdn:0.152 lr:6.6e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 22:40:36 ts/train.py:232 step:358K smpl:6M ep:27K epch:90.25 loss:0.006 grdn:0.162 lr:6.6e-05 updt_s:0.222 data_s:0.035 +INFO 2025-05-19 22:41:28 ts/train.py:232 step:358K smpl:6M ep:27K epch:90.30 loss:0.006 grdn:0.165 lr:6.6e-05 updt_s:0.222 data_s:0.036 +INFO 2025-05-19 22:42:19 ts/train.py:232 step:358K smpl:6M ep:27K epch:90.35 loss:0.006 grdn:0.158 lr:6.6e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 22:43:11 ts/train.py:232 step:358K smpl:6M ep:27K epch:90.40 loss:0.006 grdn:0.164 lr:6.6e-05 updt_s:0.222 data_s:0.034 +INFO 2025-05-19 22:44:01 ts/train.py:232 step:359K smpl:6M ep:27K epch:90.45 loss:0.005 grdn:0.148 lr:6.6e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-19 22:44:52 ts/train.py:232 step:359K smpl:6M ep:27K epch:90.50 loss:0.006 grdn:0.157 lr:6.6e-05 updt_s:0.222 data_s:0.034 +INFO 2025-05-19 22:45:44 ts/train.py:232 step:359K smpl:6M ep:27K epch:90.55 loss:0.005 grdn:0.152 lr:6.6e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-19 22:46:37 ts/train.py:232 step:359K smpl:6M ep:27K epch:90.60 loss:0.006 grdn:0.159 lr:6.6e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-19 22:47:31 ts/train.py:232 step:359K smpl:6M ep:27K epch:90.65 loss:0.005 grdn:0.147 lr:6.6e-05 updt_s:0.221 data_s:0.050 +INFO 2025-05-19 22:48:22 ts/train.py:232 step:360K smpl:6M ep:27K epch:90.70 loss:0.006 grdn:0.167 lr:6.6e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-19 22:49:26 ts/train.py:232 step:360K smpl:6M ep:27K epch:90.75 loss:0.006 grdn:0.148 lr:6.6e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-19 22:50:33 ts/train.py:232 step:360K smpl:6M ep:27K epch:90.80 loss:0.006 grdn:0.151 lr:6.6e-05 updt_s:0.220 data_s:0.114 +INFO 2025-05-19 22:50:33 ts/train.py:241 Checkpoint policy after step 360000 +INFO 2025-05-19 22:51:42 ts/train.py:232 step:360K smpl:6M ep:27K epch:90.85 loss:0.006 grdn:0.168 lr:6.5e-05 updt_s:0.220 data_s:0.105 +INFO 2025-05-19 22:52:50 ts/train.py:232 step:360K smpl:6M ep:27K epch:90.90 loss:0.005 grdn:0.155 lr:6.5e-05 updt_s:0.220 data_s:0.115 +INFO 2025-05-19 22:53:55 ts/train.py:232 step:361K smpl:6M ep:27K epch:90.95 loss:0.006 grdn:0.158 lr:6.5e-05 updt_s:0.220 data_s:0.107 +INFO 2025-05-19 22:55:02 ts/train.py:232 step:361K smpl:6M ep:27K epch:91.00 loss:0.006 grdn:0.165 lr:6.5e-05 updt_s:0.220 data_s:0.111 +INFO 2025-05-19 22:56:02 ts/train.py:232 step:361K smpl:6M ep:27K epch:91.05 loss:0.007 grdn:0.165 lr:6.5e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-19 22:56:53 ts/train.py:232 step:361K smpl:6M ep:27K epch:91.10 loss:0.006 grdn:0.155 lr:6.5e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-19 22:57:44 ts/train.py:232 step:361K smpl:6M ep:27K epch:91.15 loss:0.006 grdn:0.158 lr:6.5e-05 updt_s:0.222 data_s:0.034 +INFO 2025-05-19 22:58:35 ts/train.py:232 step:362K smpl:6M ep:27K epch:91.21 loss:0.006 grdn:0.158 lr:6.5e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 22:59:26 ts/train.py:232 step:362K smpl:6M ep:27K epch:91.26 loss:0.006 grdn:0.161 lr:6.5e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 23:00:17 ts/train.py:232 step:362K smpl:6M ep:27K epch:91.31 loss:0.005 grdn:0.158 lr:6.5e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 23:01:07 ts/train.py:232 step:362K smpl:6M ep:27K epch:91.36 loss:0.006 grdn:0.168 lr:6.5e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 23:01:57 ts/train.py:232 step:362K smpl:6M ep:27K epch:91.41 loss:0.005 grdn:0.156 lr:6.5e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 23:02:47 ts/train.py:232 step:363K smpl:6M ep:27K epch:91.46 loss:0.006 grdn:0.150 lr:6.5e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 23:03:39 ts/train.py:232 step:363K smpl:6M ep:27K epch:91.51 loss:0.006 grdn:0.163 lr:6.5e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-19 23:04:29 ts/train.py:232 step:363K smpl:6M ep:27K epch:91.56 loss:0.006 grdn:0.160 lr:6.5e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-19 23:05:20 ts/train.py:232 step:363K smpl:6M ep:27K epch:91.61 loss:0.005 grdn:0.150 lr:6.5e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-19 23:06:11 ts/train.py:232 step:363K smpl:6M ep:27K epch:91.66 loss:0.006 grdn:0.155 lr:6.5e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 23:07:10 ts/train.py:232 step:364K smpl:6M ep:28K epch:91.71 loss:0.006 grdn:0.169 lr:6.5e-05 updt_s:0.220 data_s:0.071 +INFO 2025-05-19 23:08:17 ts/train.py:232 step:364K smpl:6M ep:28K epch:91.76 loss:0.006 grdn:0.165 lr:6.5e-05 updt_s:0.221 data_s:0.113 +INFO 2025-05-19 23:09:22 ts/train.py:232 step:364K smpl:6M ep:28K epch:91.81 loss:0.006 grdn:0.153 lr:6.5e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-19 23:10:28 ts/train.py:232 step:364K smpl:6M ep:28K epch:91.86 loss:0.006 grdn:0.152 lr:6.5e-05 updt_s:0.221 data_s:0.109 +INFO 2025-05-19 23:11:33 ts/train.py:232 step:364K smpl:6M ep:28K epch:91.91 loss:0.005 grdn:0.151 lr:6.5e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 23:12:40 ts/train.py:232 step:365K smpl:6M ep:28K epch:91.96 loss:0.005 grdn:0.156 lr:6.5e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-19 23:13:40 ts/train.py:232 step:365K smpl:6M ep:28K epch:92.01 loss:0.006 grdn:0.164 lr:6.5e-05 updt_s:0.221 data_s:0.080 +INFO 2025-05-19 23:14:31 ts/train.py:232 step:365K smpl:6M ep:28K epch:92.06 loss:0.006 grdn:0.158 lr:6.5e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 23:15:21 ts/train.py:232 step:365K smpl:6M ep:28K epch:92.11 loss:0.005 grdn:0.145 lr:6.5e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 23:16:10 ts/train.py:232 step:365K smpl:6M ep:28K epch:92.16 loss:0.006 grdn:0.150 lr:6.5e-05 updt_s:0.222 data_s:0.024 +INFO 2025-05-19 23:17:01 ts/train.py:232 step:366K smpl:6M ep:28K epch:92.21 loss:0.006 grdn:0.157 lr:6.5e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 23:17:51 ts/train.py:232 step:366K smpl:6M ep:28K epch:92.26 loss:0.005 grdn:0.147 lr:6.5e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-19 23:18:41 ts/train.py:232 step:366K smpl:6M ep:28K epch:92.31 loss:0.006 grdn:0.158 lr:6.5e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 23:19:30 ts/train.py:232 step:366K smpl:6M ep:28K epch:92.37 loss:0.005 grdn:0.145 lr:6.4e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-19 23:20:20 ts/train.py:232 step:366K smpl:6M ep:28K epch:92.42 loss:0.005 grdn:0.155 lr:6.4e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-19 23:21:10 ts/train.py:232 step:367K smpl:6M ep:28K epch:92.47 loss:0.006 grdn:0.151 lr:6.4e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-19 23:22:01 ts/train.py:232 step:367K smpl:6M ep:28K epch:92.52 loss:0.006 grdn:0.160 lr:6.4e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-19 23:22:50 ts/train.py:232 step:367K smpl:6M ep:28K epch:92.57 loss:0.006 grdn:0.156 lr:6.4e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-19 23:23:41 ts/train.py:232 step:367K smpl:6M ep:28K epch:92.62 loss:0.006 grdn:0.158 lr:6.4e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-19 23:24:36 ts/train.py:232 step:367K smpl:6M ep:28K epch:92.67 loss:0.005 grdn:0.149 lr:6.4e-05 updt_s:0.222 data_s:0.056 +INFO 2025-05-19 23:25:42 ts/train.py:232 step:368K smpl:6M ep:28K epch:92.72 loss:0.006 grdn:0.168 lr:6.4e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-19 23:26:47 ts/train.py:232 step:368K smpl:6M ep:28K epch:92.77 loss:0.005 grdn:0.156 lr:6.4e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-19 23:27:53 ts/train.py:232 step:368K smpl:6M ep:28K epch:92.82 loss:0.006 grdn:0.150 lr:6.4e-05 updt_s:0.220 data_s:0.109 +INFO 2025-05-19 23:28:58 ts/train.py:232 step:368K smpl:6M ep:28K epch:92.87 loss:0.006 grdn:0.148 lr:6.4e-05 updt_s:0.220 data_s:0.102 +INFO 2025-05-19 23:30:03 ts/train.py:232 step:368K smpl:6M ep:28K epch:92.92 loss:0.006 grdn:0.155 lr:6.4e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-19 23:31:07 ts/train.py:232 step:369K smpl:6M ep:28K epch:92.97 loss:0.006 grdn:0.157 lr:6.4e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-19 23:31:58 ts/train.py:232 step:369K smpl:6M ep:28K epch:93.02 loss:0.005 grdn:0.152 lr:6.4e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 23:32:48 ts/train.py:232 step:369K smpl:6M ep:28K epch:93.07 loss:0.006 grdn:0.164 lr:6.4e-05 updt_s:0.222 data_s:0.026 +INFO 2025-05-19 23:33:40 ts/train.py:232 step:369K smpl:6M ep:28K epch:93.12 loss:0.005 grdn:0.150 lr:6.4e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-19 23:34:31 ts/train.py:232 step:369K smpl:6M ep:28K epch:93.17 loss:0.005 grdn:0.149 lr:6.4e-05 updt_s:0.222 data_s:0.032 +INFO 2025-05-19 23:35:23 ts/train.py:232 step:370K smpl:6M ep:28K epch:93.22 loss:0.005 grdn:0.155 lr:6.4e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-19 23:36:15 ts/train.py:232 step:370K smpl:6M ep:28K epch:93.27 loss:0.006 grdn:0.155 lr:6.4e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-19 23:37:06 ts/train.py:232 step:370K smpl:6M ep:28K epch:93.32 loss:0.006 grdn:0.165 lr:6.4e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 23:37:58 ts/train.py:232 step:370K smpl:6M ep:28K epch:93.37 loss:0.005 grdn:0.152 lr:6.4e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-19 23:38:50 ts/train.py:232 step:370K smpl:6M ep:28K epch:93.42 loss:0.006 grdn:0.154 lr:6.4e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-19 23:39:42 ts/train.py:232 step:371K smpl:6M ep:28K epch:93.48 loss:0.006 grdn:0.166 lr:6.4e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-19 23:40:33 ts/train.py:232 step:371K smpl:6M ep:28K epch:93.53 loss:0.005 grdn:0.156 lr:6.4e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-19 23:41:25 ts/train.py:232 step:371K smpl:6M ep:28K epch:93.58 loss:0.006 grdn:0.162 lr:6.4e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-19 23:42:18 ts/train.py:232 step:371K smpl:6M ep:28K epch:93.63 loss:0.006 grdn:0.164 lr:6.4e-05 updt_s:0.220 data_s:0.048 +INFO 2025-05-19 23:43:24 ts/train.py:232 step:371K smpl:6M ep:28K epch:93.68 loss:0.005 grdn:0.149 lr:6.4e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-19 23:44:28 ts/train.py:232 step:372K smpl:6M ep:28K epch:93.73 loss:0.006 grdn:0.169 lr:6.4e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-19 23:45:33 ts/train.py:232 step:372K smpl:6M ep:28K epch:93.78 loss:0.005 grdn:0.154 lr:6.4e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-19 23:46:38 ts/train.py:232 step:372K smpl:6M ep:28K epch:93.83 loss:0.006 grdn:0.157 lr:6.4e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-19 23:47:43 ts/train.py:232 step:372K smpl:6M ep:28K epch:93.88 loss:0.006 grdn:0.170 lr:6.3e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-19 23:48:47 ts/train.py:232 step:372K smpl:6M ep:28K epch:93.93 loss:0.005 grdn:0.145 lr:6.3e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-19 23:49:38 ts/train.py:232 step:373K smpl:6M ep:28K epch:93.98 loss:0.006 grdn:0.159 lr:6.3e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-19 23:50:30 ts/train.py:232 step:373K smpl:6M ep:28K epch:94.03 loss:0.006 grdn:0.173 lr:6.3e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-19 23:51:21 ts/train.py:232 step:373K smpl:6M ep:28K epch:94.08 loss:0.006 grdn:0.161 lr:6.3e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-19 23:52:16 ts/train.py:232 step:373K smpl:6M ep:28K epch:94.13 loss:0.006 grdn:0.158 lr:6.3e-05 updt_s:0.221 data_s:0.050 +INFO 2025-05-19 23:53:08 ts/train.py:232 step:373K smpl:6M ep:28K epch:94.18 loss:0.006 grdn:0.162 lr:6.3e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-19 23:54:02 ts/train.py:232 step:374K smpl:6M ep:28K epch:94.23 loss:0.006 grdn:0.157 lr:6.3e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-19 23:54:55 ts/train.py:232 step:374K smpl:6M ep:28K epch:94.28 loss:0.005 grdn:0.152 lr:6.3e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-19 23:55:47 ts/train.py:232 step:374K smpl:6M ep:28K epch:94.33 loss:0.006 grdn:0.153 lr:6.3e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 23:56:38 ts/train.py:232 step:374K smpl:6M ep:28K epch:94.38 loss:0.006 grdn:0.167 lr:6.3e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-19 23:57:31 ts/train.py:232 step:374K smpl:6M ep:28K epch:94.43 loss:0.006 grdn:0.159 lr:6.3e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-19 23:58:24 ts/train.py:232 step:375K smpl:6M ep:28K epch:94.48 loss:0.004 grdn:0.142 lr:6.3e-05 updt_s:0.221 data_s:0.045 +INFO 2025-05-19 23:59:17 ts/train.py:232 step:375K smpl:6M ep:28K epch:94.53 loss:0.007 grdn:0.182 lr:6.3e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-20 00:00:10 ts/train.py:232 step:375K smpl:6M ep:28K epch:94.59 loss:0.005 grdn:0.139 lr:6.3e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 00:01:17 ts/train.py:232 step:375K smpl:6M ep:28K epch:94.64 loss:0.006 grdn:0.151 lr:6.3e-05 updt_s:0.220 data_s:0.112 +INFO 2025-05-20 00:02:22 ts/train.py:232 step:375K smpl:6M ep:28K epch:94.69 loss:0.006 grdn:0.156 lr:6.3e-05 updt_s:0.220 data_s:0.106 +INFO 2025-05-20 00:03:27 ts/train.py:232 step:376K smpl:6M ep:28K epch:94.74 loss:0.006 grdn:0.156 lr:6.3e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-20 00:04:32 ts/train.py:232 step:376K smpl:6M ep:28K epch:94.79 loss:0.005 grdn:0.148 lr:6.3e-05 updt_s:0.220 data_s:0.103 +INFO 2025-05-20 00:05:38 ts/train.py:232 step:376K smpl:6M ep:28K epch:94.84 loss:0.006 grdn:0.164 lr:6.3e-05 updt_s:0.220 data_s:0.111 +INFO 2025-05-20 00:06:44 ts/train.py:232 step:376K smpl:6M ep:28K epch:94.89 loss:0.005 grdn:0.152 lr:6.3e-05 updt_s:0.220 data_s:0.108 +INFO 2025-05-20 00:07:40 ts/train.py:232 step:376K smpl:6M ep:28K epch:94.94 loss:0.006 grdn:0.168 lr:6.3e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-20 00:08:34 ts/train.py:232 step:377K smpl:6M ep:28K epch:94.99 loss:0.006 grdn:0.161 lr:6.3e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 00:09:28 ts/train.py:232 step:377K smpl:6M ep:29K epch:95.04 loss:0.006 grdn:0.151 lr:6.3e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 00:10:21 ts/train.py:232 step:377K smpl:6M ep:29K epch:95.09 loss:0.005 grdn:0.152 lr:6.3e-05 updt_s:0.222 data_s:0.043 +INFO 2025-05-20 00:11:16 ts/train.py:232 step:377K smpl:6M ep:29K epch:95.14 loss:0.006 grdn:0.158 lr:6.3e-05 updt_s:0.221 data_s:0.051 +INFO 2025-05-20 00:12:09 ts/train.py:232 step:377K smpl:6M ep:29K epch:95.19 loss:0.006 grdn:0.153 lr:6.3e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 00:13:04 ts/train.py:232 step:378K smpl:6M ep:29K epch:95.24 loss:0.005 grdn:0.155 lr:6.3e-05 updt_s:0.221 data_s:0.052 +INFO 2025-05-20 00:13:57 ts/train.py:232 step:378K smpl:6M ep:29K epch:95.29 loss:0.006 grdn:0.158 lr:6.3e-05 updt_s:0.221 data_s:0.044 +INFO 2025-05-20 00:14:50 ts/train.py:232 step:378K smpl:6M ep:29K epch:95.34 loss:0.006 grdn:0.154 lr:6.3e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 00:15:43 ts/train.py:232 step:378K smpl:6M ep:29K epch:95.39 loss:0.005 grdn:0.149 lr:6.2e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 00:16:36 ts/train.py:232 step:378K smpl:6M ep:29K epch:95.44 loss:0.006 grdn:0.166 lr:6.2e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 00:17:28 ts/train.py:232 step:379K smpl:6M ep:29K epch:95.49 loss:0.006 grdn:0.156 lr:6.2e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 00:18:20 ts/train.py:232 step:379K smpl:6M ep:29K epch:95.54 loss:0.006 grdn:0.156 lr:6.2e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-20 00:19:25 ts/train.py:232 step:379K smpl:6M ep:29K epch:95.59 loss:0.006 grdn:0.162 lr:6.2e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 00:20:30 ts/train.py:232 step:379K smpl:6M ep:29K epch:95.64 loss:0.005 grdn:0.147 lr:6.2e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-20 00:21:35 ts/train.py:232 step:379K smpl:6M ep:29K epch:95.69 loss:0.005 grdn:0.156 lr:6.2e-05 updt_s:0.220 data_s:0.102 +INFO 2025-05-20 00:22:41 ts/train.py:232 step:380K smpl:6M ep:29K epch:95.75 loss:0.005 grdn:0.153 lr:6.2e-05 updt_s:0.221 data_s:0.111 +INFO 2025-05-20 00:23:46 ts/train.py:232 step:380K smpl:6M ep:29K epch:95.80 loss:0.006 grdn:0.164 lr:6.2e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-20 00:24:53 ts/train.py:232 step:380K smpl:6M ep:29K epch:95.85 loss:0.005 grdn:0.157 lr:6.2e-05 updt_s:0.221 data_s:0.110 +INFO 2025-05-20 00:24:53 ts/train.py:241 Checkpoint policy after step 380000 +INFO 2025-05-20 00:25:55 ts/train.py:232 step:380K smpl:6M ep:29K epch:95.90 loss:0.006 grdn:0.161 lr:6.2e-05 updt_s:0.221 data_s:0.056 +INFO 2025-05-20 00:26:45 ts/train.py:232 step:380K smpl:6M ep:29K epch:95.95 loss:0.006 grdn:0.166 lr:6.2e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 00:27:36 ts/train.py:232 step:381K smpl:6M ep:29K epch:96.00 loss:0.005 grdn:0.150 lr:6.2e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 00:28:28 ts/train.py:232 step:381K smpl:6M ep:29K epch:96.05 loss:0.006 grdn:0.168 lr:6.2e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 00:29:19 ts/train.py:232 step:381K smpl:6M ep:29K epch:96.10 loss:0.005 grdn:0.156 lr:6.2e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 00:30:10 ts/train.py:232 step:381K smpl:6M ep:29K epch:96.15 loss:0.005 grdn:0.147 lr:6.2e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 00:31:02 ts/train.py:232 step:381K smpl:6M ep:29K epch:96.20 loss:0.005 grdn:0.152 lr:6.2e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 00:31:53 ts/train.py:232 step:382K smpl:6M ep:29K epch:96.25 loss:0.006 grdn:0.157 lr:6.2e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 00:32:44 ts/train.py:232 step:382K smpl:6M ep:29K epch:96.30 loss:0.006 grdn:0.163 lr:6.2e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 00:33:36 ts/train.py:232 step:382K smpl:6M ep:29K epch:96.35 loss:0.005 grdn:0.150 lr:6.2e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 00:34:28 ts/train.py:232 step:382K smpl:6M ep:29K epch:96.40 loss:0.006 grdn:0.167 lr:6.2e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-20 00:35:18 ts/train.py:232 step:382K smpl:6M ep:29K epch:96.45 loss:0.005 grdn:0.157 lr:6.2e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 00:36:08 ts/train.py:232 step:383K smpl:6M ep:29K epch:96.50 loss:0.005 grdn:0.153 lr:6.2e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-20 00:37:11 ts/train.py:232 step:383K smpl:6M ep:29K epch:96.55 loss:0.006 grdn:0.157 lr:6.2e-05 updt_s:0.220 data_s:0.089 +INFO 2025-05-20 00:38:18 ts/train.py:232 step:383K smpl:6M ep:29K epch:96.60 loss:0.006 grdn:0.160 lr:6.2e-05 updt_s:0.221 data_s:0.115 +INFO 2025-05-20 00:39:25 ts/train.py:232 step:383K smpl:6M ep:29K epch:96.65 loss:0.005 grdn:0.152 lr:6.2e-05 updt_s:0.221 data_s:0.114 +INFO 2025-05-20 00:40:31 ts/train.py:232 step:383K smpl:6M ep:29K epch:96.70 loss:0.006 grdn:0.156 lr:6.2e-05 updt_s:0.221 data_s:0.110 +INFO 2025-05-20 00:41:37 ts/train.py:232 step:384K smpl:6M ep:29K epch:96.75 loss:0.006 grdn:0.166 lr:6.2e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-20 00:42:43 ts/train.py:232 step:384K smpl:6M ep:29K epch:96.80 loss:0.006 grdn:0.168 lr:6.2e-05 updt_s:0.221 data_s:0.108 +INFO 2025-05-20 00:43:42 ts/train.py:232 step:384K smpl:6M ep:29K epch:96.86 loss:0.006 grdn:0.158 lr:6.1e-05 updt_s:0.221 data_s:0.075 +INFO 2025-05-20 00:44:36 ts/train.py:232 step:384K smpl:6M ep:29K epch:96.91 loss:0.005 grdn:0.147 lr:6.1e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 00:45:30 ts/train.py:232 step:384K smpl:6M ep:29K epch:96.96 loss:0.005 grdn:0.152 lr:6.1e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 00:46:22 ts/train.py:232 step:385K smpl:6M ep:29K epch:97.01 loss:0.006 grdn:0.150 lr:6.1e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 00:47:16 ts/train.py:232 step:385K smpl:6M ep:29K epch:97.06 loss:0.005 grdn:0.146 lr:6.1e-05 updt_s:0.221 data_s:0.045 +INFO 2025-05-20 00:48:10 ts/train.py:232 step:385K smpl:6M ep:29K epch:97.11 loss:0.005 grdn:0.152 lr:6.1e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 00:49:02 ts/train.py:232 step:385K smpl:6M ep:29K epch:97.16 loss:0.005 grdn:0.153 lr:6.1e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 00:49:57 ts/train.py:232 step:385K smpl:6M ep:29K epch:97.21 loss:0.006 grdn:0.166 lr:6.1e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 00:50:48 ts/train.py:232 step:386K smpl:6M ep:29K epch:97.26 loss:0.005 grdn:0.155 lr:6.1e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 00:51:42 ts/train.py:232 step:386K smpl:6M ep:29K epch:97.31 loss:0.005 grdn:0.158 lr:6.1e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 00:52:36 ts/train.py:232 step:386K smpl:6M ep:29K epch:97.36 loss:0.005 grdn:0.151 lr:6.1e-05 updt_s:0.221 data_s:0.044 +INFO 2025-05-20 00:53:30 ts/train.py:232 step:386K smpl:6M ep:29K epch:97.41 loss:0.005 grdn:0.159 lr:6.1e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 00:54:23 ts/train.py:232 step:386K smpl:6M ep:29K epch:97.46 loss:0.005 grdn:0.153 lr:6.1e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 00:55:24 ts/train.py:232 step:387K smpl:6M ep:29K epch:97.51 loss:0.005 grdn:0.143 lr:6.1e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-20 00:56:29 ts/train.py:232 step:387K smpl:6M ep:29K epch:97.56 loss:0.005 grdn:0.152 lr:6.1e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-20 00:57:34 ts/train.py:232 step:387K smpl:6M ep:29K epch:97.61 loss:0.006 grdn:0.164 lr:6.1e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-20 00:58:39 ts/train.py:232 step:387K smpl:6M ep:29K epch:97.66 loss:0.005 grdn:0.156 lr:6.1e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-20 00:59:44 ts/train.py:232 step:387K smpl:6M ep:29K epch:97.71 loss:0.005 grdn:0.151 lr:6.1e-05 updt_s:0.221 data_s:0.106 +INFO 2025-05-20 01:00:49 ts/train.py:232 step:388K smpl:6M ep:29K epch:97.76 loss:0.005 grdn:0.149 lr:6.1e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-20 01:01:50 ts/train.py:232 step:388K smpl:6M ep:29K epch:97.81 loss:0.005 grdn:0.146 lr:6.1e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-20 01:02:41 ts/train.py:232 step:388K smpl:6M ep:29K epch:97.86 loss:0.006 grdn:0.160 lr:6.1e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 01:03:31 ts/train.py:232 step:388K smpl:6M ep:29K epch:97.91 loss:0.006 grdn:0.162 lr:6.1e-05 updt_s:0.222 data_s:0.031 +INFO 2025-05-20 01:04:22 ts/train.py:232 step:388K smpl:6M ep:29K epch:97.96 loss:0.005 grdn:0.146 lr:6.1e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-20 01:05:12 ts/train.py:232 step:389K smpl:6M ep:29K epch:98.02 loss:0.005 grdn:0.150 lr:6.1e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 01:06:03 ts/train.py:232 step:389K smpl:6M ep:29K epch:98.07 loss:0.006 grdn:0.158 lr:6.1e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 01:06:53 ts/train.py:232 step:389K smpl:6M ep:29K epch:98.12 loss:0.005 grdn:0.145 lr:6.1e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 01:07:44 ts/train.py:232 step:389K smpl:6M ep:29K epch:98.17 loss:0.006 grdn:0.167 lr:6.1e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 01:08:34 ts/train.py:232 step:389K smpl:6M ep:29K epch:98.22 loss:0.005 grdn:0.156 lr:6.1e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 01:09:24 ts/train.py:232 step:390K smpl:6M ep:29K epch:98.27 loss:0.006 grdn:0.156 lr:6.1e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 01:10:15 ts/train.py:232 step:390K smpl:6M ep:29K epch:98.32 loss:0.006 grdn:0.163 lr:6.0e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 01:11:05 ts/train.py:232 step:390K smpl:6M ep:30K epch:98.37 loss:0.005 grdn:0.157 lr:6.0e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 01:11:56 ts/train.py:232 step:390K smpl:6M ep:30K epch:98.42 loss:0.005 grdn:0.166 lr:6.0e-05 updt_s:0.222 data_s:0.034 +INFO 2025-05-20 01:12:52 ts/train.py:232 step:390K smpl:6M ep:30K epch:98.47 loss:0.005 grdn:0.145 lr:6.0e-05 updt_s:0.221 data_s:0.060 +INFO 2025-05-20 01:14:00 ts/train.py:232 step:391K smpl:6M ep:30K epch:98.52 loss:0.006 grdn:0.152 lr:6.0e-05 updt_s:0.220 data_s:0.115 +INFO 2025-05-20 01:15:04 ts/train.py:232 step:391K smpl:6M ep:30K epch:98.57 loss:0.005 grdn:0.151 lr:6.0e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-20 01:16:09 ts/train.py:232 step:391K smpl:6M ep:30K epch:98.62 loss:0.006 grdn:0.154 lr:6.0e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-20 01:17:14 ts/train.py:232 step:391K smpl:6M ep:30K epch:98.67 loss:0.005 grdn:0.156 lr:6.0e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-20 01:18:20 ts/train.py:232 step:391K smpl:6M ep:30K epch:98.72 loss:0.005 grdn:0.147 lr:6.0e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-20 01:19:23 ts/train.py:232 step:392K smpl:6M ep:30K epch:98.77 loss:0.005 grdn:0.149 lr:6.0e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-20 01:20:12 ts/train.py:232 step:392K smpl:6M ep:30K epch:98.82 loss:0.005 grdn:0.152 lr:6.0e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 01:21:02 ts/train.py:232 step:392K smpl:6M ep:30K epch:98.87 loss:0.006 grdn:0.162 lr:6.0e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-20 01:21:53 ts/train.py:232 step:392K smpl:6M ep:30K epch:98.92 loss:0.006 grdn:0.161 lr:6.0e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 01:22:43 ts/train.py:232 step:392K smpl:6M ep:30K epch:98.97 loss:0.006 grdn:0.161 lr:6.0e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 01:23:34 ts/train.py:232 step:393K smpl:6M ep:30K epch:99.02 loss:0.006 grdn:0.161 lr:6.0e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 01:24:25 ts/train.py:232 step:393K smpl:6M ep:30K epch:99.07 loss:0.005 grdn:0.154 lr:6.0e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 01:25:15 ts/train.py:232 step:393K smpl:6M ep:30K epch:99.13 loss:0.005 grdn:0.154 lr:6.0e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-20 01:26:04 ts/train.py:232 step:393K smpl:6M ep:30K epch:99.18 loss:0.005 grdn:0.159 lr:6.0e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-20 01:26:55 ts/train.py:232 step:393K smpl:6M ep:30K epch:99.23 loss:0.005 grdn:0.153 lr:6.0e-05 updt_s:0.222 data_s:0.031 +INFO 2025-05-20 01:27:45 ts/train.py:232 step:394K smpl:6M ep:30K epch:99.28 loss:0.005 grdn:0.155 lr:6.0e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 01:28:36 ts/train.py:232 step:394K smpl:6M ep:30K epch:99.33 loss:0.005 grdn:0.165 lr:6.0e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 01:29:26 ts/train.py:232 step:394K smpl:6M ep:30K epch:99.38 loss:0.005 grdn:0.151 lr:6.0e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-20 01:30:20 ts/train.py:232 step:394K smpl:6M ep:30K epch:99.43 loss:0.005 grdn:0.161 lr:6.0e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 01:31:26 ts/train.py:232 step:394K smpl:6M ep:30K epch:99.48 loss:0.005 grdn:0.161 lr:6.0e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-20 01:32:30 ts/train.py:232 step:395K smpl:6M ep:30K epch:99.53 loss:0.005 grdn:0.155 lr:6.0e-05 updt_s:0.221 data_s:0.102 +INFO 2025-05-20 01:33:35 ts/train.py:232 step:395K smpl:6M ep:30K epch:99.58 loss:0.005 grdn:0.163 lr:6.0e-05 updt_s:0.220 data_s:0.103 +INFO 2025-05-20 01:34:40 ts/train.py:232 step:395K smpl:6M ep:30K epch:99.63 loss:0.006 grdn:0.175 lr:6.0e-05 updt_s:0.220 data_s:0.105 +INFO 2025-05-20 01:35:45 ts/train.py:232 step:395K smpl:6M ep:30K epch:99.68 loss:0.006 grdn:0.165 lr:6.0e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-20 01:36:49 ts/train.py:232 step:395K smpl:6M ep:30K epch:99.73 loss:0.006 grdn:0.161 lr:6.0e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-20 01:37:40 ts/train.py:232 step:396K smpl:6M ep:30K epch:99.78 loss:0.005 grdn:0.145 lr:6.0e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 01:38:31 ts/train.py:232 step:396K smpl:6M ep:30K epch:99.83 loss:0.005 grdn:0.159 lr:5.9e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 01:39:22 ts/train.py:232 step:396K smpl:6M ep:30K epch:99.88 loss:0.005 grdn:0.152 lr:5.9e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 01:40:12 ts/train.py:232 step:396K smpl:6M ep:30K epch:99.93 loss:0.005 grdn:0.158 lr:5.9e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-20 01:41:03 ts/train.py:232 step:396K smpl:6M ep:30K epch:99.98 loss:0.006 grdn:0.166 lr:5.9e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-20 01:41:53 ts/train.py:232 step:397K smpl:6M ep:30K epch:100.03 loss:0.006 grdn:0.165 lr:5.9e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 01:42:43 ts/train.py:232 step:397K smpl:6M ep:30K epch:100.08 loss:0.005 grdn:0.151 lr:5.9e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-20 01:43:34 ts/train.py:232 step:397K smpl:6M ep:30K epch:100.13 loss:0.005 grdn:0.157 lr:5.9e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-20 01:44:24 ts/train.py:232 step:397K smpl:6M ep:30K epch:100.18 loss:0.005 grdn:0.154 lr:5.9e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 01:45:14 ts/train.py:232 step:397K smpl:6M ep:30K epch:100.23 loss:0.006 grdn:0.164 lr:5.9e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 01:46:04 ts/train.py:232 step:398K smpl:6M ep:30K epch:100.29 loss:0.005 grdn:0.157 lr:5.9e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-20 01:46:54 ts/train.py:232 step:398K smpl:6M ep:30K epch:100.34 loss:0.006 grdn:0.165 lr:5.9e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 01:47:45 ts/train.py:232 step:398K smpl:6M ep:30K epch:100.39 loss:0.005 grdn:0.140 lr:5.9e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 01:48:51 ts/train.py:232 step:398K smpl:6M ep:30K epch:100.44 loss:0.005 grdn:0.159 lr:5.9e-05 updt_s:0.220 data_s:0.109 +INFO 2025-05-20 01:49:56 ts/train.py:232 step:398K smpl:6M ep:30K epch:100.49 loss:0.006 grdn:0.167 lr:5.9e-05 updt_s:0.220 data_s:0.106 +INFO 2025-05-20 01:51:02 ts/train.py:232 step:399K smpl:6M ep:30K epch:100.54 loss:0.006 grdn:0.160 lr:5.9e-05 updt_s:0.220 data_s:0.105 +INFO 2025-05-20 01:52:06 ts/train.py:232 step:399K smpl:6M ep:30K epch:100.59 loss:0.005 grdn:0.154 lr:5.9e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-20 01:53:10 ts/train.py:232 step:399K smpl:6M ep:30K epch:100.64 loss:0.005 grdn:0.142 lr:5.9e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 01:54:15 ts/train.py:232 step:399K smpl:6M ep:30K epch:100.69 loss:0.006 grdn:0.158 lr:5.9e-05 updt_s:0.220 data_s:0.103 +INFO 2025-05-20 01:55:11 ts/train.py:232 step:399K smpl:6M ep:30K epch:100.74 loss:0.006 grdn:0.166 lr:5.9e-05 updt_s:0.221 data_s:0.061 +INFO 2025-05-20 01:56:03 ts/train.py:232 step:400K smpl:6M ep:30K epch:100.79 loss:0.005 grdn:0.149 lr:5.9e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 01:56:56 ts/train.py:232 step:400K smpl:6M ep:30K epch:100.84 loss:0.005 grdn:0.146 lr:5.9e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 01:57:48 ts/train.py:232 step:400K smpl:6M ep:30K epch:100.89 loss:0.005 grdn:0.152 lr:5.9e-05 updt_s:0.222 data_s:0.039 +INFO 2025-05-20 01:57:48 ts/train.py:241 Checkpoint policy after step 400000 +INFO 2025-05-20 01:58:46 ts/train.py:232 step:400K smpl:6M ep:30K epch:100.94 loss:0.005 grdn:0.157 lr:5.9e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 01:59:41 ts/train.py:232 step:400K smpl:6M ep:30K epch:100.99 loss:0.005 grdn:0.153 lr:5.9e-05 updt_s:0.221 data_s:0.056 +INFO 2025-05-20 02:00:35 ts/train.py:232 step:401K smpl:6M ep:30K epch:101.04 loss:0.005 grdn:0.145 lr:5.9e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 02:01:28 ts/train.py:232 step:401K smpl:6M ep:30K epch:101.09 loss:0.005 grdn:0.156 lr:5.9e-05 updt_s:0.221 data_s:0.045 +INFO 2025-05-20 02:02:22 ts/train.py:232 step:401K smpl:6M ep:30K epch:101.14 loss:0.005 grdn:0.162 lr:5.9e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 02:03:16 ts/train.py:232 step:401K smpl:6M ep:30K epch:101.19 loss:0.005 grdn:0.146 lr:5.9e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 02:04:07 ts/train.py:232 step:401K smpl:6M ep:30K epch:101.24 loss:0.005 grdn:0.156 lr:5.9e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 02:05:01 ts/train.py:232 step:402K smpl:6M ep:30K epch:101.29 loss:0.005 grdn:0.154 lr:5.8e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 02:05:54 ts/train.py:232 step:402K smpl:6M ep:30K epch:101.34 loss:0.005 grdn:0.162 lr:5.8e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 02:06:59 ts/train.py:232 step:402K smpl:6M ep:30K epch:101.40 loss:0.006 grdn:0.161 lr:5.8e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-20 02:08:05 ts/train.py:232 step:402K smpl:6M ep:30K epch:101.45 loss:0.006 grdn:0.161 lr:5.8e-05 updt_s:0.221 data_s:0.113 +INFO 2025-05-20 02:09:11 ts/train.py:232 step:402K smpl:6M ep:30K epch:101.50 loss:0.005 grdn:0.147 lr:5.8e-05 updt_s:0.220 data_s:0.109 +INFO 2025-05-20 02:10:17 ts/train.py:232 step:403K smpl:6M ep:30K epch:101.55 loss:0.005 grdn:0.155 lr:5.8e-05 updt_s:0.220 data_s:0.108 +INFO 2025-05-20 02:11:22 ts/train.py:232 step:403K smpl:6M ep:30K epch:101.60 loss:0.005 grdn:0.145 lr:5.8e-05 updt_s:0.220 data_s:0.103 +INFO 2025-05-20 02:12:28 ts/train.py:232 step:403K smpl:6M ep:30K epch:101.65 loss:0.005 grdn:0.155 lr:5.8e-05 updt_s:0.220 data_s:0.108 +INFO 2025-05-20 02:13:24 ts/train.py:232 step:403K smpl:6M ep:31K epch:101.70 loss:0.005 grdn:0.164 lr:5.8e-05 updt_s:0.221 data_s:0.061 +INFO 2025-05-20 02:14:15 ts/train.py:232 step:403K smpl:6M ep:31K epch:101.75 loss:0.006 grdn:0.161 lr:5.8e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 02:15:04 ts/train.py:232 step:404K smpl:6M ep:31K epch:101.80 loss:0.006 grdn:0.166 lr:5.8e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 02:15:55 ts/train.py:232 step:404K smpl:6M ep:31K epch:101.85 loss:0.005 grdn:0.157 lr:5.8e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 02:16:46 ts/train.py:232 step:404K smpl:6M ep:31K epch:101.90 loss:0.005 grdn:0.140 lr:5.8e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 02:17:37 ts/train.py:232 step:404K smpl:6M ep:31K epch:101.95 loss:0.005 grdn:0.149 lr:5.8e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 02:18:28 ts/train.py:232 step:404K smpl:6M ep:31K epch:102.00 loss:0.006 grdn:0.162 lr:5.8e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 02:19:19 ts/train.py:232 step:405K smpl:6M ep:31K epch:102.05 loss:0.005 grdn:0.145 lr:5.8e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 02:20:09 ts/train.py:232 step:405K smpl:6M ep:31K epch:102.10 loss:0.005 grdn:0.151 lr:5.8e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 02:20:59 ts/train.py:232 step:405K smpl:6M ep:31K epch:102.15 loss:0.005 grdn:0.151 lr:5.8e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 02:21:49 ts/train.py:232 step:405K smpl:6M ep:31K epch:102.20 loss:0.006 grdn:0.177 lr:5.8e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 02:22:39 ts/train.py:232 step:405K smpl:6M ep:31K epch:102.25 loss:0.005 grdn:0.164 lr:5.8e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 02:23:28 ts/train.py:232 step:406K smpl:6M ep:31K epch:102.30 loss:0.005 grdn:0.156 lr:5.8e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 02:24:29 ts/train.py:232 step:406K smpl:6M ep:31K epch:102.35 loss:0.006 grdn:0.160 lr:5.8e-05 updt_s:0.220 data_s:0.081 +INFO 2025-05-20 02:25:35 ts/train.py:232 step:406K smpl:6M ep:31K epch:102.40 loss:0.006 grdn:0.163 lr:5.8e-05 updt_s:0.220 data_s:0.108 +INFO 2025-05-20 02:26:40 ts/train.py:232 step:406K smpl:6M ep:31K epch:102.45 loss:0.005 grdn:0.154 lr:5.8e-05 updt_s:0.220 data_s:0.106 +INFO 2025-05-20 02:27:46 ts/train.py:232 step:406K smpl:7M ep:31K epch:102.50 loss:0.006 grdn:0.167 lr:5.8e-05 updt_s:0.220 data_s:0.109 +INFO 2025-05-20 02:28:51 ts/train.py:232 step:407K smpl:7M ep:31K epch:102.56 loss:0.006 grdn:0.163 lr:5.8e-05 updt_s:0.220 data_s:0.107 +INFO 2025-05-20 02:29:56 ts/train.py:232 step:407K smpl:7M ep:31K epch:102.61 loss:0.005 grdn:0.154 lr:5.8e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-20 02:30:54 ts/train.py:232 step:407K smpl:7M ep:31K epch:102.66 loss:0.006 grdn:0.157 lr:5.8e-05 updt_s:0.220 data_s:0.070 +INFO 2025-05-20 02:31:47 ts/train.py:232 step:407K smpl:7M ep:31K epch:102.71 loss:0.005 grdn:0.157 lr:5.8e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 02:32:39 ts/train.py:232 step:407K smpl:7M ep:31K epch:102.76 loss:0.004 grdn:0.137 lr:5.7e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-20 02:33:32 ts/train.py:232 step:408K smpl:7M ep:31K epch:102.81 loss:0.005 grdn:0.149 lr:5.7e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 02:34:24 ts/train.py:232 step:408K smpl:7M ep:31K epch:102.86 loss:0.005 grdn:0.159 lr:5.7e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 02:35:16 ts/train.py:232 step:408K smpl:7M ep:31K epch:102.91 loss:0.005 grdn:0.161 lr:5.7e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 02:36:07 ts/train.py:232 step:408K smpl:7M ep:31K epch:102.96 loss:0.005 grdn:0.157 lr:5.7e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 02:37:02 ts/train.py:232 step:408K smpl:7M ep:31K epch:103.01 loss:0.005 grdn:0.155 lr:5.7e-05 updt_s:0.221 data_s:0.053 +INFO 2025-05-20 02:37:53 ts/train.py:232 step:409K smpl:7M ep:31K epch:103.06 loss:0.005 grdn:0.151 lr:5.7e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 02:38:48 ts/train.py:232 step:409K smpl:7M ep:31K epch:103.11 loss:0.006 grdn:0.167 lr:5.7e-05 updt_s:0.221 data_s:0.052 +INFO 2025-05-20 02:39:39 ts/train.py:232 step:409K smpl:7M ep:31K epch:103.16 loss:0.005 grdn:0.148 lr:5.7e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 02:40:29 ts/train.py:232 step:409K smpl:7M ep:31K epch:103.21 loss:0.005 grdn:0.151 lr:5.7e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 02:41:24 ts/train.py:232 step:409K smpl:7M ep:31K epch:103.26 loss:0.005 grdn:0.157 lr:5.7e-05 updt_s:0.221 data_s:0.051 +INFO 2025-05-20 02:42:23 ts/train.py:232 step:410K smpl:7M ep:31K epch:103.31 loss:0.005 grdn:0.153 lr:5.7e-05 updt_s:0.220 data_s:0.078 +INFO 2025-05-20 02:43:28 ts/train.py:232 step:410K smpl:7M ep:31K epch:103.36 loss:0.005 grdn:0.161 lr:5.7e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 02:44:32 ts/train.py:232 step:410K smpl:7M ep:31K epch:103.41 loss:0.005 grdn:0.150 lr:5.7e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 02:45:36 ts/train.py:232 step:410K smpl:7M ep:31K epch:103.46 loss:0.005 grdn:0.157 lr:5.7e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-20 02:46:41 ts/train.py:232 step:410K smpl:7M ep:31K epch:103.51 loss:0.006 grdn:0.166 lr:5.7e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-20 02:47:45 ts/train.py:232 step:411K smpl:7M ep:31K epch:103.56 loss:0.005 grdn:0.147 lr:5.7e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-20 02:48:47 ts/train.py:232 step:411K smpl:7M ep:31K epch:103.61 loss:0.006 grdn:0.170 lr:5.7e-05 updt_s:0.220 data_s:0.089 +INFO 2025-05-20 02:49:39 ts/train.py:232 step:411K smpl:7M ep:31K epch:103.67 loss:0.006 grdn:0.156 lr:5.7e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 02:50:28 ts/train.py:232 step:411K smpl:7M ep:31K epch:103.72 loss:0.005 grdn:0.155 lr:5.7e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 02:51:17 ts/train.py:232 step:411K smpl:7M ep:31K epch:103.77 loss:0.005 grdn:0.156 lr:5.7e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 02:52:08 ts/train.py:232 step:412K smpl:7M ep:31K epch:103.82 loss:0.005 grdn:0.156 lr:5.7e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 02:53:00 ts/train.py:232 step:412K smpl:7M ep:31K epch:103.87 loss:0.006 grdn:0.160 lr:5.7e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 02:53:52 ts/train.py:232 step:412K smpl:7M ep:31K epch:103.92 loss:0.005 grdn:0.153 lr:5.7e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 02:54:43 ts/train.py:232 step:412K smpl:7M ep:31K epch:103.97 loss:0.006 grdn:0.163 lr:5.7e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 02:55:33 ts/train.py:232 step:412K smpl:7M ep:31K epch:104.02 loss:0.005 grdn:0.159 lr:5.7e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 02:56:25 ts/train.py:232 step:413K smpl:7M ep:31K epch:104.07 loss:0.006 grdn:0.161 lr:5.7e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 02:57:14 ts/train.py:232 step:413K smpl:7M ep:31K epch:104.12 loss:0.005 grdn:0.159 lr:5.7e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 02:58:07 ts/train.py:232 step:413K smpl:7M ep:31K epch:104.17 loss:0.005 grdn:0.156 lr:5.7e-05 updt_s:0.221 data_s:0.044 +INFO 2025-05-20 02:58:59 ts/train.py:232 step:413K smpl:7M ep:31K epch:104.22 loss:0.005 grdn:0.153 lr:5.6e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 02:59:55 ts/train.py:232 step:413K smpl:7M ep:31K epch:104.27 loss:0.005 grdn:0.144 lr:5.6e-05 updt_s:0.221 data_s:0.059 +INFO 2025-05-20 03:01:00 ts/train.py:232 step:414K smpl:7M ep:31K epch:104.32 loss:0.005 grdn:0.145 lr:5.6e-05 updt_s:0.220 data_s:0.102 +INFO 2025-05-20 03:02:05 ts/train.py:232 step:414K smpl:7M ep:31K epch:104.37 loss:0.005 grdn:0.165 lr:5.6e-05 updt_s:0.220 data_s:0.105 +INFO 2025-05-20 03:03:10 ts/train.py:232 step:414K smpl:7M ep:31K epch:104.42 loss:0.005 grdn:0.150 lr:5.6e-05 updt_s:0.220 data_s:0.103 +INFO 2025-05-20 03:04:13 ts/train.py:232 step:414K smpl:7M ep:31K epch:104.47 loss:0.005 grdn:0.155 lr:5.6e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 03:05:17 ts/train.py:232 step:414K smpl:7M ep:31K epch:104.52 loss:0.005 grdn:0.158 lr:5.6e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 03:06:21 ts/train.py:232 step:415K smpl:7M ep:31K epch:104.57 loss:0.005 grdn:0.152 lr:5.6e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 03:07:13 ts/train.py:232 step:415K smpl:7M ep:31K epch:104.62 loss:0.005 grdn:0.158 lr:5.6e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-20 03:08:08 ts/train.py:232 step:415K smpl:7M ep:31K epch:104.67 loss:0.005 grdn:0.145 lr:5.6e-05 updt_s:0.221 data_s:0.053 +INFO 2025-05-20 03:09:01 ts/train.py:232 step:415K smpl:7M ep:31K epch:104.72 loss:0.005 grdn:0.145 lr:5.6e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 03:09:54 ts/train.py:232 step:415K smpl:7M ep:31K epch:104.77 loss:0.005 grdn:0.162 lr:5.6e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 03:10:44 ts/train.py:232 step:416K smpl:7M ep:31K epch:104.83 loss:0.006 grdn:0.163 lr:5.6e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 03:11:38 ts/train.py:232 step:416K smpl:7M ep:31K epch:104.88 loss:0.005 grdn:0.142 lr:5.6e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 03:12:29 ts/train.py:232 step:416K smpl:7M ep:31K epch:104.93 loss:0.005 grdn:0.150 lr:5.6e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 03:13:21 ts/train.py:232 step:416K smpl:7M ep:31K epch:104.98 loss:0.005 grdn:0.155 lr:5.6e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 03:14:13 ts/train.py:232 step:416K smpl:7M ep:32K epch:105.03 loss:0.006 grdn:0.168 lr:5.6e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 03:15:08 ts/train.py:232 step:417K smpl:7M ep:32K epch:105.08 loss:0.005 grdn:0.156 lr:5.6e-05 updt_s:0.221 data_s:0.051 +INFO 2025-05-20 03:16:01 ts/train.py:232 step:417K smpl:7M ep:32K epch:105.13 loss:0.005 grdn:0.156 lr:5.6e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 03:16:53 ts/train.py:232 step:417K smpl:7M ep:32K epch:105.18 loss:0.005 grdn:0.151 lr:5.6e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 03:17:46 ts/train.py:232 step:417K smpl:7M ep:32K epch:105.23 loss:0.005 grdn:0.167 lr:5.6e-05 updt_s:0.220 data_s:0.045 +INFO 2025-05-20 03:18:51 ts/train.py:232 step:417K smpl:7M ep:32K epch:105.28 loss:0.005 grdn:0.149 lr:5.6e-05 updt_s:0.220 data_s:0.103 +INFO 2025-05-20 03:19:55 ts/train.py:232 step:418K smpl:7M ep:32K epch:105.33 loss:0.005 grdn:0.150 lr:5.6e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 03:21:00 ts/train.py:232 step:418K smpl:7M ep:32K epch:105.38 loss:0.005 grdn:0.150 lr:5.6e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 03:22:03 ts/train.py:232 step:418K smpl:7M ep:32K epch:105.43 loss:0.005 grdn:0.159 lr:5.6e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 03:23:07 ts/train.py:232 step:418K smpl:7M ep:32K epch:105.48 loss:0.005 grdn:0.153 lr:5.6e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 03:24:12 ts/train.py:232 step:418K smpl:7M ep:32K epch:105.53 loss:0.005 grdn:0.156 lr:5.6e-05 updt_s:0.220 data_s:0.103 +INFO 2025-05-20 03:25:03 ts/train.py:232 step:419K smpl:7M ep:32K epch:105.58 loss:0.005 grdn:0.166 lr:5.6e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 03:25:52 ts/train.py:232 step:419K smpl:7M ep:32K epch:105.63 loss:0.005 grdn:0.158 lr:5.5e-05 updt_s:0.221 data_s:0.021 +INFO 2025-05-20 03:26:41 ts/train.py:232 step:419K smpl:7M ep:32K epch:105.68 loss:0.005 grdn:0.156 lr:5.5e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-20 03:27:30 ts/train.py:232 step:419K smpl:7M ep:32K epch:105.73 loss:0.005 grdn:0.164 lr:5.5e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-20 03:28:19 ts/train.py:232 step:419K smpl:7M ep:32K epch:105.78 loss:0.004 grdn:0.150 lr:5.5e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 03:29:08 ts/train.py:232 step:420K smpl:7M ep:32K epch:105.83 loss:0.005 grdn:0.153 lr:5.5e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 03:29:57 ts/train.py:232 step:420K smpl:7M ep:32K epch:105.88 loss:0.005 grdn:0.155 lr:5.5e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-20 03:30:46 ts/train.py:232 step:420K smpl:7M ep:32K epch:105.94 loss:0.005 grdn:0.163 lr:5.5e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-20 03:30:46 ts/train.py:241 Checkpoint policy after step 420000 +INFO 2025-05-20 03:31:40 ts/train.py:232 step:420K smpl:7M ep:32K epch:105.99 loss:0.005 grdn:0.156 lr:5.5e-05 updt_s:0.222 data_s:0.023 +INFO 2025-05-20 03:32:30 ts/train.py:232 step:420K smpl:7M ep:32K epch:106.04 loss:0.005 grdn:0.155 lr:5.5e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-20 03:33:20 ts/train.py:232 step:421K smpl:7M ep:32K epch:106.09 loss:0.005 grdn:0.149 lr:5.5e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 03:34:10 ts/train.py:232 step:421K smpl:7M ep:32K epch:106.14 loss:0.005 grdn:0.161 lr:5.5e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-20 03:35:01 ts/train.py:232 step:421K smpl:7M ep:32K epch:106.19 loss:0.005 grdn:0.158 lr:5.5e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 03:36:06 ts/train.py:232 step:421K smpl:7M ep:32K epch:106.24 loss:0.005 grdn:0.163 lr:5.5e-05 updt_s:0.220 data_s:0.107 +INFO 2025-05-20 03:37:12 ts/train.py:232 step:421K smpl:7M ep:32K epch:106.29 loss:0.005 grdn:0.152 lr:5.5e-05 updt_s:0.220 data_s:0.108 +INFO 2025-05-20 03:38:16 ts/train.py:232 step:422K smpl:7M ep:32K epch:106.34 loss:0.005 grdn:0.153 lr:5.5e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 03:39:21 ts/train.py:232 step:422K smpl:7M ep:32K epch:106.39 loss:0.005 grdn:0.151 lr:5.5e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-20 03:40:25 ts/train.py:232 step:422K smpl:7M ep:32K epch:106.44 loss:0.005 grdn:0.156 lr:5.5e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-20 03:41:30 ts/train.py:232 step:422K smpl:7M ep:32K epch:106.49 loss:0.005 grdn:0.153 lr:5.5e-05 updt_s:0.221 data_s:0.104 +INFO 2025-05-20 03:42:24 ts/train.py:232 step:422K smpl:7M ep:32K epch:106.54 loss:0.005 grdn:0.155 lr:5.5e-05 updt_s:0.222 data_s:0.049 +INFO 2025-05-20 03:43:14 ts/train.py:232 step:423K smpl:7M ep:32K epch:106.59 loss:0.005 grdn:0.158 lr:5.5e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 03:44:04 ts/train.py:232 step:423K smpl:7M ep:32K epch:106.64 loss:0.005 grdn:0.156 lr:5.5e-05 updt_s:0.223 data_s:0.029 +INFO 2025-05-20 03:44:55 ts/train.py:232 step:423K smpl:7M ep:32K epch:106.69 loss:0.005 grdn:0.159 lr:5.5e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-20 03:45:45 ts/train.py:232 step:423K smpl:7M ep:32K epch:106.74 loss:0.005 grdn:0.141 lr:5.5e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 03:46:34 ts/train.py:232 step:423K smpl:7M ep:32K epch:106.79 loss:0.006 grdn:0.165 lr:5.5e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 03:47:25 ts/train.py:232 step:424K smpl:7M ep:32K epch:106.84 loss:0.005 grdn:0.159 lr:5.5e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 03:48:15 ts/train.py:232 step:424K smpl:7M ep:32K epch:106.89 loss:0.005 grdn:0.152 lr:5.5e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 03:49:06 ts/train.py:232 step:424K smpl:7M ep:32K epch:106.94 loss:0.005 grdn:0.163 lr:5.5e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 03:49:56 ts/train.py:232 step:424K smpl:7M ep:32K epch:106.99 loss:0.005 grdn:0.155 lr:5.5e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-20 03:50:46 ts/train.py:232 step:424K smpl:7M ep:32K epch:107.05 loss:0.005 grdn:0.153 lr:5.5e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-20 03:51:37 ts/train.py:232 step:425K smpl:7M ep:32K epch:107.10 loss:0.005 grdn:0.161 lr:5.4e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 03:52:27 ts/train.py:232 step:425K smpl:7M ep:32K epch:107.15 loss:0.005 grdn:0.148 lr:5.4e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-20 03:53:31 ts/train.py:232 step:425K smpl:7M ep:32K epch:107.20 loss:0.005 grdn:0.143 lr:5.4e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 03:54:37 ts/train.py:232 step:425K smpl:7M ep:32K epch:107.25 loss:0.005 grdn:0.162 lr:5.4e-05 updt_s:0.221 data_s:0.110 +INFO 2025-05-20 03:55:42 ts/train.py:232 step:425K smpl:7M ep:32K epch:107.30 loss:0.005 grdn:0.152 lr:5.4e-05 updt_s:0.220 data_s:0.102 +INFO 2025-05-20 03:56:48 ts/train.py:232 step:426K smpl:7M ep:32K epch:107.35 loss:0.005 grdn:0.162 lr:5.4e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-20 03:57:52 ts/train.py:232 step:426K smpl:7M ep:32K epch:107.40 loss:0.005 grdn:0.152 lr:5.4e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 03:58:57 ts/train.py:232 step:426K smpl:7M ep:32K epch:107.45 loss:0.005 grdn:0.159 lr:5.4e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-20 03:59:53 ts/train.py:232 step:426K smpl:7M ep:32K epch:107.50 loss:0.005 grdn:0.158 lr:5.4e-05 updt_s:0.221 data_s:0.059 +INFO 2025-05-20 04:00:43 ts/train.py:232 step:426K smpl:7M ep:32K epch:107.55 loss:0.005 grdn:0.156 lr:5.4e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 04:01:33 ts/train.py:232 step:427K smpl:7M ep:32K epch:107.60 loss:0.005 grdn:0.164 lr:5.4e-05 updt_s:0.222 data_s:0.029 +INFO 2025-05-20 04:02:22 ts/train.py:232 step:427K smpl:7M ep:32K epch:107.65 loss:0.005 grdn:0.155 lr:5.4e-05 updt_s:0.222 data_s:0.021 +INFO 2025-05-20 04:03:12 ts/train.py:232 step:427K smpl:7M ep:32K epch:107.70 loss:0.004 grdn:0.143 lr:5.4e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-20 04:04:02 ts/train.py:232 step:427K smpl:7M ep:32K epch:107.75 loss:0.005 grdn:0.159 lr:5.4e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 04:04:52 ts/train.py:232 step:427K smpl:7M ep:32K epch:107.80 loss:0.006 grdn:0.164 lr:5.4e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 04:05:43 ts/train.py:232 step:428K smpl:7M ep:32K epch:107.85 loss:0.005 grdn:0.152 lr:5.4e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 04:06:33 ts/train.py:232 step:428K smpl:7M ep:32K epch:107.90 loss:0.005 grdn:0.160 lr:5.4e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-20 04:07:23 ts/train.py:232 step:428K smpl:7M ep:32K epch:107.95 loss:0.005 grdn:0.157 lr:5.4e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 04:08:14 ts/train.py:232 step:428K smpl:7M ep:32K epch:108.00 loss:0.005 grdn:0.158 lr:5.4e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 04:09:04 ts/train.py:232 step:428K smpl:7M ep:32K epch:108.05 loss:0.005 grdn:0.152 lr:5.4e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 04:09:54 ts/train.py:232 step:429K smpl:7M ep:32K epch:108.10 loss:0.005 grdn:0.154 lr:5.4e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 04:10:55 ts/train.py:232 step:429K smpl:7M ep:32K epch:108.15 loss:0.005 grdn:0.153 lr:5.4e-05 updt_s:0.221 data_s:0.084 +INFO 2025-05-20 04:12:00 ts/train.py:232 step:429K smpl:7M ep:32K epch:108.21 loss:0.005 grdn:0.149 lr:5.4e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-20 04:13:05 ts/train.py:232 step:429K smpl:7M ep:32K epch:108.26 loss:0.005 grdn:0.144 lr:5.4e-05 updt_s:0.220 data_s:0.105 +INFO 2025-05-20 04:14:09 ts/train.py:232 step:429K smpl:7M ep:32K epch:108.31 loss:0.005 grdn:0.152 lr:5.4e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-20 04:15:14 ts/train.py:232 step:430K smpl:7M ep:33K epch:108.36 loss:0.005 grdn:0.154 lr:5.4e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-20 04:16:17 ts/train.py:232 step:430K smpl:7M ep:33K epch:108.41 loss:0.005 grdn:0.158 lr:5.4e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 04:17:15 ts/train.py:232 step:430K smpl:7M ep:33K epch:108.46 loss:0.005 grdn:0.153 lr:5.4e-05 updt_s:0.221 data_s:0.070 +INFO 2025-05-20 04:18:10 ts/train.py:232 step:430K smpl:7M ep:33K epch:108.51 loss:0.005 grdn:0.156 lr:5.4e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 04:19:03 ts/train.py:232 step:430K smpl:7M ep:33K epch:108.56 loss:0.005 grdn:0.158 lr:5.3e-05 updt_s:0.221 data_s:0.045 +INFO 2025-05-20 04:19:56 ts/train.py:232 step:431K smpl:7M ep:33K epch:108.61 loss:0.005 grdn:0.158 lr:5.3e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 04:20:48 ts/train.py:232 step:431K smpl:7M ep:33K epch:108.66 loss:0.005 grdn:0.151 lr:5.3e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 04:21:38 ts/train.py:232 step:431K smpl:7M ep:33K epch:108.71 loss:0.005 grdn:0.156 lr:5.3e-05 updt_s:0.222 data_s:0.028 +INFO 2025-05-20 04:22:31 ts/train.py:232 step:431K smpl:7M ep:33K epch:108.76 loss:0.005 grdn:0.152 lr:5.3e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 04:23:24 ts/train.py:232 step:431K smpl:7M ep:33K epch:108.81 loss:0.005 grdn:0.164 lr:5.3e-05 updt_s:0.221 data_s:0.044 +INFO 2025-05-20 04:24:15 ts/train.py:232 step:432K smpl:7M ep:33K epch:108.86 loss:0.005 grdn:0.151 lr:5.3e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 04:25:06 ts/train.py:232 step:432K smpl:7M ep:33K epch:108.91 loss:0.005 grdn:0.157 lr:5.3e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 04:25:58 ts/train.py:232 step:432K smpl:7M ep:33K epch:108.96 loss:0.004 grdn:0.146 lr:5.3e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 04:26:51 ts/train.py:232 step:432K smpl:7M ep:33K epch:109.01 loss:0.005 grdn:0.159 lr:5.3e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 04:27:44 ts/train.py:232 step:432K smpl:7M ep:33K epch:109.06 loss:0.005 grdn:0.146 lr:5.3e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 04:28:42 ts/train.py:232 step:433K smpl:7M ep:33K epch:109.11 loss:0.005 grdn:0.164 lr:5.3e-05 updt_s:0.220 data_s:0.068 +INFO 2025-05-20 04:29:46 ts/train.py:232 step:433K smpl:7M ep:33K epch:109.16 loss:0.005 grdn:0.152 lr:5.3e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 04:30:50 ts/train.py:232 step:433K smpl:7M ep:33K epch:109.21 loss:0.006 grdn:0.169 lr:5.3e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 04:31:54 ts/train.py:232 step:433K smpl:7M ep:33K epch:109.26 loss:0.005 grdn:0.150 lr:5.3e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 04:32:57 ts/train.py:232 step:433K smpl:7M ep:33K epch:109.32 loss:0.005 grdn:0.154 lr:5.3e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-20 04:34:02 ts/train.py:232 step:434K smpl:7M ep:33K epch:109.37 loss:0.005 grdn:0.152 lr:5.3e-05 updt_s:0.220 data_s:0.103 +INFO 2025-05-20 04:35:03 ts/train.py:232 step:434K smpl:7M ep:33K epch:109.42 loss:0.005 grdn:0.158 lr:5.3e-05 updt_s:0.220 data_s:0.082 +INFO 2025-05-20 04:35:53 ts/train.py:232 step:434K smpl:7M ep:33K epch:109.47 loss:0.005 grdn:0.146 lr:5.3e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 04:36:42 ts/train.py:232 step:434K smpl:7M ep:33K epch:109.52 loss:0.005 grdn:0.148 lr:5.3e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-20 04:37:31 ts/train.py:232 step:434K smpl:7M ep:33K epch:109.57 loss:0.005 grdn:0.161 lr:5.3e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 04:38:20 ts/train.py:232 step:435K smpl:7M ep:33K epch:109.62 loss:0.005 grdn:0.154 lr:5.3e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-20 04:39:10 ts/train.py:232 step:435K smpl:7M ep:33K epch:109.67 loss:0.005 grdn:0.151 lr:5.3e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 04:39:59 ts/train.py:232 step:435K smpl:7M ep:33K epch:109.72 loss:0.005 grdn:0.164 lr:5.3e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 04:40:49 ts/train.py:232 step:435K smpl:7M ep:33K epch:109.77 loss:0.005 grdn:0.163 lr:5.3e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 04:41:38 ts/train.py:232 step:435K smpl:7M ep:33K epch:109.82 loss:0.005 grdn:0.152 lr:5.3e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-20 04:42:27 ts/train.py:232 step:436K smpl:7M ep:33K epch:109.87 loss:0.005 grdn:0.154 lr:5.3e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-20 04:43:16 ts/train.py:232 step:436K smpl:7M ep:33K epch:109.92 loss:0.005 grdn:0.157 lr:5.3e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 04:44:06 ts/train.py:232 step:436K smpl:7M ep:33K epch:109.97 loss:0.005 grdn:0.147 lr:5.3e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 04:44:55 ts/train.py:232 step:436K smpl:7M ep:33K epch:110.02 loss:0.005 grdn:0.150 lr:5.2e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 04:45:50 ts/train.py:232 step:436K smpl:7M ep:33K epch:110.07 loss:0.005 grdn:0.159 lr:5.2e-05 updt_s:0.220 data_s:0.050 +INFO 2025-05-20 04:46:55 ts/train.py:232 step:437K smpl:7M ep:33K epch:110.12 loss:0.005 grdn:0.144 lr:5.2e-05 updt_s:0.220 data_s:0.107 +INFO 2025-05-20 04:48:00 ts/train.py:232 step:437K smpl:7M ep:33K epch:110.17 loss:0.005 grdn:0.154 lr:5.2e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-20 04:49:05 ts/train.py:232 step:437K smpl:7M ep:33K epch:110.22 loss:0.005 grdn:0.150 lr:5.2e-05 updt_s:0.220 data_s:0.105 +INFO 2025-05-20 04:50:10 ts/train.py:232 step:437K smpl:7M ep:33K epch:110.27 loss:0.005 grdn:0.166 lr:5.2e-05 updt_s:0.220 data_s:0.102 +INFO 2025-05-20 04:51:14 ts/train.py:232 step:437K smpl:7M ep:33K epch:110.32 loss:0.005 grdn:0.158 lr:5.2e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-20 04:52:18 ts/train.py:232 step:438K smpl:7M ep:33K epch:110.37 loss:0.005 grdn:0.154 lr:5.2e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 04:53:18 ts/train.py:232 step:438K smpl:7M ep:33K epch:110.42 loss:0.006 grdn:0.169 lr:5.2e-05 updt_s:0.221 data_s:0.081 +INFO 2025-05-20 04:54:20 ts/train.py:232 step:438K smpl:7M ep:33K epch:110.48 loss:0.004 grdn:0.148 lr:5.2e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-20 04:55:22 ts/train.py:232 step:438K smpl:7M ep:33K epch:110.53 loss:0.005 grdn:0.155 lr:5.2e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-20 04:56:23 ts/train.py:232 step:438K smpl:7M ep:33K epch:110.58 loss:0.005 grdn:0.150 lr:5.2e-05 updt_s:0.221 data_s:0.085 +INFO 2025-05-20 04:57:25 ts/train.py:232 step:439K smpl:7M ep:33K epch:110.63 loss:0.005 grdn:0.157 lr:5.2e-05 updt_s:0.221 data_s:0.087 +INFO 2025-05-20 04:58:28 ts/train.py:232 step:439K smpl:7M ep:33K epch:110.68 loss:0.005 grdn:0.165 lr:5.2e-05 updt_s:0.221 data_s:0.091 +INFO 2025-05-20 04:59:30 ts/train.py:232 step:439K smpl:7M ep:33K epch:110.73 loss:0.005 grdn:0.152 lr:5.2e-05 updt_s:0.220 data_s:0.088 +INFO 2025-05-20 05:00:32 ts/train.py:232 step:439K smpl:7M ep:33K epch:110.78 loss:0.005 grdn:0.155 lr:5.2e-05 updt_s:0.220 data_s:0.089 +INFO 2025-05-20 05:01:33 ts/train.py:232 step:439K smpl:7M ep:33K epch:110.83 loss:0.005 grdn:0.162 lr:5.2e-05 updt_s:0.220 data_s:0.085 +INFO 2025-05-20 05:02:34 ts/train.py:232 step:440K smpl:7M ep:33K epch:110.88 loss:0.005 grdn:0.151 lr:5.2e-05 updt_s:0.220 data_s:0.086 +INFO 2025-05-20 05:03:37 ts/train.py:232 step:440K smpl:7M ep:33K epch:110.93 loss:0.005 grdn:0.152 lr:5.2e-05 updt_s:0.220 data_s:0.091 +INFO 2025-05-20 05:04:40 ts/train.py:232 step:440K smpl:7M ep:33K epch:110.98 loss:0.005 grdn:0.163 lr:5.2e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-20 05:04:40 ts/train.py:241 Checkpoint policy after step 440000 +INFO 2025-05-20 05:05:48 ts/train.py:232 step:440K smpl:7M ep:33K epch:111.03 loss:0.004 grdn:0.149 lr:5.2e-05 updt_s:0.220 data_s:0.095 +INFO 2025-05-20 05:06:53 ts/train.py:232 step:440K smpl:7M ep:33K epch:111.08 loss:0.005 grdn:0.158 lr:5.2e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-20 05:07:58 ts/train.py:232 step:441K smpl:7M ep:33K epch:111.13 loss:0.004 grdn:0.140 lr:5.2e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-20 05:09:02 ts/train.py:232 step:441K smpl:7M ep:33K epch:111.18 loss:0.004 grdn:0.144 lr:5.2e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 05:10:06 ts/train.py:232 step:441K smpl:7M ep:33K epch:111.23 loss:0.004 grdn:0.152 lr:5.2e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-20 05:11:10 ts/train.py:232 step:441K smpl:7M ep:33K epch:111.28 loss:0.005 grdn:0.158 lr:5.2e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-20 05:12:14 ts/train.py:232 step:441K smpl:7M ep:33K epch:111.33 loss:0.005 grdn:0.160 lr:5.2e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 05:13:07 ts/train.py:232 step:442K smpl:7M ep:33K epch:111.38 loss:0.005 grdn:0.153 lr:5.2e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-20 05:13:59 ts/train.py:232 step:442K smpl:7M ep:33K epch:111.43 loss:0.005 grdn:0.154 lr:5.1e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 05:14:52 ts/train.py:232 step:442K smpl:7M ep:33K epch:111.48 loss:0.005 grdn:0.161 lr:5.1e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-20 05:15:43 ts/train.py:232 step:442K smpl:7M ep:33K epch:111.53 loss:0.005 grdn:0.156 lr:5.1e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 05:16:35 ts/train.py:232 step:442K smpl:7M ep:33K epch:111.59 loss:0.005 grdn:0.151 lr:5.1e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 05:17:28 ts/train.py:232 step:443K smpl:7M ep:33K epch:111.64 loss:0.005 grdn:0.166 lr:5.1e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 05:18:20 ts/train.py:232 step:443K smpl:7M ep:34K epch:111.69 loss:0.005 grdn:0.151 lr:5.1e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 05:19:11 ts/train.py:232 step:443K smpl:7M ep:34K epch:111.74 loss:0.005 grdn:0.160 lr:5.1e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 05:20:05 ts/train.py:232 step:443K smpl:7M ep:34K epch:111.79 loss:0.005 grdn:0.154 lr:5.1e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 05:20:58 ts/train.py:232 step:443K smpl:7M ep:34K epch:111.84 loss:0.006 grdn:0.181 lr:5.1e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 05:21:53 ts/train.py:232 step:444K smpl:7M ep:34K epch:111.89 loss:0.005 grdn:0.160 lr:5.1e-05 updt_s:0.221 data_s:0.055 +INFO 2025-05-20 05:22:48 ts/train.py:232 step:444K smpl:7M ep:34K epch:111.94 loss:0.005 grdn:0.165 lr:5.1e-05 updt_s:0.221 data_s:0.052 +INFO 2025-05-20 05:23:42 ts/train.py:232 step:444K smpl:7M ep:34K epch:111.99 loss:0.005 grdn:0.156 lr:5.1e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 05:24:48 ts/train.py:232 step:444K smpl:7M ep:34K epch:112.04 loss:0.005 grdn:0.148 lr:5.1e-05 updt_s:0.220 data_s:0.109 +INFO 2025-05-20 05:25:52 ts/train.py:232 step:444K smpl:7M ep:34K epch:112.09 loss:0.005 grdn:0.152 lr:5.1e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 05:26:56 ts/train.py:232 step:445K smpl:7M ep:34K epch:112.14 loss:0.005 grdn:0.149 lr:5.1e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 05:27:59 ts/train.py:232 step:445K smpl:7M ep:34K epch:112.19 loss:0.005 grdn:0.173 lr:5.1e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 05:29:03 ts/train.py:232 step:445K smpl:7M ep:34K epch:112.24 loss:0.005 grdn:0.166 lr:5.1e-05 updt_s:0.220 data_s:0.095 +INFO 2025-05-20 05:30:06 ts/train.py:232 step:445K smpl:7M ep:34K epch:112.29 loss:0.004 grdn:0.155 lr:5.1e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 05:31:02 ts/train.py:232 step:445K smpl:7M ep:34K epch:112.34 loss:0.005 grdn:0.166 lr:5.1e-05 updt_s:0.220 data_s:0.057 +INFO 2025-05-20 05:31:54 ts/train.py:232 step:446K smpl:7M ep:34K epch:112.39 loss:0.004 grdn:0.153 lr:5.1e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 05:32:44 ts/train.py:232 step:446K smpl:7M ep:34K epch:112.44 loss:0.005 grdn:0.144 lr:5.1e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 05:33:34 ts/train.py:232 step:446K smpl:7M ep:34K epch:112.49 loss:0.005 grdn:0.156 lr:5.1e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 05:34:24 ts/train.py:232 step:446K smpl:7M ep:34K epch:112.54 loss:0.005 grdn:0.150 lr:5.1e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 05:35:14 ts/train.py:232 step:446K smpl:7M ep:34K epch:112.59 loss:0.005 grdn:0.159 lr:5.1e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 05:36:04 ts/train.py:232 step:447K smpl:7M ep:34K epch:112.64 loss:0.005 grdn:0.156 lr:5.1e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 05:36:57 ts/train.py:232 step:447K smpl:7M ep:34K epch:112.69 loss:0.004 grdn:0.135 lr:5.1e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 05:37:49 ts/train.py:232 step:447K smpl:7M ep:34K epch:112.75 loss:0.004 grdn:0.145 lr:5.1e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 05:38:41 ts/train.py:232 step:447K smpl:7M ep:34K epch:112.80 loss:0.005 grdn:0.156 lr:5.1e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 05:39:32 ts/train.py:232 step:447K smpl:7M ep:34K epch:112.85 loss:0.004 grdn:0.144 lr:5.1e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 05:40:24 ts/train.py:232 step:448K smpl:7M ep:34K epch:112.90 loss:0.005 grdn:0.151 lr:5.0e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 05:41:17 ts/train.py:232 step:448K smpl:7M ep:34K epch:112.95 loss:0.006 grdn:0.178 lr:5.0e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 05:42:19 ts/train.py:232 step:448K smpl:7M ep:34K epch:113.00 loss:0.005 grdn:0.160 lr:5.0e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-20 05:43:22 ts/train.py:232 step:448K smpl:7M ep:34K epch:113.05 loss:0.005 grdn:0.167 lr:5.0e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 05:44:26 ts/train.py:232 step:448K smpl:7M ep:34K epch:113.10 loss:0.005 grdn:0.159 lr:5.0e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 05:45:30 ts/train.py:232 step:449K smpl:7M ep:34K epch:113.15 loss:0.004 grdn:0.147 lr:5.0e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 05:46:34 ts/train.py:232 step:449K smpl:7M ep:34K epch:113.20 loss:0.005 grdn:0.165 lr:5.0e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 05:47:39 ts/train.py:232 step:449K smpl:7M ep:34K epch:113.25 loss:0.005 grdn:0.157 lr:5.0e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-20 05:48:34 ts/train.py:232 step:449K smpl:7M ep:34K epch:113.30 loss:0.004 grdn:0.147 lr:5.0e-05 updt_s:0.220 data_s:0.058 +INFO 2025-05-20 05:49:25 ts/train.py:232 step:449K smpl:7M ep:34K epch:113.35 loss:0.005 grdn:0.161 lr:5.0e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 05:50:17 ts/train.py:232 step:450K smpl:7M ep:34K epch:113.40 loss:0.005 grdn:0.162 lr:5.0e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 05:51:09 ts/train.py:232 step:450K smpl:7M ep:34K epch:113.45 loss:0.005 grdn:0.165 lr:5.0e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-20 05:51:59 ts/train.py:232 step:450K smpl:7M ep:34K epch:113.50 loss:0.005 grdn:0.166 lr:5.0e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 05:52:54 ts/train.py:232 step:450K smpl:7M ep:34K epch:113.55 loss:0.005 grdn:0.149 lr:5.0e-05 updt_s:0.221 data_s:0.052 +INFO 2025-05-20 05:53:46 ts/train.py:232 step:450K smpl:7M ep:34K epch:113.60 loss:0.005 grdn:0.169 lr:5.0e-05 updt_s:0.220 data_s:0.041 +INFO 2025-05-20 05:54:41 ts/train.py:232 step:451K smpl:7M ep:34K epch:113.65 loss:0.004 grdn:0.141 lr:5.0e-05 updt_s:0.220 data_s:0.053 +INFO 2025-05-20 05:55:32 ts/train.py:232 step:451K smpl:7M ep:34K epch:113.70 loss:0.005 grdn:0.155 lr:5.0e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 05:56:23 ts/train.py:232 step:451K smpl:7M ep:34K epch:113.75 loss:0.005 grdn:0.162 lr:5.0e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 05:57:17 ts/train.py:232 step:451K smpl:7M ep:34K epch:113.80 loss:0.005 grdn:0.149 lr:5.0e-05 updt_s:0.220 data_s:0.047 +INFO 2025-05-20 05:58:11 ts/train.py:232 step:451K smpl:7M ep:34K epch:113.86 loss:0.005 grdn:0.148 lr:5.0e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 05:59:05 ts/train.py:232 step:452K smpl:7M ep:34K epch:113.91 loss:0.004 grdn:0.149 lr:5.0e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 06:00:08 ts/train.py:232 step:452K smpl:7M ep:34K epch:113.96 loss:0.005 grdn:0.156 lr:5.0e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 06:01:12 ts/train.py:232 step:452K smpl:7M ep:34K epch:114.01 loss:0.005 grdn:0.155 lr:5.0e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 06:02:16 ts/train.py:232 step:452K smpl:7M ep:34K epch:114.06 loss:0.005 grdn:0.156 lr:5.0e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 06:03:19 ts/train.py:232 step:452K smpl:7M ep:34K epch:114.11 loss:0.005 grdn:0.144 lr:5.0e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-20 06:04:22 ts/train.py:232 step:453K smpl:7M ep:34K epch:114.16 loss:0.005 grdn:0.154 lr:5.0e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-20 06:05:24 ts/train.py:232 step:453K smpl:7M ep:34K epch:114.21 loss:0.004 grdn:0.145 lr:5.0e-05 updt_s:0.220 data_s:0.092 +INFO 2025-05-20 06:06:24 ts/train.py:232 step:453K smpl:7M ep:34K epch:114.26 loss:0.005 grdn:0.161 lr:5.0e-05 updt_s:0.221 data_s:0.078 +INFO 2025-05-20 06:07:15 ts/train.py:232 step:453K smpl:7M ep:34K epch:114.31 loss:0.005 grdn:0.152 lr:5.0e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 06:08:04 ts/train.py:232 step:453K smpl:7M ep:34K epch:114.36 loss:0.005 grdn:0.155 lr:4.9e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 06:08:56 ts/train.py:232 step:454K smpl:7M ep:34K epch:114.41 loss:0.005 grdn:0.165 lr:4.9e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 06:09:46 ts/train.py:232 step:454K smpl:7M ep:34K epch:114.46 loss:0.005 grdn:0.161 lr:4.9e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 06:10:37 ts/train.py:232 step:454K smpl:7M ep:34K epch:114.51 loss:0.005 grdn:0.153 lr:4.9e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 06:11:28 ts/train.py:232 step:454K smpl:7M ep:34K epch:114.56 loss:0.005 grdn:0.157 lr:4.9e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 06:12:18 ts/train.py:232 step:454K smpl:7M ep:34K epch:114.61 loss:0.004 grdn:0.140 lr:4.9e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 06:13:08 ts/train.py:232 step:455K smpl:7M ep:34K epch:114.66 loss:0.005 grdn:0.159 lr:4.9e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 06:13:59 ts/train.py:232 step:455K smpl:7M ep:34K epch:114.71 loss:0.005 grdn:0.159 lr:4.9e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 06:14:49 ts/train.py:232 step:455K smpl:7M ep:34K epch:114.76 loss:0.004 grdn:0.149 lr:4.9e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 06:15:39 ts/train.py:232 step:455K smpl:7M ep:34K epch:114.81 loss:0.004 grdn:0.153 lr:4.9e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 06:16:30 ts/train.py:232 step:455K smpl:7M ep:34K epch:114.86 loss:0.005 grdn:0.159 lr:4.9e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 06:17:27 ts/train.py:232 step:456K smpl:7M ep:34K epch:114.91 loss:0.004 grdn:0.145 lr:4.9e-05 updt_s:0.220 data_s:0.065 +INFO 2025-05-20 06:18:32 ts/train.py:232 step:456K smpl:7M ep:34K epch:114.96 loss:0.005 grdn:0.161 lr:4.9e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 06:19:35 ts/train.py:232 step:456K smpl:7M ep:35K epch:115.02 loss:0.005 grdn:0.157 lr:4.9e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 06:20:40 ts/train.py:232 step:456K smpl:7M ep:35K epch:115.07 loss:0.005 grdn:0.154 lr:4.9e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-20 06:21:44 ts/train.py:232 step:456K smpl:7M ep:35K epch:115.12 loss:0.004 grdn:0.147 lr:4.9e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 06:22:47 ts/train.py:232 step:457K smpl:7M ep:35K epch:115.17 loss:0.005 grdn:0.163 lr:4.9e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-20 06:23:48 ts/train.py:232 step:457K smpl:7M ep:35K epch:115.22 loss:0.005 grdn:0.161 lr:4.9e-05 updt_s:0.220 data_s:0.085 +INFO 2025-05-20 06:24:40 ts/train.py:232 step:457K smpl:7M ep:35K epch:115.27 loss:0.005 grdn:0.154 lr:4.9e-05 updt_s:0.220 data_s:0.041 +INFO 2025-05-20 06:25:34 ts/train.py:232 step:457K smpl:7M ep:35K epch:115.32 loss:0.004 grdn:0.155 lr:4.9e-05 updt_s:0.220 data_s:0.050 +INFO 2025-05-20 06:26:29 ts/train.py:232 step:457K smpl:7M ep:35K epch:115.37 loss:0.005 grdn:0.161 lr:4.9e-05 updt_s:0.221 data_s:0.050 +INFO 2025-05-20 06:27:21 ts/train.py:232 step:458K smpl:7M ep:35K epch:115.42 loss:0.005 grdn:0.160 lr:4.9e-05 updt_s:0.220 data_s:0.041 +INFO 2025-05-20 06:28:14 ts/train.py:232 step:458K smpl:7M ep:35K epch:115.47 loss:0.005 grdn:0.164 lr:4.9e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 06:29:07 ts/train.py:232 step:458K smpl:7M ep:35K epch:115.52 loss:0.005 grdn:0.155 lr:4.9e-05 updt_s:0.221 data_s:0.044 +INFO 2025-05-20 06:29:59 ts/train.py:232 step:458K smpl:7M ep:35K epch:115.57 loss:0.005 grdn:0.152 lr:4.9e-05 updt_s:0.220 data_s:0.041 +INFO 2025-05-20 06:30:53 ts/train.py:232 step:458K smpl:7M ep:35K epch:115.62 loss:0.004 grdn:0.151 lr:4.9e-05 updt_s:0.221 data_s:0.045 +INFO 2025-05-20 06:31:48 ts/train.py:232 step:459K smpl:7M ep:35K epch:115.67 loss:0.004 grdn:0.147 lr:4.9e-05 updt_s:0.221 data_s:0.055 +INFO 2025-05-20 06:32:44 ts/train.py:232 step:459K smpl:7M ep:35K epch:115.72 loss:0.005 grdn:0.157 lr:4.9e-05 updt_s:0.221 data_s:0.059 +INFO 2025-05-20 06:33:38 ts/train.py:232 step:459K smpl:7M ep:35K epch:115.77 loss:0.005 grdn:0.162 lr:4.8e-05 updt_s:0.220 data_s:0.049 +INFO 2025-05-20 06:34:33 ts/train.py:232 step:459K smpl:7M ep:35K epch:115.82 loss:0.004 grdn:0.149 lr:4.8e-05 updt_s:0.220 data_s:0.054 +INFO 2025-05-20 06:35:30 ts/train.py:232 step:459K smpl:7M ep:35K epch:115.87 loss:0.004 grdn:0.151 lr:4.8e-05 updt_s:0.220 data_s:0.064 +INFO 2025-05-20 06:36:35 ts/train.py:232 step:460K smpl:7M ep:35K epch:115.92 loss:0.004 grdn:0.153 lr:4.8e-05 updt_s:0.221 data_s:0.107 +INFO 2025-05-20 06:37:40 ts/train.py:232 step:460K smpl:7M ep:35K epch:115.97 loss:0.005 grdn:0.159 lr:4.8e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-20 06:38:46 ts/train.py:232 step:460K smpl:7M ep:35K epch:116.02 loss:0.005 grdn:0.154 lr:4.8e-05 updt_s:0.220 data_s:0.107 +INFO 2025-05-20 06:38:46 ts/train.py:241 Checkpoint policy after step 460000 +INFO 2025-05-20 06:39:54 ts/train.py:232 step:460K smpl:7M ep:35K epch:116.07 loss:0.005 grdn:0.149 lr:4.8e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 06:40:57 ts/train.py:232 step:460K smpl:7M ep:35K epch:116.13 loss:0.005 grdn:0.167 lr:4.8e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 06:42:00 ts/train.py:232 step:461K smpl:7M ep:35K epch:116.18 loss:0.004 grdn:0.150 lr:4.8e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-20 06:42:52 ts/train.py:232 step:461K smpl:7M ep:35K epch:116.23 loss:0.005 grdn:0.155 lr:4.8e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 06:43:43 ts/train.py:232 step:461K smpl:7M ep:35K epch:116.28 loss:0.004 grdn:0.139 lr:4.8e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 06:44:36 ts/train.py:232 step:461K smpl:7M ep:35K epch:116.33 loss:0.004 grdn:0.153 lr:4.8e-05 updt_s:0.221 data_s:0.044 +INFO 2025-05-20 06:45:27 ts/train.py:232 step:461K smpl:7M ep:35K epch:116.38 loss:0.005 grdn:0.159 lr:4.8e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 06:46:17 ts/train.py:232 step:462K smpl:7M ep:35K epch:116.43 loss:0.005 grdn:0.154 lr:4.8e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 06:47:07 ts/train.py:232 step:462K smpl:7M ep:35K epch:116.48 loss:0.004 grdn:0.150 lr:4.8e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 06:47:57 ts/train.py:232 step:462K smpl:7M ep:35K epch:116.53 loss:0.004 grdn:0.142 lr:4.8e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 06:48:46 ts/train.py:232 step:462K smpl:7M ep:35K epch:116.58 loss:0.005 grdn:0.163 lr:4.8e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-20 06:49:36 ts/train.py:232 step:462K smpl:7M ep:35K epch:116.63 loss:0.004 grdn:0.146 lr:4.8e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-20 06:50:26 ts/train.py:232 step:463K smpl:7M ep:35K epch:116.68 loss:0.005 grdn:0.158 lr:4.8e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 06:51:16 ts/train.py:232 step:463K smpl:7M ep:35K epch:116.73 loss:0.005 grdn:0.159 lr:4.8e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 06:52:05 ts/train.py:232 step:463K smpl:7M ep:35K epch:116.78 loss:0.005 grdn:0.162 lr:4.8e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 06:52:55 ts/train.py:232 step:463K smpl:7M ep:35K epch:116.83 loss:0.005 grdn:0.160 lr:4.8e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-20 06:54:01 ts/train.py:232 step:463K smpl:7M ep:35K epch:116.88 loss:0.004 grdn:0.150 lr:4.8e-05 updt_s:0.220 data_s:0.109 +INFO 2025-05-20 06:55:04 ts/train.py:232 step:464K smpl:7M ep:35K epch:116.93 loss:0.005 grdn:0.157 lr:4.8e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 06:56:08 ts/train.py:232 step:464K smpl:7M ep:35K epch:116.98 loss:0.004 grdn:0.154 lr:4.8e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-20 06:57:12 ts/train.py:232 step:464K smpl:7M ep:35K epch:117.03 loss:0.005 grdn:0.157 lr:4.8e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-20 06:58:16 ts/train.py:232 step:464K smpl:7M ep:35K epch:117.08 loss:0.004 grdn:0.147 lr:4.8e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-20 06:59:20 ts/train.py:232 step:464K smpl:7M ep:35K epch:117.13 loss:0.005 grdn:0.162 lr:4.8e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-20 07:00:14 ts/train.py:232 step:465K smpl:7M ep:35K epch:117.18 loss:0.004 grdn:0.153 lr:4.8e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 07:01:05 ts/train.py:232 step:465K smpl:7M ep:35K epch:117.23 loss:0.005 grdn:0.157 lr:4.7e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 07:01:57 ts/train.py:232 step:465K smpl:7M ep:35K epch:117.29 loss:0.005 grdn:0.150 lr:4.7e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 07:02:49 ts/train.py:232 step:465K smpl:7M ep:35K epch:117.34 loss:0.004 grdn:0.147 lr:4.7e-05 updt_s:0.222 data_s:0.038 +INFO 2025-05-20 07:03:39 ts/train.py:232 step:465K smpl:7M ep:35K epch:117.39 loss:0.005 grdn:0.164 lr:4.7e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 07:04:30 ts/train.py:232 step:466K smpl:7M ep:35K epch:117.44 loss:0.004 grdn:0.153 lr:4.7e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 07:05:23 ts/train.py:232 step:466K smpl:7M ep:35K epch:117.49 loss:0.005 grdn:0.153 lr:4.7e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-20 07:06:14 ts/train.py:232 step:466K smpl:7M ep:35K epch:117.54 loss:0.005 grdn:0.157 lr:4.7e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 07:07:05 ts/train.py:232 step:466K smpl:7M ep:35K epch:117.59 loss:0.005 grdn:0.162 lr:4.7e-05 updt_s:0.222 data_s:0.033 +INFO 2025-05-20 07:07:56 ts/train.py:232 step:466K smpl:7M ep:35K epch:117.64 loss:0.004 grdn:0.154 lr:4.7e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 07:08:46 ts/train.py:232 step:467K smpl:7M ep:35K epch:117.69 loss:0.004 grdn:0.152 lr:4.7e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 07:09:37 ts/train.py:232 step:467K smpl:7M ep:35K epch:117.74 loss:0.005 grdn:0.151 lr:4.7e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 07:10:28 ts/train.py:232 step:467K smpl:7M ep:35K epch:117.79 loss:0.005 grdn:0.157 lr:4.7e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 07:11:32 ts/train.py:232 step:467K smpl:7M ep:35K epch:117.84 loss:0.005 grdn:0.156 lr:4.7e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 07:12:36 ts/train.py:232 step:467K smpl:7M ep:35K epch:117.89 loss:0.005 grdn:0.160 lr:4.7e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 07:13:41 ts/train.py:232 step:468K smpl:7M ep:35K epch:117.94 loss:0.004 grdn:0.155 lr:4.7e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-20 07:14:45 ts/train.py:232 step:468K smpl:7M ep:35K epch:117.99 loss:0.004 grdn:0.149 lr:4.7e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-20 07:15:49 ts/train.py:232 step:468K smpl:7M ep:35K epch:118.04 loss:0.004 grdn:0.158 lr:4.7e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-20 07:16:54 ts/train.py:232 step:468K smpl:7M ep:35K epch:118.09 loss:0.005 grdn:0.163 lr:4.7e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-20 07:17:48 ts/train.py:232 step:468K smpl:7M ep:35K epch:118.14 loss:0.004 grdn:0.155 lr:4.7e-05 updt_s:0.222 data_s:0.050 +INFO 2025-05-20 07:18:39 ts/train.py:232 step:469K smpl:7M ep:35K epch:118.19 loss:0.004 grdn:0.155 lr:4.7e-05 updt_s:0.222 data_s:0.030 +INFO 2025-05-20 07:19:28 ts/train.py:232 step:469K smpl:8M ep:35K epch:118.24 loss:0.005 grdn:0.163 lr:4.7e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-20 07:20:17 ts/train.py:232 step:469K smpl:8M ep:35K epch:118.29 loss:0.004 grdn:0.153 lr:4.7e-05 updt_s:0.222 data_s:0.019 +INFO 2025-05-20 07:21:06 ts/train.py:232 step:469K smpl:8M ep:36K epch:118.34 loss:0.004 grdn:0.141 lr:4.7e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 07:21:56 ts/train.py:232 step:469K smpl:8M ep:36K epch:118.40 loss:0.004 grdn:0.147 lr:4.7e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 07:22:45 ts/train.py:232 step:470K smpl:8M ep:36K epch:118.45 loss:0.004 grdn:0.155 lr:4.7e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 07:23:35 ts/train.py:232 step:470K smpl:8M ep:36K epch:118.50 loss:0.005 grdn:0.154 lr:4.7e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 07:24:24 ts/train.py:232 step:470K smpl:8M ep:36K epch:118.55 loss:0.005 grdn:0.164 lr:4.7e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 07:25:14 ts/train.py:232 step:470K smpl:8M ep:36K epch:118.60 loss:0.004 grdn:0.153 lr:4.7e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 07:26:05 ts/train.py:232 step:470K smpl:8M ep:36K epch:118.65 loss:0.004 grdn:0.143 lr:4.7e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 07:26:55 ts/train.py:232 step:471K smpl:8M ep:36K epch:118.70 loss:0.004 grdn:0.153 lr:4.6e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 07:27:45 ts/train.py:232 step:471K smpl:8M ep:36K epch:118.75 loss:0.004 grdn:0.156 lr:4.6e-05 updt_s:0.222 data_s:0.027 +INFO 2025-05-20 07:28:48 ts/train.py:232 step:471K smpl:8M ep:36K epch:118.80 loss:0.004 grdn:0.149 lr:4.6e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-20 07:29:51 ts/train.py:232 step:471K smpl:8M ep:36K epch:118.85 loss:0.005 grdn:0.160 lr:4.6e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-20 07:30:54 ts/train.py:232 step:471K smpl:8M ep:36K epch:118.90 loss:0.005 grdn:0.160 lr:4.6e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-20 07:31:57 ts/train.py:232 step:472K smpl:8M ep:36K epch:118.95 loss:0.004 grdn:0.156 lr:4.6e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-20 07:33:01 ts/train.py:232 step:472K smpl:8M ep:36K epch:119.00 loss:0.005 grdn:0.157 lr:4.6e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 07:34:04 ts/train.py:232 step:472K smpl:8M ep:36K epch:119.05 loss:0.005 grdn:0.156 lr:4.6e-05 updt_s:0.220 data_s:0.092 +INFO 2025-05-20 07:35:01 ts/train.py:232 step:472K smpl:8M ep:36K epch:119.10 loss:0.005 grdn:0.162 lr:4.6e-05 updt_s:0.221 data_s:0.063 +INFO 2025-05-20 07:35:52 ts/train.py:232 step:472K smpl:8M ep:36K epch:119.15 loss:0.005 grdn:0.170 lr:4.6e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 07:36:46 ts/train.py:232 step:473K smpl:8M ep:36K epch:119.20 loss:0.005 grdn:0.159 lr:4.6e-05 updt_s:0.221 data_s:0.044 +INFO 2025-05-20 07:37:41 ts/train.py:232 step:473K smpl:8M ep:36K epch:119.25 loss:0.004 grdn:0.161 lr:4.6e-05 updt_s:0.221 data_s:0.055 +INFO 2025-05-20 07:38:39 ts/train.py:232 step:473K smpl:8M ep:36K epch:119.30 loss:0.004 grdn:0.144 lr:4.6e-05 updt_s:0.221 data_s:0.067 +INFO 2025-05-20 07:39:33 ts/train.py:232 step:473K smpl:8M ep:36K epch:119.35 loss:0.005 grdn:0.160 lr:4.6e-05 updt_s:0.221 data_s:0.051 +INFO 2025-05-20 07:40:29 ts/train.py:232 step:473K smpl:8M ep:36K epch:119.40 loss:0.004 grdn:0.155 lr:4.6e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-20 07:41:25 ts/train.py:232 step:474K smpl:8M ep:36K epch:119.45 loss:0.004 grdn:0.151 lr:4.6e-05 updt_s:0.221 data_s:0.055 +INFO 2025-05-20 07:42:18 ts/train.py:232 step:474K smpl:8M ep:36K epch:119.51 loss:0.005 grdn:0.160 lr:4.6e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 07:43:13 ts/train.py:232 step:474K smpl:8M ep:36K epch:119.56 loss:0.004 grdn:0.151 lr:4.6e-05 updt_s:0.221 data_s:0.050 +INFO 2025-05-20 07:44:07 ts/train.py:232 step:474K smpl:8M ep:36K epch:119.61 loss:0.004 grdn:0.149 lr:4.6e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 07:45:02 ts/train.py:232 step:474K smpl:8M ep:36K epch:119.66 loss:0.005 grdn:0.162 lr:4.6e-05 updt_s:0.221 data_s:0.056 +INFO 2025-05-20 07:45:57 ts/train.py:232 step:475K smpl:8M ep:36K epch:119.71 loss:0.005 grdn:0.155 lr:4.6e-05 updt_s:0.221 data_s:0.053 +INFO 2025-05-20 07:46:57 ts/train.py:232 step:475K smpl:8M ep:36K epch:119.76 loss:0.004 grdn:0.151 lr:4.6e-05 updt_s:0.221 data_s:0.079 +INFO 2025-05-20 07:48:01 ts/train.py:232 step:475K smpl:8M ep:36K epch:119.81 loss:0.005 grdn:0.150 lr:4.6e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 07:49:04 ts/train.py:232 step:475K smpl:8M ep:36K epch:119.86 loss:0.004 grdn:0.153 lr:4.6e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-20 07:50:08 ts/train.py:232 step:475K smpl:8M ep:36K epch:119.91 loss:0.005 grdn:0.165 lr:4.6e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 07:51:13 ts/train.py:232 step:476K smpl:8M ep:36K epch:119.96 loss:0.004 grdn:0.155 lr:4.6e-05 updt_s:0.220 data_s:0.102 +INFO 2025-05-20 07:52:17 ts/train.py:232 step:476K smpl:8M ep:36K epch:120.01 loss:0.005 grdn:0.158 lr:4.6e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-20 07:53:18 ts/train.py:232 step:476K smpl:8M ep:36K epch:120.06 loss:0.004 grdn:0.150 lr:4.6e-05 updt_s:0.221 data_s:0.083 +INFO 2025-05-20 07:54:11 ts/train.py:232 step:476K smpl:8M ep:36K epch:120.11 loss:0.004 grdn:0.146 lr:4.5e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 07:55:07 ts/train.py:232 step:476K smpl:8M ep:36K epch:120.16 loss:0.004 grdn:0.152 lr:4.5e-05 updt_s:0.221 data_s:0.057 +INFO 2025-05-20 07:56:02 ts/train.py:232 step:477K smpl:8M ep:36K epch:120.21 loss:0.005 grdn:0.153 lr:4.5e-05 updt_s:0.221 data_s:0.050 +INFO 2025-05-20 07:56:56 ts/train.py:232 step:477K smpl:8M ep:36K epch:120.26 loss:0.004 grdn:0.143 lr:4.5e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 07:57:48 ts/train.py:232 step:477K smpl:8M ep:36K epch:120.31 loss:0.004 grdn:0.150 lr:4.5e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-20 07:58:42 ts/train.py:232 step:477K smpl:8M ep:36K epch:120.36 loss:0.005 grdn:0.168 lr:4.5e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 07:59:34 ts/train.py:232 step:477K smpl:8M ep:36K epch:120.41 loss:0.005 grdn:0.177 lr:4.5e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-20 08:00:28 ts/train.py:232 step:478K smpl:8M ep:36K epch:120.46 loss:0.004 grdn:0.157 lr:4.5e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 08:01:22 ts/train.py:232 step:478K smpl:8M ep:36K epch:120.51 loss:0.005 grdn:0.160 lr:4.5e-05 updt_s:0.220 data_s:0.047 +INFO 2025-05-20 08:02:14 ts/train.py:232 step:478K smpl:8M ep:36K epch:120.56 loss:0.004 grdn:0.149 lr:4.5e-05 updt_s:0.220 data_s:0.038 +INFO 2025-05-20 08:03:07 ts/train.py:232 step:478K smpl:8M ep:36K epch:120.61 loss:0.004 grdn:0.154 lr:4.5e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 08:04:00 ts/train.py:232 step:478K smpl:8M ep:36K epch:120.67 loss:0.004 grdn:0.152 lr:4.5e-05 updt_s:0.220 data_s:0.044 +INFO 2025-05-20 08:04:58 ts/train.py:232 step:479K smpl:8M ep:36K epch:120.72 loss:0.004 grdn:0.150 lr:4.5e-05 updt_s:0.220 data_s:0.069 +INFO 2025-05-20 08:06:01 ts/train.py:232 step:479K smpl:8M ep:36K epch:120.77 loss:0.005 grdn:0.169 lr:4.5e-05 updt_s:0.220 data_s:0.095 +INFO 2025-05-20 08:07:05 ts/train.py:232 step:479K smpl:8M ep:36K epch:120.82 loss:0.004 grdn:0.150 lr:4.5e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 08:08:09 ts/train.py:232 step:479K smpl:8M ep:36K epch:120.87 loss:0.004 grdn:0.160 lr:4.5e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 08:09:13 ts/train.py:232 step:479K smpl:8M ep:36K epch:120.92 loss:0.004 grdn:0.153 lr:4.5e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 08:10:16 ts/train.py:232 step:480K smpl:8M ep:36K epch:120.97 loss:0.005 grdn:0.161 lr:4.5e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 08:11:17 ts/train.py:232 step:480K smpl:8M ep:36K epch:121.02 loss:0.004 grdn:0.153 lr:4.5e-05 updt_s:0.220 data_s:0.084 +INFO 2025-05-20 08:12:07 ts/train.py:232 step:480K smpl:8M ep:36K epch:121.07 loss:0.004 grdn:0.146 lr:4.5e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 08:12:07 ts/train.py:241 Checkpoint policy after step 480000 +INFO 2025-05-20 08:13:00 ts/train.py:232 step:480K smpl:8M ep:36K epch:121.12 loss:0.005 grdn:0.160 lr:4.5e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 08:13:51 ts/train.py:232 step:480K smpl:8M ep:36K epch:121.17 loss:0.004 grdn:0.159 lr:4.5e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 08:14:40 ts/train.py:232 step:481K smpl:8M ep:36K epch:121.22 loss:0.005 grdn:0.161 lr:4.5e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 08:15:30 ts/train.py:232 step:481K smpl:8M ep:36K epch:121.27 loss:0.004 grdn:0.150 lr:4.5e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 08:16:20 ts/train.py:232 step:481K smpl:8M ep:36K epch:121.32 loss:0.005 grdn:0.161 lr:4.5e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 08:17:10 ts/train.py:232 step:481K smpl:8M ep:36K epch:121.37 loss:0.004 grdn:0.148 lr:4.5e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 08:18:00 ts/train.py:232 step:481K smpl:8M ep:36K epch:121.42 loss:0.004 grdn:0.154 lr:4.5e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 08:18:50 ts/train.py:232 step:482K smpl:8M ep:36K epch:121.47 loss:0.004 grdn:0.157 lr:4.5e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 08:19:40 ts/train.py:232 step:482K smpl:8M ep:36K epch:121.52 loss:0.005 grdn:0.159 lr:4.5e-05 updt_s:0.220 data_s:0.031 +INFO 2025-05-20 08:20:30 ts/train.py:232 step:482K smpl:8M ep:36K epch:121.57 loss:0.005 grdn:0.161 lr:4.4e-05 updt_s:0.220 data_s:0.029 +INFO 2025-05-20 08:21:20 ts/train.py:232 step:482K smpl:8M ep:36K epch:121.62 loss:0.004 grdn:0.148 lr:4.4e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 08:22:13 ts/train.py:232 step:482K smpl:8M ep:37K epch:121.67 loss:0.004 grdn:0.148 lr:4.4e-05 updt_s:0.220 data_s:0.048 +INFO 2025-05-20 08:23:18 ts/train.py:232 step:483K smpl:8M ep:37K epch:121.72 loss:0.005 grdn:0.162 lr:4.4e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-20 08:24:21 ts/train.py:232 step:483K smpl:8M ep:37K epch:121.78 loss:0.004 grdn:0.150 lr:4.4e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 08:25:25 ts/train.py:232 step:483K smpl:8M ep:37K epch:121.83 loss:0.004 grdn:0.147 lr:4.4e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 08:26:29 ts/train.py:232 step:483K smpl:8M ep:37K epch:121.88 loss:0.004 grdn:0.149 lr:4.4e-05 updt_s:0.220 data_s:0.101 +INFO 2025-05-20 08:27:33 ts/train.py:232 step:483K smpl:8M ep:37K epch:121.93 loss:0.004 grdn:0.147 lr:4.4e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 08:28:37 ts/train.py:232 step:484K smpl:8M ep:37K epch:121.98 loss:0.004 grdn:0.156 lr:4.4e-05 updt_s:0.220 data_s:0.103 +INFO 2025-05-20 08:29:30 ts/train.py:232 step:484K smpl:8M ep:37K epch:122.03 loss:0.004 grdn:0.158 lr:4.4e-05 updt_s:0.220 data_s:0.040 +INFO 2025-05-20 08:30:21 ts/train.py:232 step:484K smpl:8M ep:37K epch:122.08 loss:0.005 grdn:0.162 lr:4.4e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 08:31:13 ts/train.py:232 step:484K smpl:8M ep:37K epch:122.13 loss:0.004 grdn:0.159 lr:4.4e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 08:32:05 ts/train.py:232 step:484K smpl:8M ep:37K epch:122.18 loss:0.005 grdn:0.159 lr:4.4e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 08:32:57 ts/train.py:232 step:485K smpl:8M ep:37K epch:122.23 loss:0.004 grdn:0.149 lr:4.4e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-20 08:33:50 ts/train.py:232 step:485K smpl:8M ep:37K epch:122.28 loss:0.005 grdn:0.169 lr:4.4e-05 updt_s:0.220 data_s:0.043 +INFO 2025-05-20 08:34:41 ts/train.py:232 step:485K smpl:8M ep:37K epch:122.33 loss:0.004 grdn:0.150 lr:4.4e-05 updt_s:0.220 data_s:0.037 +INFO 2025-05-20 08:35:32 ts/train.py:232 step:485K smpl:8M ep:37K epch:122.38 loss:0.004 grdn:0.147 lr:4.4e-05 updt_s:0.220 data_s:0.033 +INFO 2025-05-20 08:36:24 ts/train.py:232 step:485K smpl:8M ep:37K epch:122.43 loss:0.005 grdn:0.158 lr:4.4e-05 updt_s:0.220 data_s:0.036 +INFO 2025-05-20 08:37:15 ts/train.py:232 step:486K smpl:8M ep:37K epch:122.48 loss:0.004 grdn:0.142 lr:4.4e-05 updt_s:0.220 data_s:0.038 +INFO 2025-05-20 08:38:05 ts/train.py:232 step:486K smpl:8M ep:37K epch:122.53 loss:0.004 grdn:0.151 lr:4.4e-05 updt_s:0.220 data_s:0.029 +INFO 2025-05-20 08:38:58 ts/train.py:232 step:486K smpl:8M ep:37K epch:122.58 loss:0.004 grdn:0.151 lr:4.4e-05 updt_s:0.220 data_s:0.042 +INFO 2025-05-20 08:39:49 ts/train.py:232 step:486K smpl:8M ep:37K epch:122.63 loss:0.005 grdn:0.163 lr:4.4e-05 updt_s:0.220 data_s:0.033 +INFO 2025-05-20 08:40:55 ts/train.py:232 step:486K smpl:8M ep:37K epch:122.68 loss:0.004 grdn:0.150 lr:4.4e-05 updt_s:0.220 data_s:0.110 +INFO 2025-05-20 08:41:59 ts/train.py:232 step:487K smpl:8M ep:37K epch:122.73 loss:0.004 grdn:0.147 lr:4.4e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 08:43:03 ts/train.py:232 step:487K smpl:8M ep:37K epch:122.78 loss:0.004 grdn:0.150 lr:4.4e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 08:44:06 ts/train.py:232 step:487K smpl:8M ep:37K epch:122.83 loss:0.004 grdn:0.145 lr:4.4e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 08:45:10 ts/train.py:232 step:487K smpl:8M ep:37K epch:122.88 loss:0.005 grdn:0.153 lr:4.4e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 08:46:14 ts/train.py:232 step:487K smpl:8M ep:37K epch:122.94 loss:0.005 grdn:0.165 lr:4.4e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 08:47:07 ts/train.py:232 step:488K smpl:8M ep:37K epch:122.99 loss:0.004 grdn:0.149 lr:4.4e-05 updt_s:0.220 data_s:0.043 +INFO 2025-05-20 08:47:58 ts/train.py:232 step:488K smpl:8M ep:37K epch:123.04 loss:0.005 grdn:0.160 lr:4.3e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 08:48:51 ts/train.py:232 step:488K smpl:8M ep:37K epch:123.09 loss:0.004 grdn:0.155 lr:4.3e-05 updt_s:0.220 data_s:0.044 +INFO 2025-05-20 08:49:45 ts/train.py:232 step:488K smpl:8M ep:37K epch:123.14 loss:0.004 grdn:0.154 lr:4.3e-05 updt_s:0.220 data_s:0.047 +INFO 2025-05-20 08:50:34 ts/train.py:232 step:488K smpl:8M ep:37K epch:123.19 loss:0.005 grdn:0.165 lr:4.3e-05 updt_s:0.220 data_s:0.027 +INFO 2025-05-20 08:51:25 ts/train.py:232 step:489K smpl:8M ep:37K epch:123.24 loss:0.004 grdn:0.159 lr:4.3e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 08:52:17 ts/train.py:232 step:489K smpl:8M ep:37K epch:123.29 loss:0.004 grdn:0.149 lr:4.3e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 08:53:10 ts/train.py:232 step:489K smpl:8M ep:37K epch:123.34 loss:0.004 grdn:0.148 lr:4.3e-05 updt_s:0.220 data_s:0.046 +INFO 2025-05-20 08:54:04 ts/train.py:232 step:489K smpl:8M ep:37K epch:123.39 loss:0.005 grdn:0.159 lr:4.3e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 08:54:55 ts/train.py:232 step:489K smpl:8M ep:37K epch:123.44 loss:0.004 grdn:0.152 lr:4.3e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 08:55:47 ts/train.py:232 step:490K smpl:8M ep:37K epch:123.49 loss:0.004 grdn:0.156 lr:4.3e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 08:56:39 ts/train.py:232 step:490K smpl:8M ep:37K epch:123.54 loss:0.005 grdn:0.161 lr:4.3e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 08:57:31 ts/train.py:232 step:490K smpl:8M ep:37K epch:123.59 loss:0.004 grdn:0.155 lr:4.3e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 08:58:35 ts/train.py:232 step:490K smpl:8M ep:37K epch:123.64 loss:0.004 grdn:0.158 lr:4.3e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-20 08:59:40 ts/train.py:232 step:490K smpl:8M ep:37K epch:123.69 loss:0.004 grdn:0.156 lr:4.3e-05 updt_s:0.221 data_s:0.103 +INFO 2025-05-20 09:00:44 ts/train.py:232 step:491K smpl:8M ep:37K epch:123.74 loss:0.005 grdn:0.167 lr:4.3e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 09:01:47 ts/train.py:232 step:491K smpl:8M ep:37K epch:123.79 loss:0.004 grdn:0.163 lr:4.3e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-20 09:02:50 ts/train.py:232 step:491K smpl:8M ep:37K epch:123.84 loss:0.004 grdn:0.160 lr:4.3e-05 updt_s:0.221 data_s:0.094 +INFO 2025-05-20 09:03:54 ts/train.py:232 step:491K smpl:8M ep:37K epch:123.89 loss:0.004 grdn:0.157 lr:4.3e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 09:04:49 ts/train.py:232 step:491K smpl:8M ep:37K epch:123.94 loss:0.003 grdn:0.130 lr:4.3e-05 updt_s:0.221 data_s:0.055 +INFO 2025-05-20 09:05:42 ts/train.py:232 step:492K smpl:8M ep:37K epch:123.99 loss:0.004 grdn:0.152 lr:4.3e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-20 09:06:36 ts/train.py:232 step:492K smpl:8M ep:37K epch:124.05 loss:0.004 grdn:0.148 lr:4.3e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 09:07:28 ts/train.py:232 step:492K smpl:8M ep:37K epch:124.10 loss:0.005 grdn:0.170 lr:4.3e-05 updt_s:0.220 data_s:0.043 +INFO 2025-05-20 09:08:21 ts/train.py:232 step:492K smpl:8M ep:37K epch:124.15 loss:0.005 grdn:0.170 lr:4.3e-05 updt_s:0.220 data_s:0.042 +INFO 2025-05-20 09:09:13 ts/train.py:232 step:492K smpl:8M ep:37K epch:124.20 loss:0.005 grdn:0.156 lr:4.3e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 09:10:05 ts/train.py:232 step:493K smpl:8M ep:37K epch:124.25 loss:0.004 grdn:0.146 lr:4.3e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 09:10:58 ts/train.py:232 step:493K smpl:8M ep:37K epch:124.30 loss:0.005 grdn:0.163 lr:4.3e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 09:11:51 ts/train.py:232 step:493K smpl:8M ep:37K epch:124.35 loss:0.005 grdn:0.165 lr:4.3e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 09:12:45 ts/train.py:232 step:493K smpl:8M ep:37K epch:124.40 loss:0.004 grdn:0.154 lr:4.3e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 09:13:39 ts/train.py:232 step:493K smpl:8M ep:37K epch:124.45 loss:0.005 grdn:0.161 lr:4.3e-05 updt_s:0.220 data_s:0.049 +INFO 2025-05-20 09:14:34 ts/train.py:232 step:494K smpl:8M ep:37K epch:124.50 loss:0.004 grdn:0.159 lr:4.2e-05 updt_s:0.220 data_s:0.050 +INFO 2025-05-20 09:15:27 ts/train.py:232 step:494K smpl:8M ep:37K epch:124.55 loss:0.004 grdn:0.146 lr:4.2e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 09:16:27 ts/train.py:232 step:494K smpl:8M ep:37K epch:124.60 loss:0.004 grdn:0.152 lr:4.2e-05 updt_s:0.220 data_s:0.082 +INFO 2025-05-20 09:17:31 ts/train.py:232 step:494K smpl:8M ep:37K epch:124.65 loss:0.004 grdn:0.159 lr:4.2e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 09:18:34 ts/train.py:232 step:494K smpl:8M ep:37K epch:124.70 loss:0.004 grdn:0.156 lr:4.2e-05 updt_s:0.220 data_s:0.095 +INFO 2025-05-20 09:19:38 ts/train.py:232 step:495K smpl:8M ep:37K epch:124.75 loss:0.004 grdn:0.159 lr:4.2e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 09:20:43 ts/train.py:232 step:495K smpl:8M ep:37K epch:124.80 loss:0.004 grdn:0.151 lr:4.2e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-20 09:21:46 ts/train.py:232 step:495K smpl:8M ep:37K epch:124.85 loss:0.005 grdn:0.167 lr:4.2e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-20 09:22:42 ts/train.py:232 step:495K smpl:8M ep:37K epch:124.90 loss:0.004 grdn:0.144 lr:4.2e-05 updt_s:0.221 data_s:0.062 +INFO 2025-05-20 09:23:33 ts/train.py:232 step:495K smpl:8M ep:37K epch:124.95 loss:0.004 grdn:0.157 lr:4.2e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 09:24:24 ts/train.py:232 step:496K smpl:8M ep:38K epch:125.00 loss:0.004 grdn:0.145 lr:4.2e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 09:25:17 ts/train.py:232 step:496K smpl:8M ep:38K epch:125.05 loss:0.004 grdn:0.163 lr:4.2e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-20 09:26:11 ts/train.py:232 step:496K smpl:8M ep:38K epch:125.10 loss:0.004 grdn:0.162 lr:4.2e-05 updt_s:0.221 data_s:0.047 +INFO 2025-05-20 09:27:06 ts/train.py:232 step:496K smpl:8M ep:38K epch:125.15 loss:0.005 grdn:0.161 lr:4.2e-05 updt_s:0.221 data_s:0.056 +INFO 2025-05-20 09:28:01 ts/train.py:232 step:496K smpl:8M ep:38K epch:125.21 loss:0.004 grdn:0.149 lr:4.2e-05 updt_s:0.221 data_s:0.054 +INFO 2025-05-20 09:28:54 ts/train.py:232 step:497K smpl:8M ep:38K epch:125.26 loss:0.004 grdn:0.154 lr:4.2e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 09:29:46 ts/train.py:232 step:497K smpl:8M ep:38K epch:125.31 loss:0.005 grdn:0.163 lr:4.2e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 09:30:37 ts/train.py:232 step:497K smpl:8M ep:38K epch:125.36 loss:0.005 grdn:0.165 lr:4.2e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 09:31:28 ts/train.py:232 step:497K smpl:8M ep:38K epch:125.41 loss:0.005 grdn:0.160 lr:4.2e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 09:32:22 ts/train.py:232 step:497K smpl:8M ep:38K epch:125.46 loss:0.005 grdn:0.166 lr:4.2e-05 updt_s:0.221 data_s:0.050 +INFO 2025-05-20 09:33:17 ts/train.py:232 step:498K smpl:8M ep:38K epch:125.51 loss:0.004 grdn:0.148 lr:4.2e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 09:34:15 ts/train.py:232 step:498K smpl:8M ep:38K epch:125.56 loss:0.005 grdn:0.161 lr:4.2e-05 updt_s:0.220 data_s:0.073 +INFO 2025-05-20 09:35:19 ts/train.py:232 step:498K smpl:8M ep:38K epch:125.61 loss:0.004 grdn:0.156 lr:4.2e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 09:36:23 ts/train.py:232 step:498K smpl:8M ep:38K epch:125.66 loss:0.004 grdn:0.152 lr:4.2e-05 updt_s:0.221 data_s:0.097 +INFO 2025-05-20 09:37:26 ts/train.py:232 step:498K smpl:8M ep:38K epch:125.71 loss:0.004 grdn:0.163 lr:4.2e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-20 09:38:30 ts/train.py:232 step:499K smpl:8M ep:38K epch:125.76 loss:0.004 grdn:0.147 lr:4.2e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 09:39:33 ts/train.py:232 step:499K smpl:8M ep:38K epch:125.81 loss:0.004 grdn:0.162 lr:4.2e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 09:40:33 ts/train.py:232 step:499K smpl:8M ep:38K epch:125.86 loss:0.004 grdn:0.164 lr:4.2e-05 updt_s:0.221 data_s:0.076 +INFO 2025-05-20 09:41:28 ts/train.py:232 step:499K smpl:8M ep:38K epch:125.91 loss:0.004 grdn:0.150 lr:4.2e-05 updt_s:0.221 data_s:0.053 +INFO 2025-05-20 09:42:23 ts/train.py:232 step:499K smpl:8M ep:38K epch:125.96 loss:0.005 grdn:0.172 lr:4.1e-05 updt_s:0.220 data_s:0.052 +INFO 2025-05-20 09:43:17 ts/train.py:232 step:500K smpl:8M ep:38K epch:126.01 loss:0.004 grdn:0.150 lr:4.1e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 09:44:11 ts/train.py:232 step:500K smpl:8M ep:38K epch:126.06 loss:0.004 grdn:0.148 lr:4.1e-05 updt_s:0.221 data_s:0.051 +INFO 2025-05-20 09:45:05 ts/train.py:232 step:500K smpl:8M ep:38K epch:126.11 loss:0.004 grdn:0.153 lr:4.1e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 09:45:05 ts/train.py:241 Checkpoint policy after step 500000 +INFO 2025-05-20 09:46:04 ts/train.py:232 step:500K smpl:8M ep:38K epch:126.16 loss:0.004 grdn:0.162 lr:4.1e-05 updt_s:0.220 data_s:0.048 +INFO 2025-05-20 09:46:56 ts/train.py:232 step:500K smpl:8M ep:38K epch:126.21 loss:0.004 grdn:0.152 lr:4.1e-05 updt_s:0.220 data_s:0.042 +INFO 2025-05-20 09:47:49 ts/train.py:232 step:501K smpl:8M ep:38K epch:126.26 loss:0.004 grdn:0.161 lr:4.1e-05 updt_s:0.220 data_s:0.040 +INFO 2025-05-20 09:48:43 ts/train.py:232 step:501K smpl:8M ep:38K epch:126.32 loss:0.004 grdn:0.156 lr:4.1e-05 updt_s:0.221 data_s:0.049 +INFO 2025-05-20 09:49:37 ts/train.py:232 step:501K smpl:8M ep:38K epch:126.37 loss:0.004 grdn:0.147 lr:4.1e-05 updt_s:0.220 data_s:0.052 +INFO 2025-05-20 09:50:31 ts/train.py:232 step:501K smpl:8M ep:38K epch:126.42 loss:0.004 grdn:0.143 lr:4.1e-05 updt_s:0.220 data_s:0.048 +INFO 2025-05-20 09:51:26 ts/train.py:232 step:501K smpl:8M ep:38K epch:126.47 loss:0.004 grdn:0.149 lr:4.1e-05 updt_s:0.220 data_s:0.054 +INFO 2025-05-20 09:52:26 ts/train.py:232 step:502K smpl:8M ep:38K epch:126.52 loss:0.004 grdn:0.157 lr:4.1e-05 updt_s:0.220 data_s:0.081 +INFO 2025-05-20 09:53:30 ts/train.py:232 step:502K smpl:8M ep:38K epch:126.57 loss:0.005 grdn:0.172 lr:4.1e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 09:54:34 ts/train.py:232 step:502K smpl:8M ep:38K epch:126.62 loss:0.004 grdn:0.146 lr:4.1e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 09:55:38 ts/train.py:232 step:502K smpl:8M ep:38K epch:126.67 loss:0.004 grdn:0.158 lr:4.1e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 09:56:41 ts/train.py:232 step:502K smpl:8M ep:38K epch:126.72 loss:0.005 grdn:0.168 lr:4.1e-05 updt_s:0.220 data_s:0.095 +INFO 2025-05-20 09:57:46 ts/train.py:232 step:503K smpl:8M ep:38K epch:126.77 loss:0.005 grdn:0.164 lr:4.1e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 09:58:47 ts/train.py:232 step:503K smpl:8M ep:38K epch:126.82 loss:0.004 grdn:0.158 lr:4.1e-05 updt_s:0.220 data_s:0.088 +INFO 2025-05-20 09:59:38 ts/train.py:232 step:503K smpl:8M ep:38K epch:126.87 loss:0.004 grdn:0.163 lr:4.1e-05 updt_s:0.220 data_s:0.034 +INFO 2025-05-20 10:00:28 ts/train.py:232 step:503K smpl:8M ep:38K epch:126.92 loss:0.005 grdn:0.163 lr:4.1e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 10:01:18 ts/train.py:232 step:503K smpl:8M ep:38K epch:126.97 loss:0.004 grdn:0.152 lr:4.1e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 10:02:11 ts/train.py:232 step:504K smpl:8M ep:38K epch:127.02 loss:0.004 grdn:0.154 lr:4.1e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 10:03:02 ts/train.py:232 step:504K smpl:8M ep:38K epch:127.07 loss:0.004 grdn:0.156 lr:4.1e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 10:03:52 ts/train.py:232 step:504K smpl:8M ep:38K epch:127.12 loss:0.004 grdn:0.157 lr:4.1e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 10:04:43 ts/train.py:232 step:504K smpl:8M ep:38K epch:127.17 loss:0.004 grdn:0.145 lr:4.1e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 10:05:35 ts/train.py:232 step:504K smpl:8M ep:38K epch:127.22 loss:0.004 grdn:0.160 lr:4.1e-05 updt_s:0.220 data_s:0.037 +INFO 2025-05-20 10:06:26 ts/train.py:232 step:505K smpl:8M ep:38K epch:127.27 loss:0.004 grdn:0.164 lr:4.1e-05 updt_s:0.220 data_s:0.033 +INFO 2025-05-20 10:07:17 ts/train.py:232 step:505K smpl:8M ep:38K epch:127.32 loss:0.004 grdn:0.153 lr:4.1e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 10:08:10 ts/train.py:232 step:505K smpl:8M ep:38K epch:127.37 loss:0.004 grdn:0.162 lr:4.1e-05 updt_s:0.220 data_s:0.040 +INFO 2025-05-20 10:09:00 ts/train.py:232 step:505K smpl:8M ep:38K epch:127.42 loss:0.005 grdn:0.169 lr:4.0e-05 updt_s:0.220 data_s:0.031 +INFO 2025-05-20 10:09:56 ts/train.py:232 step:505K smpl:8M ep:38K epch:127.48 loss:0.004 grdn:0.149 lr:4.0e-05 updt_s:0.220 data_s:0.058 +INFO 2025-05-20 10:11:01 ts/train.py:232 step:506K smpl:8M ep:38K epch:127.53 loss:0.004 grdn:0.143 lr:4.0e-05 updt_s:0.221 data_s:0.105 +INFO 2025-05-20 10:12:05 ts/train.py:232 step:506K smpl:8M ep:38K epch:127.58 loss:0.004 grdn:0.155 lr:4.0e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 10:13:10 ts/train.py:232 step:506K smpl:8M ep:38K epch:127.63 loss:0.004 grdn:0.161 lr:4.0e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-20 10:14:14 ts/train.py:232 step:506K smpl:8M ep:38K epch:127.68 loss:0.004 grdn:0.162 lr:4.0e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 10:15:18 ts/train.py:232 step:506K smpl:8M ep:38K epch:127.73 loss:0.004 grdn:0.155 lr:4.0e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 10:16:22 ts/train.py:232 step:507K smpl:8M ep:38K epch:127.78 loss:0.004 grdn:0.155 lr:4.0e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 10:17:12 ts/train.py:232 step:507K smpl:8M ep:38K epch:127.83 loss:0.004 grdn:0.160 lr:4.0e-05 updt_s:0.220 data_s:0.029 +INFO 2025-05-20 10:18:01 ts/train.py:232 step:507K smpl:8M ep:38K epch:127.88 loss:0.004 grdn:0.149 lr:4.0e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 10:18:51 ts/train.py:232 step:507K smpl:8M ep:38K epch:127.93 loss:0.004 grdn:0.161 lr:4.0e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 10:19:40 ts/train.py:232 step:507K smpl:8M ep:38K epch:127.98 loss:0.004 grdn:0.167 lr:4.0e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 10:20:28 ts/train.py:232 step:508K smpl:8M ep:38K epch:128.03 loss:0.004 grdn:0.152 lr:4.0e-05 updt_s:0.221 data_s:0.020 +INFO 2025-05-20 10:21:18 ts/train.py:232 step:508K smpl:8M ep:38K epch:128.08 loss:0.004 grdn:0.155 lr:4.0e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 10:22:07 ts/train.py:232 step:508K smpl:8M ep:38K epch:128.13 loss:0.004 grdn:0.160 lr:4.0e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-20 10:22:57 ts/train.py:232 step:508K smpl:8M ep:38K epch:128.18 loss:0.004 grdn:0.151 lr:4.0e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 10:23:47 ts/train.py:232 step:508K smpl:8M ep:38K epch:128.23 loss:0.004 grdn:0.152 lr:4.0e-05 updt_s:0.220 data_s:0.027 +INFO 2025-05-20 10:24:36 ts/train.py:232 step:509K smpl:8M ep:38K epch:128.28 loss:0.004 grdn:0.161 lr:4.0e-05 updt_s:0.220 data_s:0.027 +INFO 2025-05-20 10:25:27 ts/train.py:232 step:509K smpl:8M ep:38K epch:128.33 loss:0.004 grdn:0.158 lr:4.0e-05 updt_s:0.220 data_s:0.031 +INFO 2025-05-20 10:26:15 ts/train.py:232 step:509K smpl:8M ep:39K epch:128.38 loss:0.004 grdn:0.158 lr:4.0e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-20 10:27:05 ts/train.py:232 step:509K smpl:8M ep:39K epch:128.43 loss:0.005 grdn:0.173 lr:4.0e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 10:28:11 ts/train.py:232 step:509K smpl:8M ep:39K epch:128.48 loss:0.004 grdn:0.165 lr:4.0e-05 updt_s:0.220 data_s:0.112 +INFO 2025-05-20 10:29:14 ts/train.py:232 step:510K smpl:8M ep:39K epch:128.53 loss:0.005 grdn:0.173 lr:4.0e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 10:30:18 ts/train.py:232 step:510K smpl:8M ep:39K epch:128.59 loss:0.005 grdn:0.170 lr:4.0e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 10:31:22 ts/train.py:232 step:510K smpl:8M ep:39K epch:128.64 loss:0.004 grdn:0.159 lr:4.0e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 10:32:26 ts/train.py:232 step:510K smpl:8M ep:39K epch:128.69 loss:0.004 grdn:0.148 lr:4.0e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 10:33:29 ts/train.py:232 step:510K smpl:8M ep:39K epch:128.74 loss:0.004 grdn:0.169 lr:4.0e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 10:34:25 ts/train.py:232 step:511K smpl:8M ep:39K epch:128.79 loss:0.004 grdn:0.156 lr:4.0e-05 updt_s:0.220 data_s:0.057 +INFO 2025-05-20 10:35:20 ts/train.py:232 step:511K smpl:8M ep:39K epch:128.84 loss:0.004 grdn:0.144 lr:4.0e-05 updt_s:0.220 data_s:0.055 +INFO 2025-05-20 10:36:14 ts/train.py:232 step:511K smpl:8M ep:39K epch:128.89 loss:0.004 grdn:0.145 lr:3.9e-05 updt_s:0.220 data_s:0.051 +INFO 2025-05-20 10:37:09 ts/train.py:232 step:511K smpl:8M ep:39K epch:128.94 loss:0.004 grdn:0.150 lr:3.9e-05 updt_s:0.220 data_s:0.052 +INFO 2025-05-20 10:38:00 ts/train.py:232 step:511K smpl:8M ep:39K epch:128.99 loss:0.004 grdn:0.152 lr:3.9e-05 updt_s:0.220 data_s:0.035 +INFO 2025-05-20 10:38:53 ts/train.py:232 step:512K smpl:8M ep:39K epch:129.04 loss:0.005 grdn:0.177 lr:3.9e-05 updt_s:0.220 data_s:0.044 +INFO 2025-05-20 10:39:46 ts/train.py:232 step:512K smpl:8M ep:39K epch:129.09 loss:0.004 grdn:0.160 lr:3.9e-05 updt_s:0.220 data_s:0.042 +INFO 2025-05-20 10:40:38 ts/train.py:232 step:512K smpl:8M ep:39K epch:129.14 loss:0.004 grdn:0.158 lr:3.9e-05 updt_s:0.220 data_s:0.042 +INFO 2025-05-20 10:41:32 ts/train.py:232 step:512K smpl:8M ep:39K epch:129.19 loss:0.004 grdn:0.151 lr:3.9e-05 updt_s:0.220 data_s:0.047 +INFO 2025-05-20 10:42:25 ts/train.py:232 step:512K smpl:8M ep:39K epch:129.24 loss:0.004 grdn:0.161 lr:3.9e-05 updt_s:0.220 data_s:0.041 +INFO 2025-05-20 10:43:18 ts/train.py:232 step:513K smpl:8M ep:39K epch:129.29 loss:0.004 grdn:0.151 lr:3.9e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 10:44:10 ts/train.py:232 step:513K smpl:8M ep:39K epch:129.34 loss:0.004 grdn:0.149 lr:3.9e-05 updt_s:0.220 data_s:0.037 +INFO 2025-05-20 10:45:00 ts/train.py:232 step:513K smpl:8M ep:39K epch:129.39 loss:0.004 grdn:0.147 lr:3.9e-05 updt_s:0.220 data_s:0.029 +INFO 2025-05-20 10:46:04 ts/train.py:232 step:513K smpl:8M ep:39K epch:129.44 loss:0.004 grdn:0.165 lr:3.9e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 10:47:08 ts/train.py:232 step:513K smpl:8M ep:39K epch:129.49 loss:0.004 grdn:0.153 lr:3.9e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 10:48:12 ts/train.py:232 step:514K smpl:8M ep:39K epch:129.54 loss:0.004 grdn:0.143 lr:3.9e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 10:49:16 ts/train.py:232 step:514K smpl:8M ep:39K epch:129.59 loss:0.004 grdn:0.147 lr:3.9e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 10:50:20 ts/train.py:232 step:514K smpl:8M ep:39K epch:129.64 loss:0.004 grdn:0.156 lr:3.9e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 10:51:23 ts/train.py:232 step:514K smpl:8M ep:39K epch:129.69 loss:0.004 grdn:0.152 lr:3.9e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 10:52:18 ts/train.py:232 step:514K smpl:8M ep:39K epch:129.75 loss:0.004 grdn:0.167 lr:3.9e-05 updt_s:0.220 data_s:0.052 +INFO 2025-05-20 10:53:09 ts/train.py:232 step:515K smpl:8M ep:39K epch:129.80 loss:0.004 grdn:0.157 lr:3.9e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 10:54:00 ts/train.py:232 step:515K smpl:8M ep:39K epch:129.85 loss:0.004 grdn:0.157 lr:3.9e-05 updt_s:0.220 data_s:0.035 +INFO 2025-05-20 10:54:50 ts/train.py:232 step:515K smpl:8M ep:39K epch:129.90 loss:0.004 grdn:0.164 lr:3.9e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 10:55:41 ts/train.py:232 step:515K smpl:8M ep:39K epch:129.95 loss:0.004 grdn:0.157 lr:3.9e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 10:56:32 ts/train.py:232 step:515K smpl:8M ep:39K epch:130.00 loss:0.004 grdn:0.156 lr:3.9e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 10:57:22 ts/train.py:232 step:516K smpl:8M ep:39K epch:130.05 loss:0.004 grdn:0.153 lr:3.9e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 10:58:14 ts/train.py:232 step:516K smpl:8M ep:39K epch:130.10 loss:0.004 grdn:0.163 lr:3.9e-05 updt_s:0.220 data_s:0.034 +INFO 2025-05-20 10:59:04 ts/train.py:232 step:516K smpl:8M ep:39K epch:130.15 loss:0.004 grdn:0.146 lr:3.9e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 10:59:55 ts/train.py:232 step:516K smpl:8M ep:39K epch:130.20 loss:0.004 grdn:0.156 lr:3.9e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 11:00:44 ts/train.py:232 step:516K smpl:8M ep:39K epch:130.25 loss:0.004 grdn:0.150 lr:3.9e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 11:01:35 ts/train.py:232 step:517K smpl:8M ep:39K epch:130.30 loss:0.004 grdn:0.160 lr:3.9e-05 updt_s:0.220 data_s:0.031 +INFO 2025-05-20 11:02:24 ts/train.py:232 step:517K smpl:8M ep:39K epch:130.35 loss:0.004 grdn:0.152 lr:3.8e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 11:03:26 ts/train.py:232 step:517K smpl:8M ep:39K epch:130.40 loss:0.004 grdn:0.161 lr:3.8e-05 updt_s:0.220 data_s:0.087 +INFO 2025-05-20 11:04:30 ts/train.py:232 step:517K smpl:8M ep:39K epch:130.45 loss:0.004 grdn:0.148 lr:3.8e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 11:05:34 ts/train.py:232 step:517K smpl:8M ep:39K epch:130.50 loss:0.004 grdn:0.158 lr:3.8e-05 updt_s:0.220 data_s:0.100 +INFO 2025-05-20 11:06:38 ts/train.py:232 step:518K smpl:8M ep:39K epch:130.55 loss:0.005 grdn:0.175 lr:3.8e-05 updt_s:0.220 data_s:0.099 +INFO 2025-05-20 11:07:42 ts/train.py:232 step:518K smpl:8M ep:39K epch:130.60 loss:0.004 grdn:0.157 lr:3.8e-05 updt_s:0.220 data_s:0.098 +INFO 2025-05-20 11:08:45 ts/train.py:232 step:518K smpl:8M ep:39K epch:130.65 loss:0.004 grdn:0.157 lr:3.8e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 11:09:43 ts/train.py:232 step:518K smpl:8M ep:39K epch:130.70 loss:0.004 grdn:0.154 lr:3.8e-05 updt_s:0.220 data_s:0.071 +INFO 2025-05-20 11:10:36 ts/train.py:232 step:518K smpl:8M ep:39K epch:130.75 loss:0.004 grdn:0.143 lr:3.8e-05 updt_s:0.220 data_s:0.040 +INFO 2025-05-20 11:11:29 ts/train.py:232 step:519K smpl:8M ep:39K epch:130.80 loss:0.004 grdn:0.161 lr:3.8e-05 updt_s:0.220 data_s:0.046 +INFO 2025-05-20 11:12:20 ts/train.py:232 step:519K smpl:8M ep:39K epch:130.86 loss:0.004 grdn:0.165 lr:3.8e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 11:13:14 ts/train.py:232 step:519K smpl:8M ep:39K epch:130.91 loss:0.004 grdn:0.162 lr:3.8e-05 updt_s:0.220 data_s:0.048 +INFO 2025-05-20 11:14:06 ts/train.py:232 step:519K smpl:8M ep:39K epch:130.96 loss:0.004 grdn:0.163 lr:3.8e-05 updt_s:0.220 data_s:0.039 +INFO 2025-05-20 11:14:59 ts/train.py:232 step:519K smpl:8M ep:39K epch:131.01 loss:0.004 grdn:0.163 lr:3.8e-05 updt_s:0.220 data_s:0.047 +INFO 2025-05-20 11:15:52 ts/train.py:232 step:520K smpl:8M ep:39K epch:131.06 loss:0.004 grdn:0.157 lr:3.8e-05 updt_s:0.220 data_s:0.045 +INFO 2025-05-20 11:16:45 ts/train.py:232 step:520K smpl:8M ep:39K epch:131.11 loss:0.004 grdn:0.154 lr:3.8e-05 updt_s:0.220 data_s:0.041 +INFO 2025-05-20 11:17:37 ts/train.py:232 step:520K smpl:8M ep:39K epch:131.16 loss:0.004 grdn:0.153 lr:3.8e-05 updt_s:0.220 data_s:0.040 +INFO 2025-05-20 11:17:37 ts/train.py:241 Checkpoint policy after step 520000 +INFO 2025-05-20 11:18:32 ts/train.py:232 step:520K smpl:8M ep:39K epch:131.21 loss:0.004 grdn:0.143 lr:3.8e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 11:19:25 ts/train.py:232 step:520K smpl:8M ep:39K epch:131.26 loss:0.004 grdn:0.143 lr:3.8e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 11:20:16 ts/train.py:232 step:521K smpl:8M ep:39K epch:131.31 loss:0.004 grdn:0.159 lr:3.8e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 11:21:16 ts/train.py:232 step:521K smpl:8M ep:39K epch:131.36 loss:0.004 grdn:0.164 lr:3.8e-05 updt_s:0.220 data_s:0.077 +INFO 2025-05-20 11:22:21 ts/train.py:232 step:521K smpl:8M ep:39K epch:131.41 loss:0.004 grdn:0.153 lr:3.8e-05 updt_s:0.220 data_s:0.104 +INFO 2025-05-20 11:23:25 ts/train.py:232 step:521K smpl:8M ep:39K epch:131.46 loss:0.004 grdn:0.154 lr:3.8e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-20 11:24:29 ts/train.py:232 step:521K smpl:8M ep:39K epch:131.51 loss:0.003 grdn:0.148 lr:3.8e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 11:25:33 ts/train.py:232 step:522K smpl:8M ep:39K epch:131.56 loss:0.004 grdn:0.164 lr:3.8e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-20 11:26:36 ts/train.py:232 step:522K smpl:8M ep:39K epch:131.61 loss:0.004 grdn:0.156 lr:3.8e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-20 11:27:35 ts/train.py:232 step:522K smpl:8M ep:39K epch:131.66 loss:0.004 grdn:0.156 lr:3.8e-05 updt_s:0.221 data_s:0.072 +INFO 2025-05-20 11:28:25 ts/train.py:232 step:522K smpl:8M ep:40K epch:131.71 loss:0.004 grdn:0.144 lr:3.8e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 11:29:15 ts/train.py:232 step:522K smpl:8M ep:40K epch:131.76 loss:0.004 grdn:0.153 lr:3.8e-05 updt_s:0.221 data_s:0.030 +INFO 2025-05-20 11:30:05 ts/train.py:232 step:523K smpl:8M ep:40K epch:131.81 loss:0.004 grdn:0.156 lr:3.8e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 11:30:56 ts/train.py:232 step:523K smpl:8M ep:40K epch:131.86 loss:0.004 grdn:0.154 lr:3.7e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 11:31:46 ts/train.py:232 step:523K smpl:8M ep:40K epch:131.91 loss:0.004 grdn:0.161 lr:3.7e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 11:32:36 ts/train.py:232 step:523K smpl:8M ep:40K epch:131.97 loss:0.004 grdn:0.141 lr:3.7e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 11:33:27 ts/train.py:232 step:523K smpl:8M ep:40K epch:132.02 loss:0.004 grdn:0.153 lr:3.7e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 11:34:17 ts/train.py:232 step:524K smpl:8M ep:40K epch:132.07 loss:0.004 grdn:0.151 lr:3.7e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 11:35:09 ts/train.py:232 step:524K smpl:8M ep:40K epch:132.12 loss:0.004 grdn:0.142 lr:3.7e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 11:35:58 ts/train.py:232 step:524K smpl:8M ep:40K epch:132.17 loss:0.003 grdn:0.139 lr:3.7e-05 updt_s:0.222 data_s:0.023 +INFO 2025-05-20 11:36:48 ts/train.py:232 step:524K smpl:8M ep:40K epch:132.22 loss:0.004 grdn:0.157 lr:3.7e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 11:37:38 ts/train.py:232 step:524K smpl:8M ep:40K epch:132.27 loss:0.003 grdn:0.144 lr:3.7e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 11:38:34 ts/train.py:232 step:525K smpl:8M ep:40K epch:132.32 loss:0.005 grdn:0.172 lr:3.7e-05 updt_s:0.221 data_s:0.061 +INFO 2025-05-20 11:39:38 ts/train.py:232 step:525K smpl:8M ep:40K epch:132.37 loss:0.004 grdn:0.142 lr:3.7e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-20 11:40:41 ts/train.py:232 step:525K smpl:8M ep:40K epch:132.42 loss:0.004 grdn:0.163 lr:3.7e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 11:41:44 ts/train.py:232 step:525K smpl:8M ep:40K epch:132.47 loss:0.004 grdn:0.148 lr:3.7e-05 updt_s:0.221 data_s:0.095 +INFO 2025-05-20 11:42:47 ts/train.py:232 step:525K smpl:8M ep:40K epch:132.52 loss:0.004 grdn:0.162 lr:3.7e-05 updt_s:0.220 data_s:0.091 +INFO 2025-05-20 11:43:49 ts/train.py:232 step:526K smpl:8M ep:40K epch:132.57 loss:0.004 grdn:0.165 lr:3.7e-05 updt_s:0.220 data_s:0.092 +INFO 2025-05-20 11:44:51 ts/train.py:232 step:526K smpl:8M ep:40K epch:132.62 loss:0.004 grdn:0.164 lr:3.7e-05 updt_s:0.220 data_s:0.087 +INFO 2025-05-20 11:45:40 ts/train.py:232 step:526K smpl:8M ep:40K epch:132.67 loss:0.004 grdn:0.163 lr:3.7e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 11:46:30 ts/train.py:232 step:526K smpl:8M ep:40K epch:132.72 loss:0.004 grdn:0.150 lr:3.7e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 11:47:20 ts/train.py:232 step:526K smpl:8M ep:40K epch:132.77 loss:0.004 grdn:0.154 lr:3.7e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 11:48:10 ts/train.py:232 step:527K smpl:8M ep:40K epch:132.82 loss:0.004 grdn:0.151 lr:3.7e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 11:49:00 ts/train.py:232 step:527K smpl:8M ep:40K epch:132.87 loss:0.004 grdn:0.151 lr:3.7e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 11:49:50 ts/train.py:232 step:527K smpl:8M ep:40K epch:132.92 loss:0.004 grdn:0.156 lr:3.7e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 11:50:40 ts/train.py:232 step:527K smpl:8M ep:40K epch:132.97 loss:0.004 grdn:0.171 lr:3.7e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 11:51:30 ts/train.py:232 step:527K smpl:8M ep:40K epch:133.02 loss:0.004 grdn:0.164 lr:3.7e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 11:52:20 ts/train.py:232 step:528K smpl:8M ep:40K epch:133.07 loss:0.004 grdn:0.156 lr:3.7e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 11:53:10 ts/train.py:232 step:528K smpl:8M ep:40K epch:133.13 loss:0.004 grdn:0.158 lr:3.7e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-20 11:53:59 ts/train.py:232 step:528K smpl:8M ep:40K epch:133.18 loss:0.004 grdn:0.154 lr:3.7e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 11:54:49 ts/train.py:232 step:528K smpl:8M ep:40K epch:133.23 loss:0.004 grdn:0.159 lr:3.7e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 11:55:41 ts/train.py:232 step:528K smpl:8M ep:40K epch:133.28 loss:0.004 grdn:0.163 lr:3.7e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 11:56:45 ts/train.py:232 step:529K smpl:8M ep:40K epch:133.33 loss:0.004 grdn:0.151 lr:3.7e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-20 11:57:49 ts/train.py:232 step:529K smpl:8M ep:40K epch:133.38 loss:0.004 grdn:0.153 lr:3.6e-05 updt_s:0.220 data_s:0.102 +INFO 2025-05-20 11:58:53 ts/train.py:232 step:529K smpl:8M ep:40K epch:133.43 loss:0.004 grdn:0.156 lr:3.6e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 11:59:57 ts/train.py:232 step:529K smpl:8M ep:40K epch:133.48 loss:0.003 grdn:0.140 lr:3.6e-05 updt_s:0.221 data_s:0.100 +INFO 2025-05-20 12:01:01 ts/train.py:232 step:529K smpl:8M ep:40K epch:133.53 loss:0.004 grdn:0.153 lr:3.6e-05 updt_s:0.221 data_s:0.098 +INFO 2025-05-20 12:02:05 ts/train.py:232 step:530K smpl:8M ep:40K epch:133.58 loss:0.004 grdn:0.151 lr:3.6e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-20 12:02:55 ts/train.py:232 step:530K smpl:8M ep:40K epch:133.63 loss:0.004 grdn:0.156 lr:3.6e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 12:03:44 ts/train.py:232 step:530K smpl:8M ep:40K epch:133.68 loss:0.004 grdn:0.156 lr:3.6e-05 updt_s:0.221 data_s:0.022 +INFO 2025-05-20 12:04:34 ts/train.py:232 step:530K smpl:8M ep:40K epch:133.73 loss:0.004 grdn:0.158 lr:3.6e-05 updt_s:0.222 data_s:0.025 +INFO 2025-05-20 12:05:23 ts/train.py:232 step:530K smpl:8M ep:40K epch:133.78 loss:0.003 grdn:0.146 lr:3.6e-05 updt_s:0.221 data_s:0.023 +INFO 2025-05-20 12:06:12 ts/train.py:232 step:531K smpl:8M ep:40K epch:133.83 loss:0.004 grdn:0.159 lr:3.6e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 12:07:02 ts/train.py:232 step:531K smpl:8M ep:40K epch:133.88 loss:0.004 grdn:0.153 lr:3.6e-05 updt_s:0.221 data_s:0.024 +INFO 2025-05-20 12:07:51 ts/train.py:232 step:531K smpl:8M ep:40K epch:133.93 loss:0.004 grdn:0.139 lr:3.6e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 12:08:40 ts/train.py:232 step:531K smpl:8M ep:40K epch:133.98 loss:0.004 grdn:0.161 lr:3.6e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 12:09:30 ts/train.py:232 step:531K smpl:9M ep:40K epch:134.03 loss:0.004 grdn:0.149 lr:3.6e-05 updt_s:0.221 data_s:0.027 +INFO 2025-05-20 12:10:20 ts/train.py:232 step:532K smpl:9M ep:40K epch:134.08 loss:0.004 grdn:0.152 lr:3.6e-05 updt_s:0.221 data_s:0.025 +INFO 2025-05-20 12:11:09 ts/train.py:232 step:532K smpl:9M ep:40K epch:134.13 loss:0.004 grdn:0.156 lr:3.6e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 12:12:00 ts/train.py:232 step:532K smpl:9M ep:40K epch:134.18 loss:0.004 grdn:0.147 lr:3.6e-05 updt_s:0.221 data_s:0.029 +INFO 2025-05-20 12:12:49 ts/train.py:232 step:532K smpl:9M ep:40K epch:134.24 loss:0.004 grdn:0.155 lr:3.6e-05 updt_s:0.221 data_s:0.026 +INFO 2025-05-20 12:13:55 ts/train.py:232 step:532K smpl:9M ep:40K epch:134.29 loss:0.004 grdn:0.156 lr:3.6e-05 updt_s:0.220 data_s:0.108 +INFO 2025-05-20 12:14:58 ts/train.py:232 step:533K smpl:9M ep:40K epch:134.34 loss:0.004 grdn:0.162 lr:3.6e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 12:16:00 ts/train.py:232 step:533K smpl:9M ep:40K epch:134.39 loss:0.004 grdn:0.151 lr:3.6e-05 updt_s:0.221 data_s:0.092 +INFO 2025-05-20 12:17:04 ts/train.py:232 step:533K smpl:9M ep:40K epch:134.44 loss:0.004 grdn:0.168 lr:3.6e-05 updt_s:0.220 data_s:0.095 +INFO 2025-05-20 12:18:07 ts/train.py:232 step:533K smpl:9M ep:40K epch:134.49 loss:0.004 grdn:0.158 lr:3.6e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 12:19:10 ts/train.py:232 step:533K smpl:9M ep:40K epch:134.54 loss:0.003 grdn:0.142 lr:3.6e-05 updt_s:0.221 data_s:0.093 +INFO 2025-05-20 12:20:05 ts/train.py:232 step:534K smpl:9M ep:40K epch:134.59 loss:0.003 grdn:0.141 lr:3.6e-05 updt_s:0.221 data_s:0.053 +INFO 2025-05-20 12:20:57 ts/train.py:232 step:534K smpl:9M ep:40K epch:134.64 loss:0.004 grdn:0.158 lr:3.6e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 12:21:48 ts/train.py:232 step:534K smpl:9M ep:40K epch:134.69 loss:0.003 grdn:0.146 lr:3.6e-05 updt_s:0.221 data_s:0.033 +INFO 2025-05-20 12:22:39 ts/train.py:232 step:534K smpl:9M ep:40K epch:134.74 loss:0.004 grdn:0.152 lr:3.6e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 12:23:30 ts/train.py:232 step:534K smpl:9M ep:40K epch:134.79 loss:0.004 grdn:0.161 lr:3.6e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 12:24:21 ts/train.py:232 step:535K smpl:9M ep:40K epch:134.84 loss:0.003 grdn:0.147 lr:3.5e-05 updt_s:0.221 data_s:0.032 +INFO 2025-05-20 12:25:14 ts/train.py:232 step:535K smpl:9M ep:40K epch:134.89 loss:0.003 grdn:0.144 lr:3.5e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 12:26:05 ts/train.py:232 step:535K smpl:9M ep:40K epch:134.94 loss:0.004 grdn:0.165 lr:3.5e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 12:26:58 ts/train.py:232 step:535K smpl:9M ep:40K epch:134.99 loss:0.004 grdn:0.159 lr:3.5e-05 updt_s:0.221 data_s:0.045 +INFO 2025-05-20 12:27:50 ts/train.py:232 step:535K smpl:9M ep:41K epch:135.04 loss:0.004 grdn:0.170 lr:3.5e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-20 12:28:43 ts/train.py:232 step:536K smpl:9M ep:41K epch:135.09 loss:0.004 grdn:0.146 lr:3.5e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 12:29:37 ts/train.py:232 step:536K smpl:9M ep:41K epch:135.14 loss:0.004 grdn:0.160 lr:3.5e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 12:30:30 ts/train.py:232 step:536K smpl:9M ep:41K epch:135.19 loss:0.004 grdn:0.166 lr:3.5e-05 updt_s:0.221 data_s:0.048 +INFO 2025-05-20 12:31:34 ts/train.py:232 step:536K smpl:9M ep:41K epch:135.24 loss:0.003 grdn:0.137 lr:3.5e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 12:32:37 ts/train.py:232 step:536K smpl:9M ep:41K epch:135.29 loss:0.004 grdn:0.150 lr:3.5e-05 updt_s:0.220 data_s:0.097 +INFO 2025-05-20 12:33:40 ts/train.py:232 step:537K smpl:9M ep:41K epch:135.34 loss:0.003 grdn:0.140 lr:3.5e-05 updt_s:0.220 data_s:0.094 +INFO 2025-05-20 12:34:44 ts/train.py:232 step:537K smpl:9M ep:41K epch:135.40 loss:0.003 grdn:0.150 lr:3.5e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 12:35:45 ts/train.py:232 step:537K smpl:9M ep:41K epch:135.45 loss:0.004 grdn:0.159 lr:3.5e-05 updt_s:0.220 data_s:0.088 +INFO 2025-05-20 12:36:48 ts/train.py:232 step:537K smpl:9M ep:41K epch:135.50 loss:0.004 grdn:0.160 lr:3.5e-05 updt_s:0.220 data_s:0.090 +INFO 2025-05-20 12:37:43 ts/train.py:232 step:537K smpl:9M ep:41K epch:135.55 loss:0.004 grdn:0.157 lr:3.5e-05 updt_s:0.220 data_s:0.056 +INFO 2025-05-20 12:38:35 ts/train.py:232 step:538K smpl:9M ep:41K epch:135.60 loss:0.004 grdn:0.157 lr:3.5e-05 updt_s:0.221 data_s:0.040 +INFO 2025-05-20 12:39:29 ts/train.py:232 step:538K smpl:9M ep:41K epch:135.65 loss:0.004 grdn:0.159 lr:3.5e-05 updt_s:0.221 data_s:0.046 +INFO 2025-05-20 12:40:20 ts/train.py:232 step:538K smpl:9M ep:41K epch:135.70 loss:0.004 grdn:0.146 lr:3.5e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 12:41:11 ts/train.py:232 step:538K smpl:9M ep:41K epch:135.75 loss:0.004 grdn:0.151 lr:3.5e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 12:42:03 ts/train.py:232 step:538K smpl:9M ep:41K epch:135.80 loss:0.004 grdn:0.156 lr:3.5e-05 updt_s:0.220 data_s:0.039 +INFO 2025-05-20 12:42:55 ts/train.py:232 step:539K smpl:9M ep:41K epch:135.85 loss:0.004 grdn:0.162 lr:3.5e-05 updt_s:0.220 data_s:0.036 +INFO 2025-05-20 12:43:45 ts/train.py:232 step:539K smpl:9M ep:41K epch:135.90 loss:0.004 grdn:0.156 lr:3.5e-05 updt_s:0.221 data_s:0.031 +INFO 2025-05-20 12:44:36 ts/train.py:232 step:539K smpl:9M ep:41K epch:135.95 loss:0.004 grdn:0.147 lr:3.5e-05 updt_s:0.220 data_s:0.034 +INFO 2025-05-20 12:45:27 ts/train.py:232 step:539K smpl:9M ep:41K epch:136.00 loss:0.004 grdn:0.162 lr:3.5e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 12:46:17 ts/train.py:232 step:539K smpl:9M ep:41K epch:136.05 loss:0.004 grdn:0.154 lr:3.5e-05 updt_s:0.221 data_s:0.028 +INFO 2025-05-20 12:47:08 ts/train.py:232 step:540K smpl:9M ep:41K epch:136.10 loss:0.004 grdn:0.164 lr:3.5e-05 updt_s:0.220 data_s:0.034 +INFO 2025-05-20 12:48:00 ts/train.py:232 step:540K smpl:9M ep:41K epch:136.15 loss:0.004 grdn:0.168 lr:3.5e-05 updt_s:0.221 data_s:0.037 +INFO 2025-05-20 12:49:01 ts/train.py:232 step:540K smpl:9M ep:41K epch:136.20 loss:0.003 grdn:0.139 lr:3.5e-05 updt_s:0.220 data_s:0.086 +INFO 2025-05-20 12:49:01 ts/train.py:241 Checkpoint policy after step 540000 +INFO 2025-05-20 12:50:09 ts/train.py:232 step:540K smpl:9M ep:41K epch:136.25 loss:0.004 grdn:0.162 lr:3.5e-05 updt_s:0.220 data_s:0.096 +INFO 2025-05-20 12:51:11 ts/train.py:232 step:540K smpl:9M ep:41K epch:136.30 loss:0.004 grdn:0.154 lr:3.5e-05 updt_s:0.220 data_s:0.093 +INFO 2025-05-20 12:52:14 ts/train.py:232 step:541K smpl:9M ep:41K epch:136.35 loss:0.004 grdn:0.149 lr:3.4e-05 updt_s:0.221 data_s:0.090 +INFO 2025-05-20 12:53:16 ts/train.py:232 step:541K smpl:9M ep:41K epch:136.40 loss:0.004 grdn:0.152 lr:3.4e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-20 12:54:18 ts/train.py:232 step:541K smpl:9M ep:41K epch:136.45 loss:0.004 grdn:0.154 lr:3.4e-05 updt_s:0.221 data_s:0.089 +INFO 2025-05-20 12:55:16 ts/train.py:232 step:541K smpl:9M ep:41K epch:136.51 loss:0.004 grdn:0.153 lr:3.4e-05 updt_s:0.221 data_s:0.067 +INFO 2025-05-20 12:56:07 ts/train.py:232 step:541K smpl:9M ep:41K epch:136.56 loss:0.004 grdn:0.152 lr:3.4e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 12:57:00 ts/train.py:232 step:542K smpl:9M ep:41K epch:136.61 loss:0.003 grdn:0.149 lr:3.4e-05 updt_s:0.221 data_s:0.041 +INFO 2025-05-20 12:57:53 ts/train.py:232 step:542K smpl:9M ep:41K epch:136.66 loss:0.004 grdn:0.154 lr:3.4e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 12:58:45 ts/train.py:232 step:542K smpl:9M ep:41K epch:136.71 loss:0.004 grdn:0.157 lr:3.4e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-20 12:59:36 ts/train.py:232 step:542K smpl:9M ep:41K epch:136.76 loss:0.004 grdn:0.153 lr:3.4e-05 updt_s:0.221 data_s:0.036 +INFO 2025-05-20 13:00:29 ts/train.py:232 step:542K smpl:9M ep:41K epch:136.81 loss:0.003 grdn:0.144 lr:3.4e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 13:01:22 ts/train.py:232 step:543K smpl:9M ep:41K epch:136.86 loss:0.004 grdn:0.153 lr:3.4e-05 updt_s:0.221 data_s:0.044 +INFO 2025-05-20 13:02:18 ts/train.py:232 step:543K smpl:9M ep:41K epch:136.91 loss:0.003 grdn:0.145 lr:3.4e-05 updt_s:0.221 data_s:0.056 +INFO 2025-05-20 13:03:10 ts/train.py:232 step:543K smpl:9M ep:41K epch:136.96 loss:0.004 grdn:0.158 lr:3.4e-05 updt_s:0.221 data_s:0.039 +INFO 2025-05-20 13:04:04 ts/train.py:232 step:543K smpl:9M ep:41K epch:137.01 loss:0.004 grdn:0.157 lr:3.4e-05 updt_s:0.221 data_s:0.051 +INFO 2025-05-20 13:04:57 ts/train.py:232 step:543K smpl:9M ep:41K epch:137.06 loss:0.004 grdn:0.163 lr:3.4e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 13:05:52 ts/train.py:232 step:544K smpl:9M ep:41K epch:137.11 loss:0.004 grdn:0.152 lr:3.4e-05 updt_s:0.221 data_s:0.052 +INFO 2025-05-20 13:06:52 ts/train.py:232 step:544K smpl:9M ep:41K epch:137.16 loss:0.004 grdn:0.150 lr:3.4e-05 updt_s:0.220 data_s:0.078 +INFO 2025-05-20 13:07:57 ts/train.py:232 step:544K smpl:9M ep:41K epch:137.21 loss:0.004 grdn:0.158 lr:3.4e-05 updt_s:0.220 data_s:0.105 +INFO 2025-05-20 13:09:01 ts/train.py:232 step:544K smpl:9M ep:41K epch:137.26 loss:0.003 grdn:0.144 lr:3.4e-05 updt_s:0.221 data_s:0.099 +INFO 2025-05-20 13:10:06 ts/train.py:232 step:544K smpl:9M ep:41K epch:137.31 loss:0.004 grdn:0.154 lr:3.4e-05 updt_s:0.221 data_s:0.101 +INFO 2025-05-20 13:11:09 ts/train.py:232 step:545K smpl:9M ep:41K epch:137.36 loss:0.004 grdn:0.148 lr:3.4e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 13:12:13 ts/train.py:232 step:545K smpl:9M ep:41K epch:137.41 loss:0.003 grdn:0.145 lr:3.4e-05 updt_s:0.221 data_s:0.096 +INFO 2025-05-20 13:13:14 ts/train.py:232 step:545K smpl:9M ep:41K epch:137.46 loss:0.004 grdn:0.153 lr:3.4e-05 updt_s:0.221 data_s:0.086 +INFO 2025-05-20 13:14:07 ts/train.py:232 step:545K smpl:9M ep:41K epch:137.51 loss:0.004 grdn:0.172 lr:3.4e-05 updt_s:0.221 data_s:0.044 +INFO 2025-05-20 13:15:00 ts/train.py:232 step:545K smpl:9M ep:41K epch:137.56 loss:0.004 grdn:0.163 lr:3.4e-05 updt_s:0.221 data_s:0.043 +INFO 2025-05-20 13:15:53 ts/train.py:232 step:546K smpl:9M ep:41K epch:137.61 loss:0.004 grdn:0.150 lr:3.4e-05 updt_s:0.221 data_s:0.042 +INFO 2025-05-20 13:16:44 ts/train.py:232 step:546K smpl:9M ep:41K epch:137.67 loss:0.004 grdn:0.155 lr:3.4e-05 updt_s:0.221 data_s:0.035 +INFO 2025-05-20 13:17:36 ts/train.py:232 step:546K smpl:9M ep:41K epch:137.72 loss:0.004 grdn:0.152 lr:3.4e-05 updt_s:0.221 data_s:0.038 +INFO 2025-05-20 13:18:27 ts/train.py:232 step:546K smpl:9M ep:41K epch:137.77 loss:0.004 grdn:0.149 lr:3.4e-05 updt_s:0.221 data_s:0.034 +INFO 2025-05-20 13:19:18 ts/train.py:232 step:546K smpl:9M ep:41K epch:137.82 loss:0.004 grdn:0.152 lr:3.4e-05 updt_s:0.221 data_s:0.034 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/files/requirements.txt b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..68f3c0c612674471eb5bd86cd21f3a339e5e7f24 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/files/requirements.txt @@ -0,0 +1,146 @@ +pyyaml-include==1.4.1 +pillow==11.2.1 +six==1.17.0 +nvidia-cusparse-cu12==12.3.1.170 +packaging==24.2 +torch==2.6.0 +async-timeout==5.0.1 +wcwidth==0.2.13 +nvidia-nvjitlink-cu12==12.4.127 +pip==25.0.1 +requests==2.32.3 +draccus==0.10.0 +python-dateutil==2.9.0.post0 +numba==0.61.2 +antlr4-python3-runtime==4.9.3 +pyzmq==26.4.0 +termcolor==3.0.1 +pyserial==3.5 +pynput==1.8.1 +pfzy==0.3.4 +deepdiff==8.4.2 +h5py==3.13.0 +imageio==2.37.0 +typing-inspect==0.9.0 +imageio-ffmpeg==0.6.0 +multidict==6.4.3 +fasteners==0.19 +nvidia-cufft-cu12==11.2.1.3 +nvidia-cusolver-cu12==11.6.1.9 +triton==3.2.0 +beautifulsoup4==4.13.4 +annotated-types==0.7.0 +llvmlite==0.44.0 +TorchCodec==0.2.1 +hf_transfer==0.1.9 +cffi==1.17.1 +zarr==2.18.3 +opencv-python-headless==4.11.0.86 +tqdm==4.67.1 +protobuf==5.29.4 +av==14.3.0 +click==8.1.8 +pyarrow==19.0.1 +Flask==3.1.0 +zipp==3.21.0 +importlib_metadata==8.6.1 +urllib3==2.4.0 +huggingface-hub==0.30.2 +gitdb==4.0.12 +nvidia-nccl-cu12==2.21.5 +pydantic_core==2.33.1 +MarkupSafe==3.0.2 +blinker==1.9.0 +aiohappyeyeballs==2.6.1 +einops==0.8.1 +feetech-servo-sdk==1.0.0 +omegaconf==2.3.0 +nvidia-cuda-nvrtc-cu12==12.4.127 +gdown==5.2.0 +pycparser==2.22 +pandas==2.2.3 +orderly-set==5.4.0 +certifi==2025.1.31 +torchvision==0.21.0 +sentry-sdk==2.26.1 +dill==0.3.8 +mpmath==1.3.0 +python-xlib==0.33 +mergedeep==1.3.4 +Werkzeug==3.1.3 +prompt_toolkit==3.0.51 +PyYAML==6.0.2 +multiprocess==0.70.16 +toml==0.10.2 +wheel==0.45.1 +platformdirs==4.3.7 +numcodecs==0.13.1 +evdev==1.9.1 +pydantic==2.11.3 +fsspec==2024.12.0 +xxhash==3.5.0 +setproctitle==1.3.5 +propcache==0.3.1 +nvidia-cuda-cupti-cu12==12.4.127 +GitPython==3.1.44 +cloudpickle==3.1.1 +frozenlist==1.6.0 +filelock==3.18.0 +idna==3.10 +lerobot==0.1.0 +Farama-Notifications==0.0.4 +charset-normalizer==3.4.1 +wandb==0.19.9 +tzdata==2025.2 +nvidia-cublas-cu12==12.4.5.8 +pytz==2025.2 +smmap==5.0.2 +networkx==3.4.2 +numpy==2.2.4 +gymnasium==0.29.1 +aiohttp==3.11.16 +psutil==7.0.0 +attrs==25.3.0 +inquirerpy==0.3.4 +nvidia-curand-cu12==10.3.5.147 +nvidia-cudnn-cu12==9.1.0.70 +itsdangerous==2.2.0 +asciitree==0.3.3 +rerun-sdk==0.22.1 +mypy-extensions==1.0.0 +regex==2024.11.6 +aiosignal==1.3.2 +docker-pycreds==0.4.0 +sympy==1.13.1 +PySocks==1.7.1 +yarl==1.20.0 +nvidia-cuda-runtime-cu12==12.4.127 +soupsieve==2.6 +Jinja2==3.1.6 +nvidia-nvtx-cu12==12.4.127 +safetensors==0.5.3 +datasets==3.5.0 +typing_extensions==4.13.2 +diffusers==0.33.1 +pymunk==6.11.1 +nvidia-cusparselt-cu12==0.6.2 +typing-inspection==0.4.0 +setuptools==78.1.0 +cmake==4.0.0 +jsonlines==4.0.0 +zipp==3.19.2 +packaging==24.2 +importlib_metadata==8.0.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +backports.tarfile==1.2.0 +autocommand==2.2.2 +inflect==7.3.1 +typeguard==4.3.0 +more-itertools==10.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +tomli==2.0.1 +jaraco.collections==5.1.0 +platformdirs==4.2.2 +jaraco.text==3.12.1 diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/files/wandb-metadata.json b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..809afa51b1c5c99ac4a27b956e73d58df5c36817 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/files/wandb-metadata.json @@ -0,0 +1,90 @@ +{ + "os": "Linux-5.15.0-139-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.17", + "startedAt": "2025-05-19T15:05:23.416605Z", + "args": [ + "--resume=true", + "--config_path=/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/checkpoints/260000/pretrained_model/train_config.json", + "--save_freq=20000", + "--steps=900000", + "--wandb.run_id=bqr8y636" + ], + "program": "/NAS/LSY/lerobot/lerobot/scripts/train.py", + "codePath": "lerobot/scripts/train.py", + "git": { + "remote": "https://github.com/huggingface/lerobot.git", + "commit": "a445d9c9da6bea99a8972daa4fe1fdd053d711d2" + }, + "root": "/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain", + "host": "91cf0fc7c553", + "executable": "/opt/conda/envs/lerobot/bin/python", + "codePathLocal": "lerobot/scripts/train.py", + "cpu_count": 48, + "cpu_count_logical": 96, + "gpu": "NVIDIA GeForce RTX 3090", + "gpu_count": 8, + "disk": { + "/": { + "total": "1820335460352", + "used": "1158311702528" + } + }, + "memory": { + "total": "253372637184" + }, + "cpu": { + "count": 48, + "countLogical": 96 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.2" +} \ No newline at end of file diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-core.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..3eadf075b9b0cb1d962ada3d48aaefb9138c10ea --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-core.log @@ -0,0 +1,6 @@ +{"time":"2025-05-19T15:05:23.222133236Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpzkxwah_f/port-830750.txt","pid":830750,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-05-19T15:05:23.223825548Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":830750} +{"time":"2025-05-19T15:05:23.223803757Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38643,"Zone":""}} +{"time":"2025-05-19T15:05:23.407162296Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:49462"} +{"time":"2025-05-19T15:05:23.41515606Z","level":"INFO","msg":"handleInformInit: received","streamId":"bqr8y636","id":"127.0.0.1:49462"} +{"time":"2025-05-19T15:05:24.50136861Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bqr8y636","id":"127.0.0.1:49462"} diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-internal.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5132a729786bf802307a03b958ef8b718784c2d8 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-internal.log @@ -0,0 +1,332 @@ +{"time":"2025-05-19T15:05:23.415308382Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-core.log"} +{"time":"2025-05-19T15:05:24.501288642Z","level":"INFO","msg":"created new stream","id":"bqr8y636"} +{"time":"2025-05-19T15:05:24.501356248Z","level":"INFO","msg":"stream: started","id":"bqr8y636"} +{"time":"2025-05-19T15:05:24.501413936Z","level":"INFO","msg":"writer: Do: started","stream_id":"bqr8y636"} +{"time":"2025-05-19T15:05:24.501432863Z","level":"INFO","msg":"handler: started","stream_id":"bqr8y636"} +{"time":"2025-05-19T15:05:24.501480145Z","level":"INFO","msg":"sender: started","stream_id":"bqr8y636"} +{"time":"2025-05-19T15:05:27.724845141Z","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-05-19T15:06:36.770264445Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770384152Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770395047Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770404545Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770413974Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770423472Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770431993Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770441421Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:06:36.770449034Z","level":"WARN","msg":"handler: ignoring partial history record","step":260200,"current":267201} +{"time":"2025-05-19T15:07:34.530202527Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530296603Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530307917Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530316438Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530326006Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530335434Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530344374Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530352895Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:07:34.530362323Z","level":"WARN","msg":"handler: ignoring partial history record","step":260400,"current":267201} +{"time":"2025-05-19T15:08:32.290189386Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290241836Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.29025322Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.29026223Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.29027075Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290278293Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290299106Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290308115Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:08:32.290316566Z","level":"WARN","msg":"handler: ignoring partial history record","step":260600,"current":267201} +{"time":"2025-05-19T15:09:29.426161463Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426248484Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426257075Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426265106Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.42627216Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426279703Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426287316Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426295278Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:09:29.426302401Z","level":"WARN","msg":"handler: ignoring partial history record","step":260800,"current":267201} +{"time":"2025-05-19T15:10:26.818317636Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818400397Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818411781Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818421699Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818430289Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818438321Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818448238Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818456759Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:10:26.818467095Z","level":"WARN","msg":"handler: ignoring partial history record","step":261000,"current":267201} +{"time":"2025-05-19T15:11:24.990168864Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990258819Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990269225Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990278165Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990285288Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990292831Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990300444Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990308964Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:11:24.990317904Z","level":"WARN","msg":"handler: ignoring partial history record","step":261200,"current":267201} +{"time":"2025-05-19T15:12:23.562134693Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562210401Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562220318Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562228839Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562237359Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562246369Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562253911Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562262432Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:12:23.562271022Z","level":"WARN","msg":"handler: ignoring partial history record","step":261400,"current":267201} +{"time":"2025-05-19T15:13:21.202152787Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.20220866Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202219904Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202228425Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202236456Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202244558Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202253986Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202262018Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:13:21.202271516Z","level":"WARN","msg":"handler: ignoring partial history record","step":261600,"current":267201} +{"time":"2025-05-19T15:14:19.398158706Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398228268Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398239652Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.39824908Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398258998Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398268007Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398276039Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398285956Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:14:19.398294477Z","level":"WARN","msg":"handler: ignoring partial history record","step":261800,"current":267201} +{"time":"2025-05-19T15:15:17.182353839Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.182497711Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.182647589Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.182779588Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.182927231Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.18310323Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.18322573Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.183392789Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:15:17.183490706Z","level":"WARN","msg":"handler: ignoring partial history record","step":262000,"current":267201} +{"time":"2025-05-19T15:16:15.190403151Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190677555Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190761783Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190775053Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190783992Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190840773Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.190926956Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.191029552Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:16:15.191127539Z","level":"WARN","msg":"handler: ignoring partial history record","step":262200,"current":267201} +{"time":"2025-05-19T15:17:12.25419108Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254285645Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254297098Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254306946Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254315048Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254325384Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254334393Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254343822Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:17:12.254352831Z","level":"WARN","msg":"handler: ignoring partial history record","step":262400,"current":267201} +{"time":"2025-05-19T15:18:10.326147235Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326205901Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326215819Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326225806Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326235304Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326243336Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326266034Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326275952Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:18:10.326283564Z","level":"WARN","msg":"handler: ignoring partial history record","step":262600,"current":267201} +{"time":"2025-05-19T15:19:07.598555156Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.59867626Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598688551Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598699866Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598710342Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598720329Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598733529Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598744354Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:19:07.598758113Z","level":"WARN","msg":"handler: ignoring partial history record","step":262800,"current":267201} +{"time":"2025-05-19T15:20:05.758124648Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758190857Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758201822Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758209854Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758219771Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758229269Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758238698Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758247149Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:20:05.758256158Z","level":"WARN","msg":"handler: ignoring partial history record","step":263000,"current":267201} +{"time":"2025-05-19T15:21:03.298179167Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298248239Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298260043Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298269471Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.29827848Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298286512Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298294544Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298303064Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:21:03.298310677Z","level":"WARN","msg":"handler: ignoring partial history record","step":263200,"current":267201} +{"time":"2025-05-19T15:22:01.658324187Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658405622Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658415958Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658424968Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658433488Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658442498Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658451018Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658459469Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:01.658466593Z","level":"WARN","msg":"handler: ignoring partial history record","step":263400,"current":267201} +{"time":"2025-05-19T15:22:59.490190645Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490269146Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490277178Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490284232Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490291356Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490298898Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490306022Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490312238Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:22:59.490317895Z","level":"WARN","msg":"handler: ignoring partial history record","step":263600,"current":267201} +{"time":"2025-05-19T15:23:57.338177152Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338214028Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338225412Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.33823491Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338244269Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338254186Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338262288Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338271716Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:23:57.338280726Z","level":"WARN","msg":"handler: ignoring partial history record","step":263800,"current":267201} +{"time":"2025-05-19T15:25:00.678187256Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678278538Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678289503Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678315065Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678324494Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678333922Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678342932Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678350963Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:00.678358995Z","level":"WARN","msg":"handler: ignoring partial history record","step":264000,"current":267201} +{"time":"2025-05-19T15:25:59.510224168Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.51031119Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510321107Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510329209Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510336752Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.51034625Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510354212Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510362313Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:25:59.510370834Z","level":"WARN","msg":"handler: ignoring partial history record","step":264200,"current":267201} +{"time":"2025-05-19T15:26:58.638271252Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638333619Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638345073Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638354991Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638365327Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638376781Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638387187Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638399409Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:26:58.638408908Z","level":"WARN","msg":"handler: ignoring partial history record","step":264400,"current":267201} +{"time":"2025-05-19T15:27:58.754161763Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754256886Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754270156Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754280562Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754289571Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754299Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754324981Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754334479Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:27:58.754344466Z","level":"WARN","msg":"handler: ignoring partial history record","step":264600,"current":267201} +{"time":"2025-05-19T15:28:58.886349234Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886479837Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886493595Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886504001Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886515385Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886526211Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886537106Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886547023Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:28:58.886557011Z","level":"WARN","msg":"handler: ignoring partial history record","step":264800,"current":267201} +{"time":"2025-05-19T15:29:58.614261493Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614329588Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614342369Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614354172Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614363601Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614373099Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614382527Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614392026Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:29:58.614402432Z","level":"WARN","msg":"handler: ignoring partial history record","step":265000,"current":267201} +{"time":"2025-05-19T15:30:59.878202327Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878285088Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878294517Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878303526Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878311558Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878319589Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.87832811Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.87833712Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:30:59.878360307Z","level":"WARN","msg":"handler: ignoring partial history record","step":265200,"current":267201} +{"time":"2025-05-19T15:32:00.950145719Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950218633Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950229528Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950240005Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950249433Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950259281Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950266893Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.950276811Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:32:00.95028582Z","level":"WARN","msg":"handler: ignoring partial history record","step":265400,"current":267201} +{"time":"2025-05-19T15:33:03.554554311Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554631415Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554642799Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554652717Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554663123Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554672133Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554682539Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554692945Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:33:03.554702374Z","level":"WARN","msg":"handler: ignoring partial history record","step":265600,"current":267201} +{"time":"2025-05-19T15:34:04.774330814Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774515263Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774529022Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774536635Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.7745432Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774552698Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774561708Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.77456925Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:34:04.774577282Z","level":"WARN","msg":"handler: ignoring partial history record","step":265800,"current":267201} +{"time":"2025-05-19T15:35:07.758300095Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758414145Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.75844725Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758456678Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758465688Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758474208Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.75848224Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758490761Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:35:07.758498792Z","level":"WARN","msg":"handler: ignoring partial history record","step":266000,"current":267201} +{"time":"2025-05-19T15:36:10.370165061Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370204801Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370215696Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370225124Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370234134Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370244051Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370252572Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370261511Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:36:10.370271988Z","level":"WARN","msg":"handler: ignoring partial history record","step":266200,"current":267201} +{"time":"2025-05-19T15:37:11.914254057Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914356234Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914368107Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914378024Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914387942Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914396462Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914407357Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914415878Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:37:11.914425865Z","level":"WARN","msg":"handler: ignoring partial history record","step":266400,"current":267201} +{"time":"2025-05-19T15:38:14.480437609Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.480539437Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.480594262Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.480806158Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.4808232Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.480915948Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.481007719Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.481098582Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:38:14.481204111Z","level":"WARN","msg":"handler: ignoring partial history record","step":266600,"current":267201} +{"time":"2025-05-19T15:39:15.225596693Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.225690768Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.225947712Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226019578Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226030055Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226059388Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226135095Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226243907Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:39:15.226326668Z","level":"WARN","msg":"handler: ignoring partial history record","step":266800,"current":267201} +{"time":"2025-05-19T15:40:16.907359295Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907479421Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907536621Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907647388Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.90773874Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907825272Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.907921792Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.908025436Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:40:16.908112947Z","level":"WARN","msg":"handler: ignoring partial history record","step":267000,"current":267201} +{"time":"2025-05-19T15:41:17.796835319Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.796921991Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.796964035Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797079552Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797166504Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797258344Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797337823Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797455574Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-19T15:41:17.797553072Z","level":"WARN","msg":"handler: ignoring partial history record","step":267200,"current":267201} +{"time":"2025-05-20T05:23:13.729960737Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"} diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug.log b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7ffb1fee70422a673daf2fd9fd37b522e8c997fc --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug.log @@ -0,0 +1,23 @@ +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9 +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Configure stats pid to 830750 +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Loading settings from /NAS/LSY/lerobot/wandb/settings +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_setup.py:_flush():67] Loading settings from environment variables +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug.log +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/logs/debug-internal.log +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:init():781] calling init triggers +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:init():786] wandb.init called with sweep_config: {} +config: {'dataset': {'repo_id': 'shylee/so100_pengripG', 'root': '/SSD/LSY/lerobot_dataset/pengripG', 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'torchcodec'}, 'env': None, 'policy': {'type': 'diffusion', 'n_obs_steps': 2, 'normalization_mapping': {'VISUAL': , 'STATE': , 'ACTION': }, 'input_features': {'observation.state': {'type': , 'shape': [6]}, 'observation.images.FrontCam': {'type': , 'shape': [3, 480, 640]}, 'observation.images.TopCam': {'type': , 'shape': [3, 480, 640]}, 'observation.images.WristCam': {'type': , 'shape': [3, 480, 640]}}, 'output_features': {'action': {'type': , 'shape': [6]}}, 'device': 'cuda', 'use_amp': False, 'horizon': 16, 'n_action_steps': 8, 'drop_n_last_frames': 7, 'vision_backbone': 'resnet50', 'crop_shape': [224, 224], 'crop_is_random': False, 'pretrained_backbone_weights': 'ResNet50_Weights.IMAGENET1K_V1', 'use_group_norm': False, 'spatial_softmax_num_keypoints': 32, 'use_separate_rgb_encoder_per_camera': False, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'num_train_timesteps': 100, 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'optimizer_lr': 0.0001, 'optimizer_betas': [0.95, 0.999], 'optimizer_eps': 1e-08, 'optimizer_weight_decay': 1e-06, 'scheduler_name': 'cosine', 'scheduler_warmup_steps': 500, 'pre_resize_shape': [224, 224], 'freeze_vision_backbone': False, 'use_ac': False, 'ac_threshold': 0.99, 'use_pcg': False, 'pcg_weight': 0.1, 'use_bid': False, 'bid_batch_size': 15, 'bid_mode_size': 3, 'bid_temporal_coherence': 0.5, 'weak_model_path': None, 'weak_model_config': None}, 'output_dir': '/SSD/LSY/lerobot_model/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain', 'job_name': 'DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain', 'resume': True, 'seed': 1000, 'num_workers': 4, 'batch_size': 16, 'steps': 900000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 20000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adam', 'lr': 0.0001, 'weight_decay': 1e-06, 'grad_clip_norm': 10.0, 'betas': [0.95, 0.999], 'eps': 1e-08}, 'scheduler': {'type': 'diffuser', 'num_warmup_steps': 500, 'name': 'cosine'}, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': 'bqr8y636', 'mode': None}, '_wandb': {}} +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:init():809] starting backend +2025-05-19 15:05:23,409 INFO MainThread:830750 [wandb_init.py:init():813] sending inform_init request +2025-05-19 15:05:23,413 INFO MainThread:830750 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-05-19 15:05:23,413 INFO MainThread:830750 [wandb_init.py:init():823] backend started and connected +2025-05-19 15:05:23,418 INFO MainThread:830750 [wandb_init.py:init():915] updated telemetry +2025-05-19 15:05:26,076 INFO MainThread:830750 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout +2025-05-19 15:05:27,428 INFO MainThread:830750 [wandb_init.py:init():1009] run resumed +2025-05-19 15:05:27,431 INFO MainThread:830750 [wandb_init.py:init():1014] starting run threads in backend +2025-05-19 15:05:27,834 INFO MainThread:830750 [wandb_run.py:_console_start():2454] atexit reg +2025-05-19 15:05:27,834 INFO MainThread:830750 [wandb_run.py:_redirect():2306] redirect: wrap_raw +2025-05-19 15:05:27,834 INFO MainThread:830750 [wandb_run.py:_redirect():2371] Wrapping output streams. +2025-05-19 15:05:27,834 INFO MainThread:830750 [wandb_run.py:_redirect():2394] Redirects installed. +2025-05-19 15:05:27,836 INFO MainThread:830750 [wandb_init.py:init():1056] run started, returning control to user process diff --git a/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/run-bqr8y636.wandb b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/run-bqr8y636.wandb new file mode 100644 index 0000000000000000000000000000000000000000..79090ec8aad8b26c86194b7ab15480d42ac6e262 --- /dev/null +++ b/DP_pengripG_input224_freeze0_horizon16_transform0_1e-4_longtrain/wandb/run-20250519_150523-bqr8y636/run-bqr8y636.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:594e30d3fa7acb834fc56e109ee29452a092aac080d5176cc79bb11cf567023f +size 17989632