CombatRoad commited on
Commit
56db43f
·
verified ·
1 Parent(s): 2d4d659

Re-upload all files from training folder

Browse files
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ wandb/run-20250615_011116-h5jstcmg/run-h5jstcmg.wandb filter=lfs diff=lfs merge=lfs -text
wandb/debug-internal.log ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2025-06-15T01:11:16.672938296Z","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug-core.log"}
2
+ {"time":"2025-06-15T01:11:16.817564688Z","level":"INFO","msg":"created new stream","id":"h5jstcmg"}
3
+ {"time":"2025-06-15T01:11:16.817593605Z","level":"INFO","msg":"stream: started","id":"h5jstcmg"}
4
+ {"time":"2025-06-15T01:11:16.817622363Z","level":"INFO","msg":"writer: Do: started","stream_id":"h5jstcmg"}
5
+ {"time":"2025-06-15T01:11:16.817662835Z","level":"INFO","msg":"handler: started","stream_id":"h5jstcmg"}
6
+ {"time":"2025-06-15T01:11:16.817664338Z","level":"INFO","msg":"sender: started","stream_id":"h5jstcmg"}
7
+ {"time":"2025-06-15T01:11:16.908550251Z","level":"INFO","msg":"Starting system monitor"}
8
+ {"time":"2025-06-15T04:03:59.296805009Z","level":"INFO","msg":"stream: closing","id":"h5jstcmg"}
9
+ {"time":"2025-06-15T04:03:59.296861951Z","level":"INFO","msg":"Stopping system monitor"}
10
+ {"time":"2025-06-15T04:03:59.296897991Z","level":"INFO","msg":"Stopped system monitor"}
11
+ {"time":"2025-06-15T04:04:00.683232437Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
12
+ {"time":"2025-06-15T04:04:00.747268558Z","level":"INFO","msg":"handler: closed","stream_id":"h5jstcmg"}
13
+ {"time":"2025-06-15T04:04:00.74729338Z","level":"INFO","msg":"writer: Close: closed","stream_id":"h5jstcmg"}
14
+ {"time":"2025-06-15T04:04:00.747309168Z","level":"INFO","msg":"sender: closed","stream_id":"h5jstcmg"}
15
+ {"time":"2025-06-15T04:04:00.747351595Z","level":"INFO","msg":"stream: closed","id":"h5jstcmg"}
wandb/debug.log ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
2
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Configure stats pid to 4025658
3
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
4
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from /content/lerobot/wandb/settings
5
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from environment variables
6
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_init.py:setup_run_log_directory():724] Logging user logs to outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug.log
7
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug-internal.log
8
+ 2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():852] calling init triggers
9
+ 2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
10
+ config: {'dataset': {'repo_id': 'CombatRoad/grab-test-2', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 100, 'n_action_steps': 100, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/eval_act_grab-test-2', 'job_name': 'eval_act_grab-test-2', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 40000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 2000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
11
+ 2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():893] starting backend
12
+ 2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():897] sending inform_init request
13
+ 2025-06-15 01:11:16,671 INFO MainThread:4025658 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
14
+ 2025-06-15 01:11:16,671 INFO MainThread:4025658 [wandb_init.py:init():907] backend started and connected
15
+ 2025-06-15 01:11:16,673 INFO MainThread:4025658 [wandb_init.py:init():1005] updated telemetry
16
+ 2025-06-15 01:11:16,678 INFO MainThread:4025658 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
17
+ 2025-06-15 01:11:16,906 INFO MainThread:4025658 [wandb_init.py:init():1104] starting run threads in backend
18
+ 2025-06-15 01:11:17,324 INFO MainThread:4025658 [wandb_run.py:_console_start():2573] atexit reg
19
+ 2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2421] redirect: wrap_raw
20
+ 2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2490] Wrapping output streams.
21
+ 2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2513] Redirects installed.
22
+ 2025-06-15 01:11:17,326 INFO MainThread:4025658 [wandb_init.py:init():1150] run started, returning control to user process
23
+ 2025-06-15 04:03:59,296 INFO MsgRouterThr:4025658 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.
wandb/run-20250615_011116-h5jstcmg/files/output.log ADDED
@@ -0,0 +1,245 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Logs will be synced with wandb.
2
+ INFO 2025-06-15 01:11:17 db_utils.py:103 Track this run --> https://wandb.ai/combatroad-keimyung-university/lerobot/runs/h5jstcmg
3
+ INFO 2025-06-15 01:11:17 ts/train.py:127 Creating dataset
4
+ tasks.jsonl: 100% 43.0/43.0 [00:00<00:00, 259kB/s]
5
+ info.json: 100% 3.35k/3.35k [00:00<00:00, 13.4MB/s]
6
+ episodes.jsonl: 100% 64.0/64.0 [00:00<00:00, 453kB/s]
7
+ episodes_stats.jsonl: 100% 2.19k/2.19k [00:00<00:00, 11.3MB/s]
8
+ Fetching 4 files: 100% 4/4 [00:00<00:00, 23.87it/s]s]
9
+ README.md: 100% 3.87k/3.87k [00:00<00:00, 19.5MB/s]
10
+ .gitattributes: 100% 2.46k/2.46k [00:00<00:00, 10.0MB/s]
11
+ videos/chunk-000/observation.images.fron(…): 100% 9.17M/9.17M [00:00<00:00, 14.7MB/s]
12
+ data/chunk-000/episode_000000.parquet: 100% 23.7k/23.7k [00:00<00:00, 32.6kB/s]
13
+ videos/chunk-000/observation.images.side(…): 100% 9.55M/9.55M [00:01<00:00, 7.93MB/s]
14
+ Fetching 9 files: 100% 9/9 [00:01<00:00, 6.74it/s].17M/9.17M [00:00<00:00, 14.8MB/s]
15
+ Generating train split: 425 examples [00:00, 66551.40 examples/s]:01<00:00, 7.96MB/s]
16
+ INFO 2025-06-15 01:11:19 ts/train.py:138 Creating policy
17
+ INFO 2025-06-15 01:11:20 ts/train.py:144 Creating optimizer and scheduler
18
+ INFO 2025-06-15 01:11:20 ts/train.py:156 Output dir: outputs/train/eval_act_grab-test-2
19
+ INFO 2025-06-15 01:11:20 ts/train.py:159 cfg.steps=40000 (40K)
20
+ INFO 2025-06-15 01:11:20 ts/train.py:160 dataset.num_frames=425 (425)
21
+ INFO 2025-06-15 01:11:20 ts/train.py:161 dataset.num_episodes=1
22
+ INFO 2025-06-15 01:11:20 ts/train.py:162 num_learnable_params=51597190 (52M)
23
+ INFO 2025-06-15 01:11:20 ts/train.py:163 num_total_params=51597238 (52M)
24
+ INFO 2025-06-15 01:11:20 ts/train.py:202 Start offline training on a fixed dataset
25
+ INFO 2025-06-15 01:12:18 ts/train.py:232 step:200 smpl:2K ep:4 epch:3.76 loss:5.636 grdn:146.118 lr:1.0e-05 updt_s:0.262 data_s:0.025
26
+ INFO 2025-06-15 01:13:09 ts/train.py:232 step:400 smpl:3K ep:8 epch:7.53 loss:2.653 grdn:82.455 lr:1.0e-05 updt_s:0.235 data_s:0.023
27
+ INFO 2025-06-15 01:14:01 ts/train.py:232 step:600 smpl:5K ep:11 epch:11.29 loss:2.280 grdn:75.804 lr:1.0e-05 updt_s:0.234 data_s:0.021
28
+ INFO 2025-06-15 01:14:51 ts/train.py:232 step:800 smpl:6K ep:15 epch:15.06 loss:1.965 grdn:69.023 lr:1.0e-05 updt_s:0.236 data_s:0.016
29
+ INFO 2025-06-15 01:15:43 ts/train.py:232 step:1K smpl:8K ep:19 epch:18.82 loss:1.713 grdn:66.089 lr:1.0e-05 updt_s:0.234 data_s:0.022
30
+ INFO 2025-06-15 01:16:35 ts/train.py:232 step:1K smpl:10K ep:23 epch:22.59 loss:1.522 grdn:62.062 lr:1.0e-05 updt_s:0.235 data_s:0.025
31
+ INFO 2025-06-15 01:17:26 ts/train.py:232 step:1K smpl:11K ep:26 epch:26.35 loss:1.362 grdn:59.223 lr:1.0e-05 updt_s:0.236 data_s:0.018
32
+ INFO 2025-06-15 01:18:18 ts/train.py:232 step:2K smpl:13K ep:30 epch:30.12 loss:1.217 grdn:56.104 lr:1.0e-05 updt_s:0.234 data_s:0.022
33
+ INFO 2025-06-15 01:19:09 ts/train.py:232 step:2K smpl:14K ep:34 epch:33.88 loss:1.082 grdn:52.909 lr:1.0e-05 updt_s:0.235 data_s:0.023
34
+ INFO 2025-06-15 01:20:01 ts/train.py:232 step:2K smpl:16K ep:38 epch:37.65 loss:0.979 grdn:49.833 lr:1.0e-05 updt_s:0.233 data_s:0.022
35
+ INFO 2025-06-15 01:20:01 ts/train.py:241 Checkpoint policy after step 2000
36
+ INFO 2025-06-15 01:20:54 ts/train.py:232 step:2K smpl:18K ep:41 epch:41.41 loss:0.870 grdn:46.410 lr:1.0e-05 updt_s:0.237 data_s:0.017
37
+ INFO 2025-06-15 01:21:45 ts/train.py:232 step:2K smpl:19K ep:45 epch:45.18 loss:0.775 grdn:44.919 lr:1.0e-05 updt_s:0.236 data_s:0.022
38
+ INFO 2025-06-15 01:22:37 ts/train.py:232 step:3K smpl:21K ep:49 epch:48.94 loss:0.697 grdn:42.161 lr:1.0e-05 updt_s:0.234 data_s:0.025
39
+ INFO 2025-06-15 01:23:28 ts/train.py:232 step:3K smpl:22K ep:53 epch:52.71 loss:0.620 grdn:39.421 lr:1.0e-05 updt_s:0.236 data_s:0.017
40
+ INFO 2025-06-15 01:24:20 ts/train.py:232 step:3K smpl:24K ep:56 epch:56.47 loss:0.549 grdn:37.159 lr:1.0e-05 updt_s:0.234 data_s:0.024
41
+ INFO 2025-06-15 01:25:12 ts/train.py:232 step:3K smpl:26K ep:60 epch:60.24 loss:0.489 grdn:34.932 lr:1.0e-05 updt_s:0.235 data_s:0.022
42
+ INFO 2025-06-15 01:26:03 ts/train.py:232 step:3K smpl:27K ep:64 epch:64.00 loss:0.439 grdn:32.961 lr:1.0e-05 updt_s:0.236 data_s:0.018
43
+ INFO 2025-06-15 01:26:55 ts/train.py:232 step:4K smpl:29K ep:68 epch:67.76 loss:0.387 grdn:30.758 lr:1.0e-05 updt_s:0.234 data_s:0.024
44
+ INFO 2025-06-15 01:27:47 ts/train.py:232 step:4K smpl:30K ep:72 epch:71.53 loss:0.355 grdn:29.736 lr:1.0e-05 updt_s:0.235 data_s:0.025
45
+ INFO 2025-06-15 01:28:38 ts/train.py:232 step:4K smpl:32K ep:75 epch:75.29 loss:0.318 grdn:28.297 lr:1.0e-05 updt_s:0.233 data_s:0.023
46
+ INFO 2025-06-15 01:28:38 ts/train.py:241 Checkpoint policy after step 4000
47
+ INFO 2025-06-15 01:29:31 ts/train.py:232 step:4K smpl:34K ep:79 epch:79.06 loss:0.289 grdn:26.797 lr:1.0e-05 updt_s:0.237 data_s:0.017
48
+ INFO 2025-06-15 01:30:23 ts/train.py:232 step:4K smpl:35K ep:83 epch:82.82 loss:0.268 grdn:25.509 lr:1.0e-05 updt_s:0.235 data_s:0.023
49
+ INFO 2025-06-15 01:31:15 ts/train.py:232 step:5K smpl:37K ep:87 epch:86.59 loss:0.245 grdn:24.868 lr:1.0e-05 updt_s:0.234 data_s:0.025
50
+ INFO 2025-06-15 01:32:06 ts/train.py:232 step:5K smpl:38K ep:90 epch:90.35 loss:0.228 grdn:23.587 lr:1.0e-05 updt_s:0.236 data_s:0.017
51
+ INFO 2025-06-15 01:32:58 ts/train.py:232 step:5K smpl:40K ep:94 epch:94.12 loss:0.212 grdn:22.247 lr:1.0e-05 updt_s:0.234 data_s:0.024
52
+ INFO 2025-06-15 01:33:50 ts/train.py:232 step:5K smpl:42K ep:98 epch:97.88 loss:0.201 grdn:21.935 lr:1.0e-05 updt_s:0.235 data_s:0.023
53
+ INFO 2025-06-15 01:34:40 ts/train.py:232 step:5K smpl:43K ep:102 epch:101.65 loss:0.189 grdn:21.018 lr:1.0e-05 updt_s:0.236 data_s:0.016
54
+ INFO 2025-06-15 01:35:32 ts/train.py:232 step:6K smpl:45K ep:105 epch:105.41 loss:0.177 grdn:20.272 lr:1.0e-05 updt_s:0.234 data_s:0.022
55
+ INFO 2025-06-15 01:36:24 ts/train.py:232 step:6K smpl:46K ep:109 epch:109.18 loss:0.168 grdn:19.692 lr:1.0e-05 updt_s:0.236 data_s:0.023
56
+ INFO 2025-06-15 01:37:15 ts/train.py:232 step:6K smpl:48K ep:113 epch:112.94 loss:0.162 grdn:19.346 lr:1.0e-05 updt_s:0.234 data_s:0.022
57
+ INFO 2025-06-15 01:37:15 ts/train.py:241 Checkpoint policy after step 6000
58
+ INFO 2025-06-15 01:38:08 ts/train.py:232 step:6K smpl:50K ep:117 epch:116.71 loss:0.153 grdn:18.388 lr:1.0e-05 updt_s:0.236 data_s:0.017
59
+ INFO 2025-06-15 01:38:59 ts/train.py:232 step:6K smpl:51K ep:120 epch:120.47 loss:0.146 grdn:17.518 lr:1.0e-05 updt_s:0.235 data_s:0.021
60
+ INFO 2025-06-15 01:39:51 ts/train.py:232 step:7K smpl:53K ep:124 epch:124.24 loss:0.143 grdn:17.532 lr:1.0e-05 updt_s:0.234 data_s:0.024
61
+ INFO 2025-06-15 01:40:42 ts/train.py:232 step:7K smpl:54K ep:128 epch:128.00 loss:0.134 grdn:16.715 lr:1.0e-05 updt_s:0.236 data_s:0.018
62
+ INFO 2025-06-15 01:41:33 ts/train.py:232 step:7K smpl:56K ep:132 epch:131.76 loss:0.132 grdn:16.883 lr:1.0e-05 updt_s:0.234 data_s:0.022
63
+ INFO 2025-06-15 01:42:25 ts/train.py:232 step:7K smpl:58K ep:136 epch:135.53 loss:0.127 grdn:16.502 lr:1.0e-05 updt_s:0.235 data_s:0.022
64
+ INFO 2025-06-15 01:43:18 ts/train.py:232 step:7K smpl:59K ep:139 epch:139.29 loss:0.121 grdn:15.927 lr:1.0e-05 updt_s:0.235 data_s:0.026
65
+ INFO 2025-06-15 01:44:08 ts/train.py:232 step:8K smpl:61K ep:143 epch:143.06 loss:0.119 grdn:15.558 lr:1.0e-05 updt_s:0.234 data_s:0.016
66
+ INFO 2025-06-15 01:45:00 ts/train.py:232 step:8K smpl:62K ep:147 epch:146.82 loss:0.114 grdn:15.445 lr:1.0e-05 updt_s:0.235 data_s:0.024
67
+ INFO 2025-06-15 01:45:52 ts/train.py:232 step:8K smpl:64K ep:151 epch:150.59 loss:0.113 grdn:15.088 lr:1.0e-05 updt_s:0.234 data_s:0.024
68
+ INFO 2025-06-15 01:45:52 ts/train.py:241 Checkpoint policy after step 8000
69
+ INFO 2025-06-15 01:46:45 ts/train.py:232 step:8K smpl:66K ep:154 epch:154.35 loss:0.109 grdn:14.368 lr:1.0e-05 updt_s:0.236 data_s:0.018
70
+ INFO 2025-06-15 01:47:37 ts/train.py:232 step:8K smpl:67K ep:158 epch:158.12 loss:0.108 grdn:14.542 lr:1.0e-05 updt_s:0.235 data_s:0.025
71
+ INFO 2025-06-15 01:48:28 ts/train.py:232 step:9K smpl:69K ep:162 epch:161.88 loss:0.104 grdn:14.211 lr:1.0e-05 updt_s:0.234 data_s:0.022
72
+ INFO 2025-06-15 01:49:19 ts/train.py:232 step:9K smpl:70K ep:166 epch:165.65 loss:0.102 grdn:13.837 lr:1.0e-05 updt_s:0.236 data_s:0.017
73
+ INFO 2025-06-15 01:50:11 ts/train.py:232 step:9K smpl:72K ep:169 epch:169.41 loss:0.100 grdn:13.779 lr:1.0e-05 updt_s:0.235 data_s:0.023
74
+ INFO 2025-06-15 01:51:02 ts/train.py:232 step:9K smpl:74K ep:173 epch:173.18 loss:0.097 grdn:13.687 lr:1.0e-05 updt_s:0.233 data_s:0.023
75
+ INFO 2025-06-15 01:51:54 ts/train.py:232 step:9K smpl:75K ep:177 epch:176.94 loss:0.095 grdn:13.374 lr:1.0e-05 updt_s:0.235 data_s:0.022
76
+ INFO 2025-06-15 01:52:45 ts/train.py:232 step:10K smpl:77K ep:181 epch:180.71 loss:0.093 grdn:12.952 lr:1.0e-05 updt_s:0.234 data_s:0.017
77
+ INFO 2025-06-15 01:53:36 ts/train.py:232 step:10K smpl:78K ep:184 epch:184.47 loss:0.091 grdn:12.838 lr:1.0e-05 updt_s:0.235 data_s:0.022
78
+ INFO 2025-06-15 01:54:29 ts/train.py:232 step:10K smpl:80K ep:188 epch:188.24 loss:0.088 grdn:12.684 lr:1.0e-05 updt_s:0.235 data_s:0.024
79
+ INFO 2025-06-15 01:54:29 ts/train.py:241 Checkpoint policy after step 10000
80
+ INFO 2025-06-15 01:55:21 ts/train.py:232 step:10K smpl:82K ep:192 epch:192.00 loss:0.087 grdn:12.378 lr:1.0e-05 updt_s:0.235 data_s:0.018
81
+ INFO 2025-06-15 01:56:13 ts/train.py:232 step:10K smpl:83K ep:196 epch:195.76 loss:0.085 grdn:12.049 lr:1.0e-05 updt_s:0.236 data_s:0.022
82
+ INFO 2025-06-15 01:57:04 ts/train.py:232 step:11K smpl:85K ep:200 epch:199.53 loss:0.083 grdn:12.318 lr:1.0e-05 updt_s:0.234 data_s:0.022
83
+ INFO 2025-06-15 01:57:56 ts/train.py:232 step:11K smpl:86K ep:203 epch:203.29 loss:0.083 grdn:12.118 lr:1.0e-05 updt_s:0.235 data_s:0.020
84
+ INFO 2025-06-15 01:58:48 ts/train.py:232 step:11K smpl:88K ep:207 epch:207.06 loss:0.081 grdn:11.818 lr:1.0e-05 updt_s:0.236 data_s:0.024
85
+ INFO 2025-06-15 01:59:39 ts/train.py:232 step:11K smpl:90K ep:211 epch:210.82 loss:0.079 grdn:11.620 lr:1.0e-05 updt_s:0.234 data_s:0.023
86
+ INFO 2025-06-15 02:00:31 ts/train.py:232 step:11K smpl:91K ep:215 epch:214.59 loss:0.078 grdn:11.537 lr:1.0e-05 updt_s:0.235 data_s:0.022
87
+ INFO 2025-06-15 02:01:22 ts/train.py:232 step:12K smpl:93K ep:218 epch:218.35 loss:0.077 grdn:11.357 lr:1.0e-05 updt_s:0.234 data_s:0.018
88
+ INFO 2025-06-15 02:02:14 ts/train.py:232 step:12K smpl:94K ep:222 epch:222.12 loss:0.076 grdn:11.410 lr:1.0e-05 updt_s:0.235 data_s:0.023
89
+ INFO 2025-06-15 02:03:06 ts/train.py:232 step:12K smpl:96K ep:226 epch:225.88 loss:0.074 grdn:10.901 lr:1.0e-05 updt_s:0.235 data_s:0.025
90
+ INFO 2025-06-15 02:03:06 ts/train.py:241 Checkpoint policy after step 12000
91
+ INFO 2025-06-15 02:03:59 ts/train.py:232 step:12K smpl:98K ep:230 epch:229.65 loss:0.073 grdn:10.586 lr:1.0e-05 updt_s:0.234 data_s:0.019
92
+ INFO 2025-06-15 02:04:51 ts/train.py:232 step:12K smpl:99K ep:233 epch:233.41 loss:0.072 grdn:10.887 lr:1.0e-05 updt_s:0.236 data_s:0.023
93
+ INFO 2025-06-15 02:05:42 ts/train.py:232 step:13K smpl:101K ep:237 epch:237.18 loss:0.072 grdn:10.751 lr:1.0e-05 updt_s:0.234 data_s:0.023
94
+ INFO 2025-06-15 02:06:34 ts/train.py:232 step:13K smpl:102K ep:241 epch:240.94 loss:0.069 grdn:10.319 lr:1.0e-05 updt_s:0.236 data_s:0.023
95
+ INFO 2025-06-15 02:07:25 ts/train.py:232 step:13K smpl:104K ep:245 epch:244.71 loss:0.069 grdn:10.191 lr:1.0e-05 updt_s:0.236 data_s:0.017
96
+ INFO 2025-06-15 02:08:17 ts/train.py:232 step:13K smpl:106K ep:248 epch:248.47 loss:0.068 grdn:10.289 lr:1.0e-05 updt_s:0.234 data_s:0.025
97
+ INFO 2025-06-15 02:09:09 ts/train.py:232 step:13K smpl:107K ep:252 epch:252.24 loss:0.067 grdn:10.057 lr:1.0e-05 updt_s:0.235 data_s:0.023
98
+ INFO 2025-06-15 02:10:00 ts/train.py:232 step:14K smpl:109K ep:256 epch:256.00 loss:0.066 grdn:10.003 lr:1.0e-05 updt_s:0.234 data_s:0.018
99
+ INFO 2025-06-15 02:10:52 ts/train.py:232 step:14K smpl:110K ep:260 epch:259.76 loss:0.065 grdn:10.039 lr:1.0e-05 updt_s:0.235 data_s:0.024
100
+ INFO 2025-06-15 02:11:44 ts/train.py:232 step:14K smpl:112K ep:264 epch:263.53 loss:0.064 grdn:9.852 lr:1.0e-05 updt_s:0.236 data_s:0.021
101
+ INFO 2025-06-15 02:11:44 ts/train.py:241 Checkpoint policy after step 14000
102
+ INFO 2025-06-15 02:12:36 ts/train.py:232 step:14K smpl:114K ep:267 epch:267.29 loss:0.063 grdn:9.613 lr:1.0e-05 updt_s:0.235 data_s:0.017
103
+ INFO 2025-06-15 02:13:28 ts/train.py:232 step:14K smpl:115K ep:271 epch:271.06 loss:0.063 grdn:9.216 lr:1.0e-05 updt_s:0.236 data_s:0.023
104
+ INFO 2025-06-15 02:14:20 ts/train.py:232 step:15K smpl:117K ep:275 epch:274.82 loss:0.062 grdn:9.752 lr:1.0e-05 updt_s:0.234 data_s:0.025
105
+ INFO 2025-06-15 02:15:12 ts/train.py:232 step:15K smpl:118K ep:279 epch:278.59 loss:0.061 grdn:9.354 lr:1.0e-05 updt_s:0.235 data_s:0.023
106
+ INFO 2025-06-15 02:16:03 ts/train.py:232 step:15K smpl:120K ep:282 epch:282.35 loss:0.060 grdn:9.343 lr:1.0e-05 updt_s:0.236 data_s:0.017
107
+ INFO 2025-06-15 02:16:54 ts/train.py:232 step:15K smpl:122K ep:286 epch:286.12 loss:0.060 grdn:9.347 lr:1.0e-05 updt_s:0.234 data_s:0.021
108
+ INFO 2025-06-15 02:17:47 ts/train.py:232 step:15K smpl:123K ep:290 epch:289.88 loss:0.059 grdn:9.314 lr:1.0e-05 updt_s:0.235 data_s:0.026
109
+ INFO 2025-06-15 02:18:37 ts/train.py:232 step:16K smpl:125K ep:294 epch:293.65 loss:0.058 grdn:9.231 lr:1.0e-05 updt_s:0.235 data_s:0.016
110
+ INFO 2025-06-15 02:19:29 ts/train.py:232 step:16K smpl:126K ep:297 epch:297.41 loss:0.057 grdn:9.074 lr:1.0e-05 updt_s:0.236 data_s:0.022
111
+ INFO 2025-06-15 02:20:21 ts/train.py:232 step:16K smpl:128K ep:301 epch:301.18 loss:0.057 grdn:8.898 lr:1.0e-05 updt_s:0.235 data_s:0.024
112
+ INFO 2025-06-15 02:20:21 ts/train.py:241 Checkpoint policy after step 16000
113
+ INFO 2025-06-15 02:21:14 ts/train.py:232 step:16K smpl:130K ep:305 epch:304.94 loss:0.057 grdn:8.887 lr:1.0e-05 updt_s:0.234 data_s:0.018
114
+ INFO 2025-06-15 02:22:07 ts/train.py:232 step:16K smpl:131K ep:309 epch:308.71 loss:0.056 grdn:8.590 lr:1.0e-05 updt_s:0.237 data_s:0.025
115
+ INFO 2025-06-15 02:22:58 ts/train.py:232 step:17K smpl:133K ep:312 epch:312.47 loss:0.056 grdn:8.675 lr:1.0e-05 updt_s:0.234 data_s:0.023
116
+ INFO 2025-06-15 02:23:50 ts/train.py:232 step:17K smpl:134K ep:316 epch:316.24 loss:0.054 grdn:8.383 lr:1.0e-05 updt_s:0.235 data_s:0.021
117
+ INFO 2025-06-15 02:24:41 ts/train.py:232 step:17K smpl:136K ep:320 epch:320.00 loss:0.053 grdn:8.116 lr:1.0e-05 updt_s:0.237 data_s:0.016
118
+ INFO 2025-06-15 02:25:33 ts/train.py:232 step:17K smpl:138K ep:324 epch:323.76 loss:0.053 grdn:8.273 lr:1.0e-05 updt_s:0.235 data_s:0.025
119
+ INFO 2025-06-15 02:26:25 ts/train.py:232 step:17K smpl:139K ep:328 epch:327.53 loss:0.053 grdn:8.463 lr:1.0e-05 updt_s:0.235 data_s:0.022
120
+ INFO 2025-06-15 02:27:16 ts/train.py:232 step:18K smpl:141K ep:331 epch:331.29 loss:0.053 grdn:8.071 lr:1.0e-05 updt_s:0.235 data_s:0.017
121
+ INFO 2025-06-15 02:28:07 ts/train.py:232 step:18K smpl:142K ep:335 epch:335.06 loss:0.052 grdn:8.215 lr:1.0e-05 updt_s:0.235 data_s:0.023
122
+ INFO 2025-06-15 02:28:59 ts/train.py:232 step:18K smpl:144K ep:339 epch:338.82 loss:0.052 grdn:8.441 lr:1.0e-05 updt_s:0.236 data_s:0.022
123
+ INFO 2025-06-15 02:28:59 ts/train.py:241 Checkpoint policy after step 18000
124
+ INFO 2025-06-15 02:29:53 ts/train.py:232 step:18K smpl:146K ep:343 epch:342.59 loss:0.051 grdn:7.805 lr:1.0e-05 updt_s:0.234 data_s:0.025
125
+ INFO 2025-06-15 02:30:44 ts/train.py:232 step:18K smpl:147K ep:346 epch:346.35 loss:0.051 grdn:8.160 lr:1.0e-05 updt_s:0.236 data_s:0.016
126
+ INFO 2025-06-15 02:31:35 ts/train.py:232 step:19K smpl:149K ep:350 epch:350.12 loss:0.051 grdn:8.032 lr:1.0e-05 updt_s:0.234 data_s:0.023
127
+ INFO 2025-06-15 02:32:28 ts/train.py:232 step:19K smpl:150K ep:354 epch:353.88 loss:0.049 grdn:7.788 lr:1.0e-05 updt_s:0.236 data_s:0.024
128
+ INFO 2025-06-15 02:33:19 ts/train.py:232 step:19K smpl:152K ep:358 epch:357.65 loss:0.049 grdn:7.815 lr:1.0e-05 updt_s:0.236 data_s:0.018
129
+ INFO 2025-06-15 02:34:10 ts/train.py:232 step:19K smpl:154K ep:361 epch:361.41 loss:0.049 grdn:7.959 lr:1.0e-05 updt_s:0.234 data_s:0.021
130
+ INFO 2025-06-15 02:35:02 ts/train.py:232 step:19K smpl:155K ep:365 epch:365.18 loss:0.048 grdn:7.828 lr:1.0e-05 updt_s:0.235 data_s:0.023
131
+ INFO 2025-06-15 02:35:52 ts/train.py:232 step:20K smpl:157K ep:369 epch:368.94 loss:0.048 grdn:7.767 lr:1.0e-05 updt_s:0.234 data_s:0.018
132
+ INFO 2025-06-15 02:36:44 ts/train.py:232 step:20K smpl:158K ep:373 epch:372.71 loss:0.048 grdn:7.640 lr:1.0e-05 updt_s:0.235 data_s:0.023
133
+ INFO 2025-06-15 02:37:36 ts/train.py:232 step:20K smpl:160K ep:376 epch:376.47 loss:0.047 grdn:7.615 lr:1.0e-05 updt_s:0.235 data_s:0.023
134
+ INFO 2025-06-15 02:37:36 ts/train.py:241 Checkpoint policy after step 20000
135
+ INFO 2025-06-15 02:38:29 ts/train.py:232 step:20K smpl:162K ep:380 epch:380.24 loss:0.047 grdn:7.616 lr:1.0e-05 updt_s:0.234 data_s:0.022
136
+ INFO 2025-06-15 02:39:21 ts/train.py:232 step:20K smpl:163K ep:384 epch:384.00 loss:0.047 grdn:7.348 lr:1.0e-05 updt_s:0.236 data_s:0.019
137
+ INFO 2025-06-15 02:40:12 ts/train.py:232 step:21K smpl:165K ep:388 epch:387.76 loss:0.046 grdn:7.347 lr:1.0e-05 updt_s:0.234 data_s:0.022
138
+ INFO 2025-06-15 02:41:04 ts/train.py:232 step:21K smpl:166K ep:392 epch:391.53 loss:0.046 grdn:7.450 lr:1.0e-05 updt_s:0.235 data_s:0.022
139
+ INFO 2025-06-15 02:41:55 ts/train.py:232 step:21K smpl:168K ep:395 epch:395.29 loss:0.044 grdn:7.100 lr:1.0e-05 updt_s:0.236 data_s:0.017
140
+ INFO 2025-06-15 02:42:46 ts/train.py:232 step:21K smpl:170K ep:399 epch:399.06 loss:0.045 grdn:7.292 lr:1.0e-05 updt_s:0.234 data_s:0.023
141
+ INFO 2025-06-15 02:43:38 ts/train.py:232 step:21K smpl:171K ep:403 epch:402.82 loss:0.045 grdn:7.139 lr:1.0e-05 updt_s:0.235 data_s:0.023
142
+ INFO 2025-06-15 02:44:29 ts/train.py:232 step:22K smpl:173K ep:407 epch:406.59 loss:0.044 grdn:7.002 lr:1.0e-05 updt_s:0.234 data_s:0.016
143
+ INFO 2025-06-15 02:45:21 ts/train.py:232 step:22K smpl:174K ep:410 epch:410.35 loss:0.045 grdn:7.311 lr:1.0e-05 updt_s:0.237 data_s:0.022
144
+ INFO 2025-06-15 02:46:13 ts/train.py:232 step:22K smpl:176K ep:414 epch:414.12 loss:0.044 grdn:7.108 lr:1.0e-05 updt_s:0.235 data_s:0.025
145
+ INFO 2025-06-15 02:46:13 ts/train.py:241 Checkpoint policy after step 22000
146
+ INFO 2025-06-15 02:47:07 ts/train.py:232 step:22K smpl:178K ep:418 epch:417.88 loss:0.044 grdn:7.061 lr:1.0e-05 updt_s:0.234 data_s:0.024
147
+ INFO 2025-06-15 02:47:58 ts/train.py:232 step:22K smpl:179K ep:422 epch:421.65 loss:0.043 grdn:6.996 lr:1.0e-05 updt_s:0.236 data_s:0.018
148
+ INFO 2025-06-15 02:48:50 ts/train.py:232 step:23K smpl:181K ep:425 epch:425.41 loss:0.043 grdn:6.993 lr:1.0e-05 updt_s:0.234 data_s:0.023
149
+ INFO 2025-06-15 02:49:42 ts/train.py:232 step:23K smpl:182K ep:429 epch:429.18 loss:0.042 grdn:6.992 lr:1.0e-05 updt_s:0.236 data_s:0.024
150
+ INFO 2025-06-15 02:50:33 ts/train.py:232 step:23K smpl:184K ep:433 epch:432.94 loss:0.042 grdn:6.749 lr:1.0e-05 updt_s:0.236 data_s:0.018
151
+ INFO 2025-06-15 02:51:25 ts/train.py:232 step:23K smpl:186K ep:437 epch:436.71 loss:0.042 grdn:6.906 lr:1.0e-05 updt_s:0.234 data_s:0.025
152
+ INFO 2025-06-15 02:52:17 ts/train.py:232 step:23K smpl:187K ep:440 epch:440.47 loss:0.042 grdn:6.957 lr:1.0e-05 updt_s:0.235 data_s:0.024
153
+ INFO 2025-06-15 02:53:09 ts/train.py:232 step:24K smpl:189K ep:444 epch:444.24 loss:0.041 grdn:6.609 lr:1.0e-05 updt_s:0.234 data_s:0.025
154
+ INFO 2025-06-15 02:54:00 ts/train.py:232 step:24K smpl:190K ep:448 epch:448.00 loss:0.042 grdn:6.620 lr:1.0e-05 updt_s:0.237 data_s:0.015
155
+ INFO 2025-06-15 02:54:52 ts/train.py:232 step:24K smpl:192K ep:452 epch:451.76 loss:0.041 grdn:6.505 lr:1.0e-05 updt_s:0.235 data_s:0.023
156
+ INFO 2025-06-15 02:54:52 ts/train.py:241 Checkpoint policy after step 24000
157
+ INFO 2025-06-15 02:55:46 ts/train.py:232 step:24K smpl:194K ep:456 epch:455.53 loss:0.040 grdn:6.489 lr:1.0e-05 updt_s:0.234 data_s:0.026
158
+ INFO 2025-06-15 02:56:37 ts/train.py:232 step:24K smpl:195K ep:459 epch:459.29 loss:0.040 grdn:6.752 lr:1.0e-05 updt_s:0.237 data_s:0.019
159
+ INFO 2025-06-15 02:57:29 ts/train.py:232 step:25K smpl:197K ep:463 epch:463.06 loss:0.040 grdn:6.495 lr:1.0e-05 updt_s:0.234 data_s:0.023
160
+ INFO 2025-06-15 02:58:21 ts/train.py:232 step:25K smpl:198K ep:467 epch:466.82 loss:0.040 grdn:6.566 lr:1.0e-05 updt_s:0.236 data_s:0.026
161
+ INFO 2025-06-15 02:59:12 ts/train.py:232 step:25K smpl:200K ep:471 epch:470.59 loss:0.039 grdn:6.202 lr:1.0e-05 updt_s:0.236 data_s:0.017
162
+ INFO 2025-06-15 03:00:04 ts/train.py:232 step:25K smpl:202K ep:474 epch:474.35 loss:0.040 grdn:6.561 lr:1.0e-05 updt_s:0.234 data_s:0.024
163
+ INFO 2025-06-15 03:00:57 ts/train.py:232 step:25K smpl:203K ep:478 epch:478.12 loss:0.040 grdn:6.802 lr:1.0e-05 updt_s:0.236 data_s:0.024
164
+ INFO 2025-06-15 03:01:49 ts/train.py:232 step:26K smpl:205K ep:482 epch:481.88 loss:0.039 grdn:6.358 lr:1.0e-05 updt_s:0.234 data_s:0.026
165
+ INFO 2025-06-15 03:02:40 ts/train.py:232 step:26K smpl:206K ep:486 epch:485.65 loss:0.038 grdn:6.173 lr:1.0e-05 updt_s:0.237 data_s:0.018
166
+ INFO 2025-06-15 03:03:32 ts/train.py:232 step:26K smpl:208K ep:489 epch:489.41 loss:0.038 grdn:6.151 lr:1.0e-05 updt_s:0.235 data_s:0.024
167
+ INFO 2025-06-15 03:03:32 ts/train.py:241 Checkpoint policy after step 26000
168
+ INFO 2025-06-15 03:04:25 ts/train.py:232 step:26K smpl:210K ep:493 epch:493.18 loss:0.038 grdn:6.239 lr:1.0e-05 updt_s:0.234 data_s:0.023
169
+ INFO 2025-06-15 03:05:17 ts/train.py:232 step:26K smpl:211K ep:497 epch:496.94 loss:0.038 grdn:6.299 lr:1.0e-05 updt_s:0.237 data_s:0.019
170
+ INFO 2025-06-15 03:06:08 ts/train.py:232 step:27K smpl:213K ep:501 epch:500.71 loss:0.038 grdn:6.293 lr:1.0e-05 updt_s:0.234 data_s:0.024
171
+ INFO 2025-06-15 03:07:00 ts/train.py:232 step:27K smpl:214K ep:504 epch:504.47 loss:0.037 grdn:6.017 lr:1.0e-05 updt_s:0.235 data_s:0.022
172
+ INFO 2025-06-15 03:07:51 ts/train.py:232 step:27K smpl:216K ep:508 epch:508.24 loss:0.037 grdn:6.112 lr:1.0e-05 updt_s:0.235 data_s:0.019
173
+ INFO 2025-06-15 03:08:43 ts/train.py:232 step:27K smpl:218K ep:512 epch:512.00 loss:0.038 grdn:6.114 lr:1.0e-05 updt_s:0.235 data_s:0.022
174
+ INFO 2025-06-15 03:09:35 ts/train.py:232 step:27K smpl:219K ep:516 epch:515.76 loss:0.037 grdn:5.982 lr:1.0e-05 updt_s:0.236 data_s:0.022
175
+ INFO 2025-06-15 03:10:27 ts/train.py:232 step:28K smpl:221K ep:520 epch:519.53 loss:0.036 grdn:5.900 lr:1.0e-05 updt_s:0.234 data_s:0.024
176
+ INFO 2025-06-15 03:11:18 ts/train.py:232 step:28K smpl:222K ep:523 epch:523.29 loss:0.037 grdn:6.109 lr:1.0e-05 updt_s:0.237 data_s:0.017
177
+ INFO 2025-06-15 03:12:10 ts/train.py:232 step:28K smpl:224K ep:527 epch:527.06 loss:0.036 grdn:6.173 lr:1.0e-05 updt_s:0.235 data_s:0.024
178
+ INFO 2025-06-15 03:12:10 ts/train.py:241 Checkpoint policy after step 28000
179
+ INFO 2025-06-15 03:13:03 ts/train.py:232 step:28K smpl:226K ep:531 epch:530.82 loss:0.036 grdn:5.848 lr:1.0e-05 updt_s:0.234 data_s:0.023
180
+ INFO 2025-06-15 03:13:55 ts/train.py:232 step:28K smpl:227K ep:535 epch:534.59 loss:0.036 grdn:5.773 lr:1.0e-05 updt_s:0.236 data_s:0.019
181
+ INFO 2025-06-15 03:14:47 ts/train.py:232 step:29K smpl:229K ep:538 epch:538.35 loss:0.036 grdn:6.053 lr:1.0e-05 updt_s:0.236 data_s:0.024
182
+ INFO 2025-06-15 03:15:38 ts/train.py:232 step:29K smpl:230K ep:542 epch:542.12 loss:0.036 grdn:6.106 lr:1.0e-05 updt_s:0.233 data_s:0.024
183
+ INFO 2025-06-15 03:16:30 ts/train.py:232 step:29K smpl:232K ep:546 epch:545.88 loss:0.036 grdn:5.920 lr:1.0e-05 updt_s:0.235 data_s:0.021
184
+ INFO 2025-06-15 03:17:21 ts/train.py:232 step:29K smpl:234K ep:550 epch:549.65 loss:0.035 grdn:5.828 lr:1.0e-05 updt_s:0.234 data_s:0.018
185
+ INFO 2025-06-15 03:18:13 ts/train.py:232 step:29K smpl:235K ep:553 epch:553.41 loss:0.035 grdn:5.808 lr:1.0e-05 updt_s:0.235 data_s:0.023
186
+ INFO 2025-06-15 03:19:04 ts/train.py:232 step:30K smpl:237K ep:557 epch:557.18 loss:0.034 grdn:5.820 lr:1.0e-05 updt_s:0.236 data_s:0.022
187
+ INFO 2025-06-15 03:19:55 ts/train.py:232 step:30K smpl:238K ep:561 epch:560.94 loss:0.035 grdn:5.628 lr:1.0e-05 updt_s:0.235 data_s:0.017
188
+ INFO 2025-06-15 03:20:47 ts/train.py:232 step:30K smpl:240K ep:565 epch:564.71 loss:0.035 grdn:5.823 lr:1.0e-05 updt_s:0.236 data_s:0.024
189
+ INFO 2025-06-15 03:20:47 ts/train.py:241 Checkpoint policy after step 30000
190
+ INFO 2025-06-15 03:21:40 ts/train.py:232 step:30K smpl:242K ep:568 epch:568.47 loss:0.035 grdn:5.657 lr:1.0e-05 updt_s:0.234 data_s:0.023
191
+ INFO 2025-06-15 03:22:31 ts/train.py:232 step:30K smpl:243K ep:572 epch:572.24 loss:0.033 grdn:5.368 lr:1.0e-05 updt_s:0.236 data_s:0.017
192
+ INFO 2025-06-15 03:23:23 ts/train.py:232 step:31K smpl:245K ep:576 epch:576.00 loss:0.034 grdn:5.909 lr:1.0e-05 updt_s:0.236 data_s:0.023
193
+ INFO 2025-06-15 03:24:15 ts/train.py:232 step:31K smpl:246K ep:580 epch:579.76 loss:0.034 grdn:5.666 lr:1.0e-05 updt_s:0.234 data_s:0.023
194
+ INFO 2025-06-15 03:25:06 ts/train.py:232 step:31K smpl:248K ep:584 epch:583.53 loss:0.033 grdn:5.428 lr:1.0e-05 updt_s:0.235 data_s:0.021
195
+ INFO 2025-06-15 03:25:57 ts/train.py:232 step:31K smpl:250K ep:587 epch:587.29 loss:0.034 grdn:5.512 lr:1.0e-05 updt_s:0.235 data_s:0.018
196
+ INFO 2025-06-15 03:26:49 ts/train.py:232 step:31K smpl:251K ep:591 epch:591.06 loss:0.033 grdn:5.602 lr:1.0e-05 updt_s:0.235 data_s:0.024
197
+ INFO 2025-06-15 03:27:41 ts/train.py:232 step:32K smpl:253K ep:595 epch:594.82 loss:0.033 grdn:5.531 lr:1.0e-05 updt_s:0.236 data_s:0.022
198
+ INFO 2025-06-15 03:28:32 ts/train.py:232 step:32K smpl:254K ep:599 epch:598.59 loss:0.033 grdn:5.458 lr:1.0e-05 updt_s:0.234 data_s:0.019
199
+ INFO 2025-06-15 03:29:23 ts/train.py:232 step:32K smpl:256K ep:602 epch:602.35 loss:0.033 grdn:5.469 lr:1.0e-05 updt_s:0.235 data_s:0.022
200
+ INFO 2025-06-15 03:29:23 ts/train.py:241 Checkpoint policy after step 32000
201
+ INFO 2025-06-15 03:30:17 ts/train.py:232 step:32K smpl:258K ep:606 epch:606.12 loss:0.033 grdn:5.379 lr:1.0e-05 updt_s:0.234 data_s:0.022
202
+ INFO 2025-06-15 03:31:07 ts/train.py:232 step:32K smpl:259K ep:610 epch:609.88 loss:0.032 grdn:5.519 lr:1.0e-05 updt_s:0.236 data_s:0.018
203
+ INFO 2025-06-15 03:32:00 ts/train.py:232 step:33K smpl:261K ep:614 epch:613.65 loss:0.032 grdn:5.513 lr:1.0e-05 updt_s:0.236 data_s:0.025
204
+ INFO 2025-06-15 03:32:51 ts/train.py:232 step:33K smpl:262K ep:617 epch:617.41 loss:0.032 grdn:5.423 lr:1.0e-05 updt_s:0.234 data_s:0.021
205
+ INFO 2025-06-15 03:33:43 ts/train.py:232 step:33K smpl:264K ep:621 epch:621.18 loss:0.033 grdn:5.634 lr:1.0e-05 updt_s:0.235 data_s:0.024
206
+ INFO 2025-06-15 03:34:34 ts/train.py:232 step:33K smpl:266K ep:625 epch:624.94 loss:0.032 grdn:5.357 lr:1.0e-05 updt_s:0.235 data_s:0.019
207
+ INFO 2025-06-15 03:35:26 ts/train.py:232 step:33K smpl:267K ep:629 epch:628.71 loss:0.032 grdn:5.438 lr:1.0e-05 updt_s:0.236 data_s:0.023
208
+ INFO 2025-06-15 03:36:19 ts/train.py:232 step:34K smpl:269K ep:632 epch:632.47 loss:0.031 grdn:5.147 lr:1.0e-05 updt_s:0.235 data_s:0.025
209
+ INFO 2025-06-15 03:37:09 ts/train.py:232 step:34K smpl:270K ep:636 epch:636.24 loss:0.031 grdn:5.264 lr:1.0e-05 updt_s:0.235 data_s:0.018
210
+ INFO 2025-06-15 03:38:02 ts/train.py:232 step:34K smpl:272K ep:640 epch:640.00 loss:0.032 grdn:5.300 lr:1.0e-05 updt_s:0.235 data_s:0.025
211
+ INFO 2025-06-15 03:38:02 ts/train.py:241 Checkpoint policy after step 34000
212
+ INFO 2025-06-15 03:38:55 ts/train.py:232 step:34K smpl:274K ep:644 epch:643.76 loss:0.031 grdn:5.093 lr:1.0e-05 updt_s:0.234 data_s:0.024
213
+ INFO 2025-06-15 03:39:47 ts/train.py:232 step:34K smpl:275K ep:648 epch:647.53 loss:0.031 grdn:5.174 lr:1.0e-05 updt_s:0.235 data_s:0.023
214
+ INFO 2025-06-15 03:40:38 ts/train.py:232 step:35K smpl:277K ep:651 epch:651.29 loss:0.032 grdn:5.361 lr:1.0e-05 updt_s:0.237 data_s:0.016
215
+ INFO 2025-06-15 03:41:30 ts/train.py:232 step:35K smpl:278K ep:655 epch:655.06 loss:0.031 grdn:5.447 lr:1.0e-05 updt_s:0.234 data_s:0.025
216
+ INFO 2025-06-15 03:42:22 ts/train.py:232 step:35K smpl:280K ep:659 epch:658.82 loss:0.031 grdn:5.133 lr:1.0e-05 updt_s:0.235 data_s:0.024
217
+ INFO 2025-06-15 03:43:13 ts/train.py:232 step:35K smpl:282K ep:663 epch:662.59 loss:0.030 grdn:5.086 lr:1.0e-05 updt_s:0.235 data_s:0.017
218
+ INFO 2025-06-15 03:44:05 ts/train.py:232 step:35K smpl:283K ep:666 epch:666.35 loss:0.031 grdn:5.252 lr:1.0e-05 updt_s:0.235 data_s:0.025
219
+ INFO 2025-06-15 03:44:57 ts/train.py:232 step:36K smpl:285K ep:670 epch:670.12 loss:0.030 grdn:5.170 lr:1.0e-05 updt_s:0.235 data_s:0.024
220
+ INFO 2025-06-15 03:45:48 ts/train.py:232 step:36K smpl:286K ep:674 epch:673.88 loss:0.030 grdn:5.112 lr:1.0e-05 updt_s:0.235 data_s:0.017
221
+ INFO 2025-06-15 03:46:40 ts/train.py:232 step:36K smpl:288K ep:678 epch:677.65 loss:0.030 grdn:5.352 lr:1.0e-05 updt_s:0.236 data_s:0.023
222
+ INFO 2025-06-15 03:46:40 ts/train.py:241 Checkpoint policy after step 36000
223
+ INFO 2025-06-15 03:47:33 ts/train.py:232 step:36K smpl:290K ep:681 epch:681.41 loss:0.030 grdn:5.110 lr:1.0e-05 updt_s:0.235 data_s:0.022
224
+ INFO 2025-06-15 03:48:26 ts/train.py:232 step:36K smpl:291K ep:685 epch:685.18 loss:0.031 grdn:5.180 lr:1.0e-05 updt_s:0.236 data_s:0.025
225
+ INFO 2025-06-15 03:49:17 ts/train.py:232 step:37K smpl:293K ep:689 epch:688.94 loss:0.030 grdn:5.061 lr:1.0e-05 updt_s:0.238 data_s:0.016
226
+ INFO 2025-06-15 03:50:09 ts/train.py:232 step:37K smpl:294K ep:693 epch:692.71 loss:0.031 grdn:5.173 lr:1.0e-05 updt_s:0.235 data_s:0.024
227
+ INFO 2025-06-15 03:51:01 ts/train.py:232 step:37K smpl:296K ep:696 epch:696.47 loss:0.029 grdn:5.106 lr:1.0e-05 updt_s:0.236 data_s:0.024
228
+ INFO 2025-06-15 03:51:52 ts/train.py:232 step:37K smpl:298K ep:700 epch:700.24 loss:0.029 grdn:4.976 lr:1.0e-05 updt_s:0.235 data_s:0.020
229
+ INFO 2025-06-15 03:52:44 ts/train.py:232 step:37K smpl:299K ep:704 epch:704.00 loss:0.030 grdn:5.063 lr:1.0e-05 updt_s:0.235 data_s:0.023
230
+ INFO 2025-06-15 03:53:36 ts/train.py:232 step:38K smpl:301K ep:708 epch:707.76 loss:0.030 grdn:5.009 lr:1.0e-05 updt_s:0.235 data_s:0.021
231
+ INFO 2025-06-15 03:54:26 ts/train.py:232 step:38K smpl:302K ep:712 epch:711.53 loss:0.028 grdn:4.813 lr:1.0e-05 updt_s:0.234 data_s:0.018
232
+ INFO 2025-06-15 03:55:19 ts/train.py:232 step:38K smpl:304K ep:715 epch:715.29 loss:0.029 grdn:4.888 lr:1.0e-05 updt_s:0.236 data_s:0.024
233
+ INFO 2025-06-15 03:55:19 ts/train.py:241 Checkpoint policy after step 38000
234
+ INFO 2025-06-15 03:56:12 ts/train.py:232 step:38K smpl:306K ep:719 epch:719.06 loss:0.029 grdn:5.106 lr:1.0e-05 updt_s:0.234 data_s:0.023
235
+ INFO 2025-06-15 03:57:04 ts/train.py:232 step:38K smpl:307K ep:723 epch:722.82 loss:0.029 grdn:4.884 lr:1.0e-05 updt_s:0.236 data_s:0.025
236
+ INFO 2025-06-15 03:57:56 ts/train.py:232 step:39K smpl:309K ep:727 epch:726.59 loss:0.028 grdn:4.758 lr:1.0e-05 updt_s:0.237 data_s:0.017
237
+ INFO 2025-06-15 03:58:47 ts/train.py:232 step:39K smpl:310K ep:730 epch:730.35 loss:0.028 grdn:4.824 lr:1.0e-05 updt_s:0.234 data_s:0.024
238
+ INFO 2025-06-15 03:59:39 ts/train.py:232 step:39K smpl:312K ep:734 epch:734.12 loss:0.028 grdn:4.807 lr:1.0e-05 updt_s:0.236 data_s:0.022
239
+ INFO 2025-06-15 04:00:30 ts/train.py:232 step:39K smpl:314K ep:738 epch:737.88 loss:0.029 grdn:4.805 lr:1.0e-05 updt_s:0.235 data_s:0.017
240
+ INFO 2025-06-15 04:01:22 ts/train.py:232 step:39K smpl:315K ep:742 epch:741.65 loss:0.028 grdn:4.741 lr:1.0e-05 updt_s:0.236 data_s:0.025
241
+ INFO 2025-06-15 04:02:14 ts/train.py:232 step:40K smpl:317K ep:745 epch:745.41 loss:0.028 grdn:4.920 lr:1.0e-05 updt_s:0.235 data_s:0.023
242
+ INFO 2025-06-15 04:03:05 ts/train.py:232 step:40K smpl:318K ep:749 epch:749.18 loss:0.029 grdn:5.106 lr:1.0e-05 updt_s:0.234 data_s:0.021
243
+ INFO 2025-06-15 04:03:56 ts/train.py:232 step:40K smpl:320K ep:753 epch:752.94 loss:0.028 grdn:4.900 lr:1.0e-05 updt_s:0.236 data_s:0.016
244
+ INFO 2025-06-15 04:03:56 ts/train.py:241 Checkpoint policy after step 40000
245
+ INFO 2025-06-15 04:03:58 ts/train.py:283 End of training
wandb/run-20250615_011116-h5jstcmg/logs/debug-core.log ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2025-06-15T01:11:16.477799867Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpdzbg2bgw/port-4025658.txt","pid":4025658,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
2
+ {"time":"2025-06-15T01:11:16.485807272Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":4025658}
3
+ {"time":"2025-06-15T01:11:16.48580074Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":36179,"Zone":""}}
4
+ {"time":"2025-06-15T01:11:16.665292401Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:49126"}
5
+ {"time":"2025-06-15T01:11:16.672826229Z","level":"INFO","msg":"handleInformInit: received","streamId":"h5jstcmg","id":"127.0.0.1:49126"}
6
+ {"time":"2025-06-15T01:11:16.817599146Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"h5jstcmg","id":"127.0.0.1:49126"}
7
+ {"time":"2025-06-15T04:03:59.296699125Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:49126"}
8
+ {"time":"2025-06-15T04:03:59.296771241Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:49126"}
9
+ {"time":"2025-06-15T04:03:59.296797272Z","level":"INFO","msg":"server is shutting down"}
10
+ {"time":"2025-06-15T04:03:59.296868863Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:49126"}
11
+ {"time":"2025-06-15T04:04:00.632846199Z","level":"ERROR","msg":"processOutgoingData: flush error","error":"write tcp 127.0.0.1:36179->127.0.0.1:49126: use of closed network connection","id":"127.0.0.1:49126"}
12
+ {"time":"2025-06-15T04:04:00.747361235Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:49126"}
13
+ {"time":"2025-06-15T04:04:00.747377539Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:49126"}
14
+ {"time":"2025-06-15T04:04:00.747388972Z","level":"INFO","msg":"server is closed"}
wandb/run-20250615_011116-h5jstcmg/logs/debug-internal.log ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2025-06-15T01:11:16.672938296Z","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug-core.log"}
2
+ {"time":"2025-06-15T01:11:16.817564688Z","level":"INFO","msg":"created new stream","id":"h5jstcmg"}
3
+ {"time":"2025-06-15T01:11:16.817593605Z","level":"INFO","msg":"stream: started","id":"h5jstcmg"}
4
+ {"time":"2025-06-15T01:11:16.817622363Z","level":"INFO","msg":"writer: Do: started","stream_id":"h5jstcmg"}
5
+ {"time":"2025-06-15T01:11:16.817662835Z","level":"INFO","msg":"handler: started","stream_id":"h5jstcmg"}
6
+ {"time":"2025-06-15T01:11:16.817664338Z","level":"INFO","msg":"sender: started","stream_id":"h5jstcmg"}
7
+ {"time":"2025-06-15T01:11:16.908550251Z","level":"INFO","msg":"Starting system monitor"}
8
+ {"time":"2025-06-15T04:03:59.296805009Z","level":"INFO","msg":"stream: closing","id":"h5jstcmg"}
9
+ {"time":"2025-06-15T04:03:59.296861951Z","level":"INFO","msg":"Stopping system monitor"}
10
+ {"time":"2025-06-15T04:03:59.296897991Z","level":"INFO","msg":"Stopped system monitor"}
11
+ {"time":"2025-06-15T04:04:00.683232437Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
12
+ {"time":"2025-06-15T04:04:00.747268558Z","level":"INFO","msg":"handler: closed","stream_id":"h5jstcmg"}
13
+ {"time":"2025-06-15T04:04:00.74729338Z","level":"INFO","msg":"writer: Close: closed","stream_id":"h5jstcmg"}
14
+ {"time":"2025-06-15T04:04:00.747309168Z","level":"INFO","msg":"sender: closed","stream_id":"h5jstcmg"}
15
+ {"time":"2025-06-15T04:04:00.747351595Z","level":"INFO","msg":"stream: closed","id":"h5jstcmg"}
wandb/run-20250615_011116-h5jstcmg/logs/debug.log ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
2
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Configure stats pid to 4025658
3
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
4
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from /content/lerobot/wandb/settings
5
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from environment variables
6
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_init.py:setup_run_log_directory():724] Logging user logs to outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug.log
7
+ 2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug-internal.log
8
+ 2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():852] calling init triggers
9
+ 2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
10
+ config: {'dataset': {'repo_id': 'CombatRoad/grab-test-2', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 100, 'n_action_steps': 100, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/eval_act_grab-test-2', 'job_name': 'eval_act_grab-test-2', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 40000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 2000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
11
+ 2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():893] starting backend
12
+ 2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():897] sending inform_init request
13
+ 2025-06-15 01:11:16,671 INFO MainThread:4025658 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
14
+ 2025-06-15 01:11:16,671 INFO MainThread:4025658 [wandb_init.py:init():907] backend started and connected
15
+ 2025-06-15 01:11:16,673 INFO MainThread:4025658 [wandb_init.py:init():1005] updated telemetry
16
+ 2025-06-15 01:11:16,678 INFO MainThread:4025658 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
17
+ 2025-06-15 01:11:16,906 INFO MainThread:4025658 [wandb_init.py:init():1104] starting run threads in backend
18
+ 2025-06-15 01:11:17,324 INFO MainThread:4025658 [wandb_run.py:_console_start():2573] atexit reg
19
+ 2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2421] redirect: wrap_raw
20
+ 2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2490] Wrapping output streams.
21
+ 2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2513] Redirects installed.
22
+ 2025-06-15 01:11:17,326 INFO MainThread:4025658 [wandb_init.py:init():1150] run started, returning control to user process
23
+ 2025-06-15 04:03:59,296 INFO MsgRouterThr:4025658 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.
wandb/run-20250615_011116-h5jstcmg/run-h5jstcmg.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:67783920a1168c8e55bd00fab5800b413f1af342cf4f8a585123ee8d7fb7534a
3
+ size 815916