Re-upload all files from training folder
Browse files- .gitattributes +1 -0
- wandb/debug-internal.log +15 -0
- wandb/debug.log +23 -0
- wandb/run-20250615_011116-h5jstcmg/files/output.log +245 -0
- wandb/run-20250615_011116-h5jstcmg/logs/debug-core.log +14 -0
- wandb/run-20250615_011116-h5jstcmg/logs/debug-internal.log +15 -0
- wandb/run-20250615_011116-h5jstcmg/logs/debug.log +23 -0
- wandb/run-20250615_011116-h5jstcmg/run-h5jstcmg.wandb +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
wandb/run-20250615_011116-h5jstcmg/run-h5jstcmg.wandb filter=lfs diff=lfs merge=lfs -text
|
wandb/debug-internal.log
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-06-15T01:11:16.672938296Z","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug-core.log"}
|
| 2 |
+
{"time":"2025-06-15T01:11:16.817564688Z","level":"INFO","msg":"created new stream","id":"h5jstcmg"}
|
| 3 |
+
{"time":"2025-06-15T01:11:16.817593605Z","level":"INFO","msg":"stream: started","id":"h5jstcmg"}
|
| 4 |
+
{"time":"2025-06-15T01:11:16.817622363Z","level":"INFO","msg":"writer: Do: started","stream_id":"h5jstcmg"}
|
| 5 |
+
{"time":"2025-06-15T01:11:16.817662835Z","level":"INFO","msg":"handler: started","stream_id":"h5jstcmg"}
|
| 6 |
+
{"time":"2025-06-15T01:11:16.817664338Z","level":"INFO","msg":"sender: started","stream_id":"h5jstcmg"}
|
| 7 |
+
{"time":"2025-06-15T01:11:16.908550251Z","level":"INFO","msg":"Starting system monitor"}
|
| 8 |
+
{"time":"2025-06-15T04:03:59.296805009Z","level":"INFO","msg":"stream: closing","id":"h5jstcmg"}
|
| 9 |
+
{"time":"2025-06-15T04:03:59.296861951Z","level":"INFO","msg":"Stopping system monitor"}
|
| 10 |
+
{"time":"2025-06-15T04:03:59.296897991Z","level":"INFO","msg":"Stopped system monitor"}
|
| 11 |
+
{"time":"2025-06-15T04:04:00.683232437Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 12 |
+
{"time":"2025-06-15T04:04:00.747268558Z","level":"INFO","msg":"handler: closed","stream_id":"h5jstcmg"}
|
| 13 |
+
{"time":"2025-06-15T04:04:00.74729338Z","level":"INFO","msg":"writer: Close: closed","stream_id":"h5jstcmg"}
|
| 14 |
+
{"time":"2025-06-15T04:04:00.747309168Z","level":"INFO","msg":"sender: closed","stream_id":"h5jstcmg"}
|
| 15 |
+
{"time":"2025-06-15T04:04:00.747351595Z","level":"INFO","msg":"stream: closed","id":"h5jstcmg"}
|
wandb/debug.log
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
|
| 2 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Configure stats pid to 4025658
|
| 3 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
|
| 4 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from /content/lerobot/wandb/settings
|
| 5 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from environment variables
|
| 6 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_init.py:setup_run_log_directory():724] Logging user logs to outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug.log
|
| 7 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug-internal.log
|
| 8 |
+
2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():852] calling init triggers
|
| 9 |
+
2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
|
| 10 |
+
config: {'dataset': {'repo_id': 'CombatRoad/grab-test-2', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 100, 'n_action_steps': 100, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/eval_act_grab-test-2', 'job_name': 'eval_act_grab-test-2', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 40000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 2000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
|
| 11 |
+
2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():893] starting backend
|
| 12 |
+
2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():897] sending inform_init request
|
| 13 |
+
2025-06-15 01:11:16,671 INFO MainThread:4025658 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
| 14 |
+
2025-06-15 01:11:16,671 INFO MainThread:4025658 [wandb_init.py:init():907] backend started and connected
|
| 15 |
+
2025-06-15 01:11:16,673 INFO MainThread:4025658 [wandb_init.py:init():1005] updated telemetry
|
| 16 |
+
2025-06-15 01:11:16,678 INFO MainThread:4025658 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
|
| 17 |
+
2025-06-15 01:11:16,906 INFO MainThread:4025658 [wandb_init.py:init():1104] starting run threads in backend
|
| 18 |
+
2025-06-15 01:11:17,324 INFO MainThread:4025658 [wandb_run.py:_console_start():2573] atexit reg
|
| 19 |
+
2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2421] redirect: wrap_raw
|
| 20 |
+
2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2490] Wrapping output streams.
|
| 21 |
+
2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2513] Redirects installed.
|
| 22 |
+
2025-06-15 01:11:17,326 INFO MainThread:4025658 [wandb_init.py:init():1150] run started, returning control to user process
|
| 23 |
+
2025-06-15 04:03:59,296 INFO MsgRouterThr:4025658 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.
|
wandb/run-20250615_011116-h5jstcmg/files/output.log
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[1m[34mLogs will be synced with wandb.[0m
|
| 2 |
+
INFO 2025-06-15 01:11:17 db_utils.py:103 Track this run --> [1m[33mhttps://wandb.ai/combatroad-keimyung-university/lerobot/runs/h5jstcmg[0m
|
| 3 |
+
INFO 2025-06-15 01:11:17 ts/train.py:127 Creating dataset
|
| 4 |
+
tasks.jsonl: 100% 43.0/43.0 [00:00<00:00, 259kB/s]
|
| 5 |
+
info.json: 100% 3.35k/3.35k [00:00<00:00, 13.4MB/s]
|
| 6 |
+
episodes.jsonl: 100% 64.0/64.0 [00:00<00:00, 453kB/s]
|
| 7 |
+
episodes_stats.jsonl: 100% 2.19k/2.19k [00:00<00:00, 11.3MB/s]
|
| 8 |
+
Fetching 4 files: 100% 4/4 [00:00<00:00, 23.87it/s]s]
|
| 9 |
+
README.md: 100% 3.87k/3.87k [00:00<00:00, 19.5MB/s]
|
| 10 |
+
.gitattributes: 100% 2.46k/2.46k [00:00<00:00, 10.0MB/s]
|
| 11 |
+
videos/chunk-000/observation.images.fron(…): 100% 9.17M/9.17M [00:00<00:00, 14.7MB/s]
|
| 12 |
+
data/chunk-000/episode_000000.parquet: 100% 23.7k/23.7k [00:00<00:00, 32.6kB/s]
|
| 13 |
+
videos/chunk-000/observation.images.side(…): 100% 9.55M/9.55M [00:01<00:00, 7.93MB/s]
|
| 14 |
+
Fetching 9 files: 100% 9/9 [00:01<00:00, 6.74it/s].17M/9.17M [00:00<00:00, 14.8MB/s]
|
| 15 |
+
Generating train split: 425 examples [00:00, 66551.40 examples/s]:01<00:00, 7.96MB/s]
|
| 16 |
+
INFO 2025-06-15 01:11:19 ts/train.py:138 Creating policy
|
| 17 |
+
INFO 2025-06-15 01:11:20 ts/train.py:144 Creating optimizer and scheduler
|
| 18 |
+
INFO 2025-06-15 01:11:20 ts/train.py:156 [1m[33mOutput dir:[0m outputs/train/eval_act_grab-test-2
|
| 19 |
+
INFO 2025-06-15 01:11:20 ts/train.py:159 cfg.steps=40000 (40K)
|
| 20 |
+
INFO 2025-06-15 01:11:20 ts/train.py:160 dataset.num_frames=425 (425)
|
| 21 |
+
INFO 2025-06-15 01:11:20 ts/train.py:161 dataset.num_episodes=1
|
| 22 |
+
INFO 2025-06-15 01:11:20 ts/train.py:162 num_learnable_params=51597190 (52M)
|
| 23 |
+
INFO 2025-06-15 01:11:20 ts/train.py:163 num_total_params=51597238 (52M)
|
| 24 |
+
INFO 2025-06-15 01:11:20 ts/train.py:202 Start offline training on a fixed dataset
|
| 25 |
+
INFO 2025-06-15 01:12:18 ts/train.py:232 step:200 smpl:2K ep:4 epch:3.76 loss:5.636 grdn:146.118 lr:1.0e-05 updt_s:0.262 data_s:0.025
|
| 26 |
+
INFO 2025-06-15 01:13:09 ts/train.py:232 step:400 smpl:3K ep:8 epch:7.53 loss:2.653 grdn:82.455 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 27 |
+
INFO 2025-06-15 01:14:01 ts/train.py:232 step:600 smpl:5K ep:11 epch:11.29 loss:2.280 grdn:75.804 lr:1.0e-05 updt_s:0.234 data_s:0.021
|
| 28 |
+
INFO 2025-06-15 01:14:51 ts/train.py:232 step:800 smpl:6K ep:15 epch:15.06 loss:1.965 grdn:69.023 lr:1.0e-05 updt_s:0.236 data_s:0.016
|
| 29 |
+
INFO 2025-06-15 01:15:43 ts/train.py:232 step:1K smpl:8K ep:19 epch:18.82 loss:1.713 grdn:66.089 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 30 |
+
INFO 2025-06-15 01:16:35 ts/train.py:232 step:1K smpl:10K ep:23 epch:22.59 loss:1.522 grdn:62.062 lr:1.0e-05 updt_s:0.235 data_s:0.025
|
| 31 |
+
INFO 2025-06-15 01:17:26 ts/train.py:232 step:1K smpl:11K ep:26 epch:26.35 loss:1.362 grdn:59.223 lr:1.0e-05 updt_s:0.236 data_s:0.018
|
| 32 |
+
INFO 2025-06-15 01:18:18 ts/train.py:232 step:2K smpl:13K ep:30 epch:30.12 loss:1.217 grdn:56.104 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 33 |
+
INFO 2025-06-15 01:19:09 ts/train.py:232 step:2K smpl:14K ep:34 epch:33.88 loss:1.082 grdn:52.909 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 34 |
+
INFO 2025-06-15 01:20:01 ts/train.py:232 step:2K smpl:16K ep:38 epch:37.65 loss:0.979 grdn:49.833 lr:1.0e-05 updt_s:0.233 data_s:0.022
|
| 35 |
+
INFO 2025-06-15 01:20:01 ts/train.py:241 Checkpoint policy after step 2000
|
| 36 |
+
INFO 2025-06-15 01:20:54 ts/train.py:232 step:2K smpl:18K ep:41 epch:41.41 loss:0.870 grdn:46.410 lr:1.0e-05 updt_s:0.237 data_s:0.017
|
| 37 |
+
INFO 2025-06-15 01:21:45 ts/train.py:232 step:2K smpl:19K ep:45 epch:45.18 loss:0.775 grdn:44.919 lr:1.0e-05 updt_s:0.236 data_s:0.022
|
| 38 |
+
INFO 2025-06-15 01:22:37 ts/train.py:232 step:3K smpl:21K ep:49 epch:48.94 loss:0.697 grdn:42.161 lr:1.0e-05 updt_s:0.234 data_s:0.025
|
| 39 |
+
INFO 2025-06-15 01:23:28 ts/train.py:232 step:3K smpl:22K ep:53 epch:52.71 loss:0.620 grdn:39.421 lr:1.0e-05 updt_s:0.236 data_s:0.017
|
| 40 |
+
INFO 2025-06-15 01:24:20 ts/train.py:232 step:3K smpl:24K ep:56 epch:56.47 loss:0.549 grdn:37.159 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 41 |
+
INFO 2025-06-15 01:25:12 ts/train.py:232 step:3K smpl:26K ep:60 epch:60.24 loss:0.489 grdn:34.932 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 42 |
+
INFO 2025-06-15 01:26:03 ts/train.py:232 step:3K smpl:27K ep:64 epch:64.00 loss:0.439 grdn:32.961 lr:1.0e-05 updt_s:0.236 data_s:0.018
|
| 43 |
+
INFO 2025-06-15 01:26:55 ts/train.py:232 step:4K smpl:29K ep:68 epch:67.76 loss:0.387 grdn:30.758 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 44 |
+
INFO 2025-06-15 01:27:47 ts/train.py:232 step:4K smpl:30K ep:72 epch:71.53 loss:0.355 grdn:29.736 lr:1.0e-05 updt_s:0.235 data_s:0.025
|
| 45 |
+
INFO 2025-06-15 01:28:38 ts/train.py:232 step:4K smpl:32K ep:75 epch:75.29 loss:0.318 grdn:28.297 lr:1.0e-05 updt_s:0.233 data_s:0.023
|
| 46 |
+
INFO 2025-06-15 01:28:38 ts/train.py:241 Checkpoint policy after step 4000
|
| 47 |
+
INFO 2025-06-15 01:29:31 ts/train.py:232 step:4K smpl:34K ep:79 epch:79.06 loss:0.289 grdn:26.797 lr:1.0e-05 updt_s:0.237 data_s:0.017
|
| 48 |
+
INFO 2025-06-15 01:30:23 ts/train.py:232 step:4K smpl:35K ep:83 epch:82.82 loss:0.268 grdn:25.509 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 49 |
+
INFO 2025-06-15 01:31:15 ts/train.py:232 step:5K smpl:37K ep:87 epch:86.59 loss:0.245 grdn:24.868 lr:1.0e-05 updt_s:0.234 data_s:0.025
|
| 50 |
+
INFO 2025-06-15 01:32:06 ts/train.py:232 step:5K smpl:38K ep:90 epch:90.35 loss:0.228 grdn:23.587 lr:1.0e-05 updt_s:0.236 data_s:0.017
|
| 51 |
+
INFO 2025-06-15 01:32:58 ts/train.py:232 step:5K smpl:40K ep:94 epch:94.12 loss:0.212 grdn:22.247 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 52 |
+
INFO 2025-06-15 01:33:50 ts/train.py:232 step:5K smpl:42K ep:98 epch:97.88 loss:0.201 grdn:21.935 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 53 |
+
INFO 2025-06-15 01:34:40 ts/train.py:232 step:5K smpl:43K ep:102 epch:101.65 loss:0.189 grdn:21.018 lr:1.0e-05 updt_s:0.236 data_s:0.016
|
| 54 |
+
INFO 2025-06-15 01:35:32 ts/train.py:232 step:6K smpl:45K ep:105 epch:105.41 loss:0.177 grdn:20.272 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 55 |
+
INFO 2025-06-15 01:36:24 ts/train.py:232 step:6K smpl:46K ep:109 epch:109.18 loss:0.168 grdn:19.692 lr:1.0e-05 updt_s:0.236 data_s:0.023
|
| 56 |
+
INFO 2025-06-15 01:37:15 ts/train.py:232 step:6K smpl:48K ep:113 epch:112.94 loss:0.162 grdn:19.346 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 57 |
+
INFO 2025-06-15 01:37:15 ts/train.py:241 Checkpoint policy after step 6000
|
| 58 |
+
INFO 2025-06-15 01:38:08 ts/train.py:232 step:6K smpl:50K ep:117 epch:116.71 loss:0.153 grdn:18.388 lr:1.0e-05 updt_s:0.236 data_s:0.017
|
| 59 |
+
INFO 2025-06-15 01:38:59 ts/train.py:232 step:6K smpl:51K ep:120 epch:120.47 loss:0.146 grdn:17.518 lr:1.0e-05 updt_s:0.235 data_s:0.021
|
| 60 |
+
INFO 2025-06-15 01:39:51 ts/train.py:232 step:7K smpl:53K ep:124 epch:124.24 loss:0.143 grdn:17.532 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 61 |
+
INFO 2025-06-15 01:40:42 ts/train.py:232 step:7K smpl:54K ep:128 epch:128.00 loss:0.134 grdn:16.715 lr:1.0e-05 updt_s:0.236 data_s:0.018
|
| 62 |
+
INFO 2025-06-15 01:41:33 ts/train.py:232 step:7K smpl:56K ep:132 epch:131.76 loss:0.132 grdn:16.883 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 63 |
+
INFO 2025-06-15 01:42:25 ts/train.py:232 step:7K smpl:58K ep:136 epch:135.53 loss:0.127 grdn:16.502 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 64 |
+
INFO 2025-06-15 01:43:18 ts/train.py:232 step:7K smpl:59K ep:139 epch:139.29 loss:0.121 grdn:15.927 lr:1.0e-05 updt_s:0.235 data_s:0.026
|
| 65 |
+
INFO 2025-06-15 01:44:08 ts/train.py:232 step:8K smpl:61K ep:143 epch:143.06 loss:0.119 grdn:15.558 lr:1.0e-05 updt_s:0.234 data_s:0.016
|
| 66 |
+
INFO 2025-06-15 01:45:00 ts/train.py:232 step:8K smpl:62K ep:147 epch:146.82 loss:0.114 grdn:15.445 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 67 |
+
INFO 2025-06-15 01:45:52 ts/train.py:232 step:8K smpl:64K ep:151 epch:150.59 loss:0.113 grdn:15.088 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 68 |
+
INFO 2025-06-15 01:45:52 ts/train.py:241 Checkpoint policy after step 8000
|
| 69 |
+
INFO 2025-06-15 01:46:45 ts/train.py:232 step:8K smpl:66K ep:154 epch:154.35 loss:0.109 grdn:14.368 lr:1.0e-05 updt_s:0.236 data_s:0.018
|
| 70 |
+
INFO 2025-06-15 01:47:37 ts/train.py:232 step:8K smpl:67K ep:158 epch:158.12 loss:0.108 grdn:14.542 lr:1.0e-05 updt_s:0.235 data_s:0.025
|
| 71 |
+
INFO 2025-06-15 01:48:28 ts/train.py:232 step:9K smpl:69K ep:162 epch:161.88 loss:0.104 grdn:14.211 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 72 |
+
INFO 2025-06-15 01:49:19 ts/train.py:232 step:9K smpl:70K ep:166 epch:165.65 loss:0.102 grdn:13.837 lr:1.0e-05 updt_s:0.236 data_s:0.017
|
| 73 |
+
INFO 2025-06-15 01:50:11 ts/train.py:232 step:9K smpl:72K ep:169 epch:169.41 loss:0.100 grdn:13.779 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 74 |
+
INFO 2025-06-15 01:51:02 ts/train.py:232 step:9K smpl:74K ep:173 epch:173.18 loss:0.097 grdn:13.687 lr:1.0e-05 updt_s:0.233 data_s:0.023
|
| 75 |
+
INFO 2025-06-15 01:51:54 ts/train.py:232 step:9K smpl:75K ep:177 epch:176.94 loss:0.095 grdn:13.374 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 76 |
+
INFO 2025-06-15 01:52:45 ts/train.py:232 step:10K smpl:77K ep:181 epch:180.71 loss:0.093 grdn:12.952 lr:1.0e-05 updt_s:0.234 data_s:0.017
|
| 77 |
+
INFO 2025-06-15 01:53:36 ts/train.py:232 step:10K smpl:78K ep:184 epch:184.47 loss:0.091 grdn:12.838 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 78 |
+
INFO 2025-06-15 01:54:29 ts/train.py:232 step:10K smpl:80K ep:188 epch:188.24 loss:0.088 grdn:12.684 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 79 |
+
INFO 2025-06-15 01:54:29 ts/train.py:241 Checkpoint policy after step 10000
|
| 80 |
+
INFO 2025-06-15 01:55:21 ts/train.py:232 step:10K smpl:82K ep:192 epch:192.00 loss:0.087 grdn:12.378 lr:1.0e-05 updt_s:0.235 data_s:0.018
|
| 81 |
+
INFO 2025-06-15 01:56:13 ts/train.py:232 step:10K smpl:83K ep:196 epch:195.76 loss:0.085 grdn:12.049 lr:1.0e-05 updt_s:0.236 data_s:0.022
|
| 82 |
+
INFO 2025-06-15 01:57:04 ts/train.py:232 step:11K smpl:85K ep:200 epch:199.53 loss:0.083 grdn:12.318 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 83 |
+
INFO 2025-06-15 01:57:56 ts/train.py:232 step:11K smpl:86K ep:203 epch:203.29 loss:0.083 grdn:12.118 lr:1.0e-05 updt_s:0.235 data_s:0.020
|
| 84 |
+
INFO 2025-06-15 01:58:48 ts/train.py:232 step:11K smpl:88K ep:207 epch:207.06 loss:0.081 grdn:11.818 lr:1.0e-05 updt_s:0.236 data_s:0.024
|
| 85 |
+
INFO 2025-06-15 01:59:39 ts/train.py:232 step:11K smpl:90K ep:211 epch:210.82 loss:0.079 grdn:11.620 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 86 |
+
INFO 2025-06-15 02:00:31 ts/train.py:232 step:11K smpl:91K ep:215 epch:214.59 loss:0.078 grdn:11.537 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 87 |
+
INFO 2025-06-15 02:01:22 ts/train.py:232 step:12K smpl:93K ep:218 epch:218.35 loss:0.077 grdn:11.357 lr:1.0e-05 updt_s:0.234 data_s:0.018
|
| 88 |
+
INFO 2025-06-15 02:02:14 ts/train.py:232 step:12K smpl:94K ep:222 epch:222.12 loss:0.076 grdn:11.410 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 89 |
+
INFO 2025-06-15 02:03:06 ts/train.py:232 step:12K smpl:96K ep:226 epch:225.88 loss:0.074 grdn:10.901 lr:1.0e-05 updt_s:0.235 data_s:0.025
|
| 90 |
+
INFO 2025-06-15 02:03:06 ts/train.py:241 Checkpoint policy after step 12000
|
| 91 |
+
INFO 2025-06-15 02:03:59 ts/train.py:232 step:12K smpl:98K ep:230 epch:229.65 loss:0.073 grdn:10.586 lr:1.0e-05 updt_s:0.234 data_s:0.019
|
| 92 |
+
INFO 2025-06-15 02:04:51 ts/train.py:232 step:12K smpl:99K ep:233 epch:233.41 loss:0.072 grdn:10.887 lr:1.0e-05 updt_s:0.236 data_s:0.023
|
| 93 |
+
INFO 2025-06-15 02:05:42 ts/train.py:232 step:13K smpl:101K ep:237 epch:237.18 loss:0.072 grdn:10.751 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 94 |
+
INFO 2025-06-15 02:06:34 ts/train.py:232 step:13K smpl:102K ep:241 epch:240.94 loss:0.069 grdn:10.319 lr:1.0e-05 updt_s:0.236 data_s:0.023
|
| 95 |
+
INFO 2025-06-15 02:07:25 ts/train.py:232 step:13K smpl:104K ep:245 epch:244.71 loss:0.069 grdn:10.191 lr:1.0e-05 updt_s:0.236 data_s:0.017
|
| 96 |
+
INFO 2025-06-15 02:08:17 ts/train.py:232 step:13K smpl:106K ep:248 epch:248.47 loss:0.068 grdn:10.289 lr:1.0e-05 updt_s:0.234 data_s:0.025
|
| 97 |
+
INFO 2025-06-15 02:09:09 ts/train.py:232 step:13K smpl:107K ep:252 epch:252.24 loss:0.067 grdn:10.057 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 98 |
+
INFO 2025-06-15 02:10:00 ts/train.py:232 step:14K smpl:109K ep:256 epch:256.00 loss:0.066 grdn:10.003 lr:1.0e-05 updt_s:0.234 data_s:0.018
|
| 99 |
+
INFO 2025-06-15 02:10:52 ts/train.py:232 step:14K smpl:110K ep:260 epch:259.76 loss:0.065 grdn:10.039 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 100 |
+
INFO 2025-06-15 02:11:44 ts/train.py:232 step:14K smpl:112K ep:264 epch:263.53 loss:0.064 grdn:9.852 lr:1.0e-05 updt_s:0.236 data_s:0.021
|
| 101 |
+
INFO 2025-06-15 02:11:44 ts/train.py:241 Checkpoint policy after step 14000
|
| 102 |
+
INFO 2025-06-15 02:12:36 ts/train.py:232 step:14K smpl:114K ep:267 epch:267.29 loss:0.063 grdn:9.613 lr:1.0e-05 updt_s:0.235 data_s:0.017
|
| 103 |
+
INFO 2025-06-15 02:13:28 ts/train.py:232 step:14K smpl:115K ep:271 epch:271.06 loss:0.063 grdn:9.216 lr:1.0e-05 updt_s:0.236 data_s:0.023
|
| 104 |
+
INFO 2025-06-15 02:14:20 ts/train.py:232 step:15K smpl:117K ep:275 epch:274.82 loss:0.062 grdn:9.752 lr:1.0e-05 updt_s:0.234 data_s:0.025
|
| 105 |
+
INFO 2025-06-15 02:15:12 ts/train.py:232 step:15K smpl:118K ep:279 epch:278.59 loss:0.061 grdn:9.354 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 106 |
+
INFO 2025-06-15 02:16:03 ts/train.py:232 step:15K smpl:120K ep:282 epch:282.35 loss:0.060 grdn:9.343 lr:1.0e-05 updt_s:0.236 data_s:0.017
|
| 107 |
+
INFO 2025-06-15 02:16:54 ts/train.py:232 step:15K smpl:122K ep:286 epch:286.12 loss:0.060 grdn:9.347 lr:1.0e-05 updt_s:0.234 data_s:0.021
|
| 108 |
+
INFO 2025-06-15 02:17:47 ts/train.py:232 step:15K smpl:123K ep:290 epch:289.88 loss:0.059 grdn:9.314 lr:1.0e-05 updt_s:0.235 data_s:0.026
|
| 109 |
+
INFO 2025-06-15 02:18:37 ts/train.py:232 step:16K smpl:125K ep:294 epch:293.65 loss:0.058 grdn:9.231 lr:1.0e-05 updt_s:0.235 data_s:0.016
|
| 110 |
+
INFO 2025-06-15 02:19:29 ts/train.py:232 step:16K smpl:126K ep:297 epch:297.41 loss:0.057 grdn:9.074 lr:1.0e-05 updt_s:0.236 data_s:0.022
|
| 111 |
+
INFO 2025-06-15 02:20:21 ts/train.py:232 step:16K smpl:128K ep:301 epch:301.18 loss:0.057 grdn:8.898 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 112 |
+
INFO 2025-06-15 02:20:21 ts/train.py:241 Checkpoint policy after step 16000
|
| 113 |
+
INFO 2025-06-15 02:21:14 ts/train.py:232 step:16K smpl:130K ep:305 epch:304.94 loss:0.057 grdn:8.887 lr:1.0e-05 updt_s:0.234 data_s:0.018
|
| 114 |
+
INFO 2025-06-15 02:22:07 ts/train.py:232 step:16K smpl:131K ep:309 epch:308.71 loss:0.056 grdn:8.590 lr:1.0e-05 updt_s:0.237 data_s:0.025
|
| 115 |
+
INFO 2025-06-15 02:22:58 ts/train.py:232 step:17K smpl:133K ep:312 epch:312.47 loss:0.056 grdn:8.675 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 116 |
+
INFO 2025-06-15 02:23:50 ts/train.py:232 step:17K smpl:134K ep:316 epch:316.24 loss:0.054 grdn:8.383 lr:1.0e-05 updt_s:0.235 data_s:0.021
|
| 117 |
+
INFO 2025-06-15 02:24:41 ts/train.py:232 step:17K smpl:136K ep:320 epch:320.00 loss:0.053 grdn:8.116 lr:1.0e-05 updt_s:0.237 data_s:0.016
|
| 118 |
+
INFO 2025-06-15 02:25:33 ts/train.py:232 step:17K smpl:138K ep:324 epch:323.76 loss:0.053 grdn:8.273 lr:1.0e-05 updt_s:0.235 data_s:0.025
|
| 119 |
+
INFO 2025-06-15 02:26:25 ts/train.py:232 step:17K smpl:139K ep:328 epch:327.53 loss:0.053 grdn:8.463 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 120 |
+
INFO 2025-06-15 02:27:16 ts/train.py:232 step:18K smpl:141K ep:331 epch:331.29 loss:0.053 grdn:8.071 lr:1.0e-05 updt_s:0.235 data_s:0.017
|
| 121 |
+
INFO 2025-06-15 02:28:07 ts/train.py:232 step:18K smpl:142K ep:335 epch:335.06 loss:0.052 grdn:8.215 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 122 |
+
INFO 2025-06-15 02:28:59 ts/train.py:232 step:18K smpl:144K ep:339 epch:338.82 loss:0.052 grdn:8.441 lr:1.0e-05 updt_s:0.236 data_s:0.022
|
| 123 |
+
INFO 2025-06-15 02:28:59 ts/train.py:241 Checkpoint policy after step 18000
|
| 124 |
+
INFO 2025-06-15 02:29:53 ts/train.py:232 step:18K smpl:146K ep:343 epch:342.59 loss:0.051 grdn:7.805 lr:1.0e-05 updt_s:0.234 data_s:0.025
|
| 125 |
+
INFO 2025-06-15 02:30:44 ts/train.py:232 step:18K smpl:147K ep:346 epch:346.35 loss:0.051 grdn:8.160 lr:1.0e-05 updt_s:0.236 data_s:0.016
|
| 126 |
+
INFO 2025-06-15 02:31:35 ts/train.py:232 step:19K smpl:149K ep:350 epch:350.12 loss:0.051 grdn:8.032 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 127 |
+
INFO 2025-06-15 02:32:28 ts/train.py:232 step:19K smpl:150K ep:354 epch:353.88 loss:0.049 grdn:7.788 lr:1.0e-05 updt_s:0.236 data_s:0.024
|
| 128 |
+
INFO 2025-06-15 02:33:19 ts/train.py:232 step:19K smpl:152K ep:358 epch:357.65 loss:0.049 grdn:7.815 lr:1.0e-05 updt_s:0.236 data_s:0.018
|
| 129 |
+
INFO 2025-06-15 02:34:10 ts/train.py:232 step:19K smpl:154K ep:361 epch:361.41 loss:0.049 grdn:7.959 lr:1.0e-05 updt_s:0.234 data_s:0.021
|
| 130 |
+
INFO 2025-06-15 02:35:02 ts/train.py:232 step:19K smpl:155K ep:365 epch:365.18 loss:0.048 grdn:7.828 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 131 |
+
INFO 2025-06-15 02:35:52 ts/train.py:232 step:20K smpl:157K ep:369 epch:368.94 loss:0.048 grdn:7.767 lr:1.0e-05 updt_s:0.234 data_s:0.018
|
| 132 |
+
INFO 2025-06-15 02:36:44 ts/train.py:232 step:20K smpl:158K ep:373 epch:372.71 loss:0.048 grdn:7.640 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 133 |
+
INFO 2025-06-15 02:37:36 ts/train.py:232 step:20K smpl:160K ep:376 epch:376.47 loss:0.047 grdn:7.615 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 134 |
+
INFO 2025-06-15 02:37:36 ts/train.py:241 Checkpoint policy after step 20000
|
| 135 |
+
INFO 2025-06-15 02:38:29 ts/train.py:232 step:20K smpl:162K ep:380 epch:380.24 loss:0.047 grdn:7.616 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 136 |
+
INFO 2025-06-15 02:39:21 ts/train.py:232 step:20K smpl:163K ep:384 epch:384.00 loss:0.047 grdn:7.348 lr:1.0e-05 updt_s:0.236 data_s:0.019
|
| 137 |
+
INFO 2025-06-15 02:40:12 ts/train.py:232 step:21K smpl:165K ep:388 epch:387.76 loss:0.046 grdn:7.347 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 138 |
+
INFO 2025-06-15 02:41:04 ts/train.py:232 step:21K smpl:166K ep:392 epch:391.53 loss:0.046 grdn:7.450 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 139 |
+
INFO 2025-06-15 02:41:55 ts/train.py:232 step:21K smpl:168K ep:395 epch:395.29 loss:0.044 grdn:7.100 lr:1.0e-05 updt_s:0.236 data_s:0.017
|
| 140 |
+
INFO 2025-06-15 02:42:46 ts/train.py:232 step:21K smpl:170K ep:399 epch:399.06 loss:0.045 grdn:7.292 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 141 |
+
INFO 2025-06-15 02:43:38 ts/train.py:232 step:21K smpl:171K ep:403 epch:402.82 loss:0.045 grdn:7.139 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 142 |
+
INFO 2025-06-15 02:44:29 ts/train.py:232 step:22K smpl:173K ep:407 epch:406.59 loss:0.044 grdn:7.002 lr:1.0e-05 updt_s:0.234 data_s:0.016
|
| 143 |
+
INFO 2025-06-15 02:45:21 ts/train.py:232 step:22K smpl:174K ep:410 epch:410.35 loss:0.045 grdn:7.311 lr:1.0e-05 updt_s:0.237 data_s:0.022
|
| 144 |
+
INFO 2025-06-15 02:46:13 ts/train.py:232 step:22K smpl:176K ep:414 epch:414.12 loss:0.044 grdn:7.108 lr:1.0e-05 updt_s:0.235 data_s:0.025
|
| 145 |
+
INFO 2025-06-15 02:46:13 ts/train.py:241 Checkpoint policy after step 22000
|
| 146 |
+
INFO 2025-06-15 02:47:07 ts/train.py:232 step:22K smpl:178K ep:418 epch:417.88 loss:0.044 grdn:7.061 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 147 |
+
INFO 2025-06-15 02:47:58 ts/train.py:232 step:22K smpl:179K ep:422 epch:421.65 loss:0.043 grdn:6.996 lr:1.0e-05 updt_s:0.236 data_s:0.018
|
| 148 |
+
INFO 2025-06-15 02:48:50 ts/train.py:232 step:23K smpl:181K ep:425 epch:425.41 loss:0.043 grdn:6.993 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 149 |
+
INFO 2025-06-15 02:49:42 ts/train.py:232 step:23K smpl:182K ep:429 epch:429.18 loss:0.042 grdn:6.992 lr:1.0e-05 updt_s:0.236 data_s:0.024
|
| 150 |
+
INFO 2025-06-15 02:50:33 ts/train.py:232 step:23K smpl:184K ep:433 epch:432.94 loss:0.042 grdn:6.749 lr:1.0e-05 updt_s:0.236 data_s:0.018
|
| 151 |
+
INFO 2025-06-15 02:51:25 ts/train.py:232 step:23K smpl:186K ep:437 epch:436.71 loss:0.042 grdn:6.906 lr:1.0e-05 updt_s:0.234 data_s:0.025
|
| 152 |
+
INFO 2025-06-15 02:52:17 ts/train.py:232 step:23K smpl:187K ep:440 epch:440.47 loss:0.042 grdn:6.957 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 153 |
+
INFO 2025-06-15 02:53:09 ts/train.py:232 step:24K smpl:189K ep:444 epch:444.24 loss:0.041 grdn:6.609 lr:1.0e-05 updt_s:0.234 data_s:0.025
|
| 154 |
+
INFO 2025-06-15 02:54:00 ts/train.py:232 step:24K smpl:190K ep:448 epch:448.00 loss:0.042 grdn:6.620 lr:1.0e-05 updt_s:0.237 data_s:0.015
|
| 155 |
+
INFO 2025-06-15 02:54:52 ts/train.py:232 step:24K smpl:192K ep:452 epch:451.76 loss:0.041 grdn:6.505 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 156 |
+
INFO 2025-06-15 02:54:52 ts/train.py:241 Checkpoint policy after step 24000
|
| 157 |
+
INFO 2025-06-15 02:55:46 ts/train.py:232 step:24K smpl:194K ep:456 epch:455.53 loss:0.040 grdn:6.489 lr:1.0e-05 updt_s:0.234 data_s:0.026
|
| 158 |
+
INFO 2025-06-15 02:56:37 ts/train.py:232 step:24K smpl:195K ep:459 epch:459.29 loss:0.040 grdn:6.752 lr:1.0e-05 updt_s:0.237 data_s:0.019
|
| 159 |
+
INFO 2025-06-15 02:57:29 ts/train.py:232 step:25K smpl:197K ep:463 epch:463.06 loss:0.040 grdn:6.495 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 160 |
+
INFO 2025-06-15 02:58:21 ts/train.py:232 step:25K smpl:198K ep:467 epch:466.82 loss:0.040 grdn:6.566 lr:1.0e-05 updt_s:0.236 data_s:0.026
|
| 161 |
+
INFO 2025-06-15 02:59:12 ts/train.py:232 step:25K smpl:200K ep:471 epch:470.59 loss:0.039 grdn:6.202 lr:1.0e-05 updt_s:0.236 data_s:0.017
|
| 162 |
+
INFO 2025-06-15 03:00:04 ts/train.py:232 step:25K smpl:202K ep:474 epch:474.35 loss:0.040 grdn:6.561 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 163 |
+
INFO 2025-06-15 03:00:57 ts/train.py:232 step:25K smpl:203K ep:478 epch:478.12 loss:0.040 grdn:6.802 lr:1.0e-05 updt_s:0.236 data_s:0.024
|
| 164 |
+
INFO 2025-06-15 03:01:49 ts/train.py:232 step:26K smpl:205K ep:482 epch:481.88 loss:0.039 grdn:6.358 lr:1.0e-05 updt_s:0.234 data_s:0.026
|
| 165 |
+
INFO 2025-06-15 03:02:40 ts/train.py:232 step:26K smpl:206K ep:486 epch:485.65 loss:0.038 grdn:6.173 lr:1.0e-05 updt_s:0.237 data_s:0.018
|
| 166 |
+
INFO 2025-06-15 03:03:32 ts/train.py:232 step:26K smpl:208K ep:489 epch:489.41 loss:0.038 grdn:6.151 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 167 |
+
INFO 2025-06-15 03:03:32 ts/train.py:241 Checkpoint policy after step 26000
|
| 168 |
+
INFO 2025-06-15 03:04:25 ts/train.py:232 step:26K smpl:210K ep:493 epch:493.18 loss:0.038 grdn:6.239 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 169 |
+
INFO 2025-06-15 03:05:17 ts/train.py:232 step:26K smpl:211K ep:497 epch:496.94 loss:0.038 grdn:6.299 lr:1.0e-05 updt_s:0.237 data_s:0.019
|
| 170 |
+
INFO 2025-06-15 03:06:08 ts/train.py:232 step:27K smpl:213K ep:501 epch:500.71 loss:0.038 grdn:6.293 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 171 |
+
INFO 2025-06-15 03:07:00 ts/train.py:232 step:27K smpl:214K ep:504 epch:504.47 loss:0.037 grdn:6.017 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 172 |
+
INFO 2025-06-15 03:07:51 ts/train.py:232 step:27K smpl:216K ep:508 epch:508.24 loss:0.037 grdn:6.112 lr:1.0e-05 updt_s:0.235 data_s:0.019
|
| 173 |
+
INFO 2025-06-15 03:08:43 ts/train.py:232 step:27K smpl:218K ep:512 epch:512.00 loss:0.038 grdn:6.114 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 174 |
+
INFO 2025-06-15 03:09:35 ts/train.py:232 step:27K smpl:219K ep:516 epch:515.76 loss:0.037 grdn:5.982 lr:1.0e-05 updt_s:0.236 data_s:0.022
|
| 175 |
+
INFO 2025-06-15 03:10:27 ts/train.py:232 step:28K smpl:221K ep:520 epch:519.53 loss:0.036 grdn:5.900 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 176 |
+
INFO 2025-06-15 03:11:18 ts/train.py:232 step:28K smpl:222K ep:523 epch:523.29 loss:0.037 grdn:6.109 lr:1.0e-05 updt_s:0.237 data_s:0.017
|
| 177 |
+
INFO 2025-06-15 03:12:10 ts/train.py:232 step:28K smpl:224K ep:527 epch:527.06 loss:0.036 grdn:6.173 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 178 |
+
INFO 2025-06-15 03:12:10 ts/train.py:241 Checkpoint policy after step 28000
|
| 179 |
+
INFO 2025-06-15 03:13:03 ts/train.py:232 step:28K smpl:226K ep:531 epch:530.82 loss:0.036 grdn:5.848 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 180 |
+
INFO 2025-06-15 03:13:55 ts/train.py:232 step:28K smpl:227K ep:535 epch:534.59 loss:0.036 grdn:5.773 lr:1.0e-05 updt_s:0.236 data_s:0.019
|
| 181 |
+
INFO 2025-06-15 03:14:47 ts/train.py:232 step:29K smpl:229K ep:538 epch:538.35 loss:0.036 grdn:6.053 lr:1.0e-05 updt_s:0.236 data_s:0.024
|
| 182 |
+
INFO 2025-06-15 03:15:38 ts/train.py:232 step:29K smpl:230K ep:542 epch:542.12 loss:0.036 grdn:6.106 lr:1.0e-05 updt_s:0.233 data_s:0.024
|
| 183 |
+
INFO 2025-06-15 03:16:30 ts/train.py:232 step:29K smpl:232K ep:546 epch:545.88 loss:0.036 grdn:5.920 lr:1.0e-05 updt_s:0.235 data_s:0.021
|
| 184 |
+
INFO 2025-06-15 03:17:21 ts/train.py:232 step:29K smpl:234K ep:550 epch:549.65 loss:0.035 grdn:5.828 lr:1.0e-05 updt_s:0.234 data_s:0.018
|
| 185 |
+
INFO 2025-06-15 03:18:13 ts/train.py:232 step:29K smpl:235K ep:553 epch:553.41 loss:0.035 grdn:5.808 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 186 |
+
INFO 2025-06-15 03:19:04 ts/train.py:232 step:30K smpl:237K ep:557 epch:557.18 loss:0.034 grdn:5.820 lr:1.0e-05 updt_s:0.236 data_s:0.022
|
| 187 |
+
INFO 2025-06-15 03:19:55 ts/train.py:232 step:30K smpl:238K ep:561 epch:560.94 loss:0.035 grdn:5.628 lr:1.0e-05 updt_s:0.235 data_s:0.017
|
| 188 |
+
INFO 2025-06-15 03:20:47 ts/train.py:232 step:30K smpl:240K ep:565 epch:564.71 loss:0.035 grdn:5.823 lr:1.0e-05 updt_s:0.236 data_s:0.024
|
| 189 |
+
INFO 2025-06-15 03:20:47 ts/train.py:241 Checkpoint policy after step 30000
|
| 190 |
+
INFO 2025-06-15 03:21:40 ts/train.py:232 step:30K smpl:242K ep:568 epch:568.47 loss:0.035 grdn:5.657 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 191 |
+
INFO 2025-06-15 03:22:31 ts/train.py:232 step:30K smpl:243K ep:572 epch:572.24 loss:0.033 grdn:5.368 lr:1.0e-05 updt_s:0.236 data_s:0.017
|
| 192 |
+
INFO 2025-06-15 03:23:23 ts/train.py:232 step:31K smpl:245K ep:576 epch:576.00 loss:0.034 grdn:5.909 lr:1.0e-05 updt_s:0.236 data_s:0.023
|
| 193 |
+
INFO 2025-06-15 03:24:15 ts/train.py:232 step:31K smpl:246K ep:580 epch:579.76 loss:0.034 grdn:5.666 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 194 |
+
INFO 2025-06-15 03:25:06 ts/train.py:232 step:31K smpl:248K ep:584 epch:583.53 loss:0.033 grdn:5.428 lr:1.0e-05 updt_s:0.235 data_s:0.021
|
| 195 |
+
INFO 2025-06-15 03:25:57 ts/train.py:232 step:31K smpl:250K ep:587 epch:587.29 loss:0.034 grdn:5.512 lr:1.0e-05 updt_s:0.235 data_s:0.018
|
| 196 |
+
INFO 2025-06-15 03:26:49 ts/train.py:232 step:31K smpl:251K ep:591 epch:591.06 loss:0.033 grdn:5.602 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 197 |
+
INFO 2025-06-15 03:27:41 ts/train.py:232 step:32K smpl:253K ep:595 epch:594.82 loss:0.033 grdn:5.531 lr:1.0e-05 updt_s:0.236 data_s:0.022
|
| 198 |
+
INFO 2025-06-15 03:28:32 ts/train.py:232 step:32K smpl:254K ep:599 epch:598.59 loss:0.033 grdn:5.458 lr:1.0e-05 updt_s:0.234 data_s:0.019
|
| 199 |
+
INFO 2025-06-15 03:29:23 ts/train.py:232 step:32K smpl:256K ep:602 epch:602.35 loss:0.033 grdn:5.469 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 200 |
+
INFO 2025-06-15 03:29:23 ts/train.py:241 Checkpoint policy after step 32000
|
| 201 |
+
INFO 2025-06-15 03:30:17 ts/train.py:232 step:32K smpl:258K ep:606 epch:606.12 loss:0.033 grdn:5.379 lr:1.0e-05 updt_s:0.234 data_s:0.022
|
| 202 |
+
INFO 2025-06-15 03:31:07 ts/train.py:232 step:32K smpl:259K ep:610 epch:609.88 loss:0.032 grdn:5.519 lr:1.0e-05 updt_s:0.236 data_s:0.018
|
| 203 |
+
INFO 2025-06-15 03:32:00 ts/train.py:232 step:33K smpl:261K ep:614 epch:613.65 loss:0.032 grdn:5.513 lr:1.0e-05 updt_s:0.236 data_s:0.025
|
| 204 |
+
INFO 2025-06-15 03:32:51 ts/train.py:232 step:33K smpl:262K ep:617 epch:617.41 loss:0.032 grdn:5.423 lr:1.0e-05 updt_s:0.234 data_s:0.021
|
| 205 |
+
INFO 2025-06-15 03:33:43 ts/train.py:232 step:33K smpl:264K ep:621 epch:621.18 loss:0.033 grdn:5.634 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 206 |
+
INFO 2025-06-15 03:34:34 ts/train.py:232 step:33K smpl:266K ep:625 epch:624.94 loss:0.032 grdn:5.357 lr:1.0e-05 updt_s:0.235 data_s:0.019
|
| 207 |
+
INFO 2025-06-15 03:35:26 ts/train.py:232 step:33K smpl:267K ep:629 epch:628.71 loss:0.032 grdn:5.438 lr:1.0e-05 updt_s:0.236 data_s:0.023
|
| 208 |
+
INFO 2025-06-15 03:36:19 ts/train.py:232 step:34K smpl:269K ep:632 epch:632.47 loss:0.031 grdn:5.147 lr:1.0e-05 updt_s:0.235 data_s:0.025
|
| 209 |
+
INFO 2025-06-15 03:37:09 ts/train.py:232 step:34K smpl:270K ep:636 epch:636.24 loss:0.031 grdn:5.264 lr:1.0e-05 updt_s:0.235 data_s:0.018
|
| 210 |
+
INFO 2025-06-15 03:38:02 ts/train.py:232 step:34K smpl:272K ep:640 epch:640.00 loss:0.032 grdn:5.300 lr:1.0e-05 updt_s:0.235 data_s:0.025
|
| 211 |
+
INFO 2025-06-15 03:38:02 ts/train.py:241 Checkpoint policy after step 34000
|
| 212 |
+
INFO 2025-06-15 03:38:55 ts/train.py:232 step:34K smpl:274K ep:644 epch:643.76 loss:0.031 grdn:5.093 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 213 |
+
INFO 2025-06-15 03:39:47 ts/train.py:232 step:34K smpl:275K ep:648 epch:647.53 loss:0.031 grdn:5.174 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 214 |
+
INFO 2025-06-15 03:40:38 ts/train.py:232 step:35K smpl:277K ep:651 epch:651.29 loss:0.032 grdn:5.361 lr:1.0e-05 updt_s:0.237 data_s:0.016
|
| 215 |
+
INFO 2025-06-15 03:41:30 ts/train.py:232 step:35K smpl:278K ep:655 epch:655.06 loss:0.031 grdn:5.447 lr:1.0e-05 updt_s:0.234 data_s:0.025
|
| 216 |
+
INFO 2025-06-15 03:42:22 ts/train.py:232 step:35K smpl:280K ep:659 epch:658.82 loss:0.031 grdn:5.133 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 217 |
+
INFO 2025-06-15 03:43:13 ts/train.py:232 step:35K smpl:282K ep:663 epch:662.59 loss:0.030 grdn:5.086 lr:1.0e-05 updt_s:0.235 data_s:0.017
|
| 218 |
+
INFO 2025-06-15 03:44:05 ts/train.py:232 step:35K smpl:283K ep:666 epch:666.35 loss:0.031 grdn:5.252 lr:1.0e-05 updt_s:0.235 data_s:0.025
|
| 219 |
+
INFO 2025-06-15 03:44:57 ts/train.py:232 step:36K smpl:285K ep:670 epch:670.12 loss:0.030 grdn:5.170 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 220 |
+
INFO 2025-06-15 03:45:48 ts/train.py:232 step:36K smpl:286K ep:674 epch:673.88 loss:0.030 grdn:5.112 lr:1.0e-05 updt_s:0.235 data_s:0.017
|
| 221 |
+
INFO 2025-06-15 03:46:40 ts/train.py:232 step:36K smpl:288K ep:678 epch:677.65 loss:0.030 grdn:5.352 lr:1.0e-05 updt_s:0.236 data_s:0.023
|
| 222 |
+
INFO 2025-06-15 03:46:40 ts/train.py:241 Checkpoint policy after step 36000
|
| 223 |
+
INFO 2025-06-15 03:47:33 ts/train.py:232 step:36K smpl:290K ep:681 epch:681.41 loss:0.030 grdn:5.110 lr:1.0e-05 updt_s:0.235 data_s:0.022
|
| 224 |
+
INFO 2025-06-15 03:48:26 ts/train.py:232 step:36K smpl:291K ep:685 epch:685.18 loss:0.031 grdn:5.180 lr:1.0e-05 updt_s:0.236 data_s:0.025
|
| 225 |
+
INFO 2025-06-15 03:49:17 ts/train.py:232 step:37K smpl:293K ep:689 epch:688.94 loss:0.030 grdn:5.061 lr:1.0e-05 updt_s:0.238 data_s:0.016
|
| 226 |
+
INFO 2025-06-15 03:50:09 ts/train.py:232 step:37K smpl:294K ep:693 epch:692.71 loss:0.031 grdn:5.173 lr:1.0e-05 updt_s:0.235 data_s:0.024
|
| 227 |
+
INFO 2025-06-15 03:51:01 ts/train.py:232 step:37K smpl:296K ep:696 epch:696.47 loss:0.029 grdn:5.106 lr:1.0e-05 updt_s:0.236 data_s:0.024
|
| 228 |
+
INFO 2025-06-15 03:51:52 ts/train.py:232 step:37K smpl:298K ep:700 epch:700.24 loss:0.029 grdn:4.976 lr:1.0e-05 updt_s:0.235 data_s:0.020
|
| 229 |
+
INFO 2025-06-15 03:52:44 ts/train.py:232 step:37K smpl:299K ep:704 epch:704.00 loss:0.030 grdn:5.063 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 230 |
+
INFO 2025-06-15 03:53:36 ts/train.py:232 step:38K smpl:301K ep:708 epch:707.76 loss:0.030 grdn:5.009 lr:1.0e-05 updt_s:0.235 data_s:0.021
|
| 231 |
+
INFO 2025-06-15 03:54:26 ts/train.py:232 step:38K smpl:302K ep:712 epch:711.53 loss:0.028 grdn:4.813 lr:1.0e-05 updt_s:0.234 data_s:0.018
|
| 232 |
+
INFO 2025-06-15 03:55:19 ts/train.py:232 step:38K smpl:304K ep:715 epch:715.29 loss:0.029 grdn:4.888 lr:1.0e-05 updt_s:0.236 data_s:0.024
|
| 233 |
+
INFO 2025-06-15 03:55:19 ts/train.py:241 Checkpoint policy after step 38000
|
| 234 |
+
INFO 2025-06-15 03:56:12 ts/train.py:232 step:38K smpl:306K ep:719 epch:719.06 loss:0.029 grdn:5.106 lr:1.0e-05 updt_s:0.234 data_s:0.023
|
| 235 |
+
INFO 2025-06-15 03:57:04 ts/train.py:232 step:38K smpl:307K ep:723 epch:722.82 loss:0.029 grdn:4.884 lr:1.0e-05 updt_s:0.236 data_s:0.025
|
| 236 |
+
INFO 2025-06-15 03:57:56 ts/train.py:232 step:39K smpl:309K ep:727 epch:726.59 loss:0.028 grdn:4.758 lr:1.0e-05 updt_s:0.237 data_s:0.017
|
| 237 |
+
INFO 2025-06-15 03:58:47 ts/train.py:232 step:39K smpl:310K ep:730 epch:730.35 loss:0.028 grdn:4.824 lr:1.0e-05 updt_s:0.234 data_s:0.024
|
| 238 |
+
INFO 2025-06-15 03:59:39 ts/train.py:232 step:39K smpl:312K ep:734 epch:734.12 loss:0.028 grdn:4.807 lr:1.0e-05 updt_s:0.236 data_s:0.022
|
| 239 |
+
INFO 2025-06-15 04:00:30 ts/train.py:232 step:39K smpl:314K ep:738 epch:737.88 loss:0.029 grdn:4.805 lr:1.0e-05 updt_s:0.235 data_s:0.017
|
| 240 |
+
INFO 2025-06-15 04:01:22 ts/train.py:232 step:39K smpl:315K ep:742 epch:741.65 loss:0.028 grdn:4.741 lr:1.0e-05 updt_s:0.236 data_s:0.025
|
| 241 |
+
INFO 2025-06-15 04:02:14 ts/train.py:232 step:40K smpl:317K ep:745 epch:745.41 loss:0.028 grdn:4.920 lr:1.0e-05 updt_s:0.235 data_s:0.023
|
| 242 |
+
INFO 2025-06-15 04:03:05 ts/train.py:232 step:40K smpl:318K ep:749 epch:749.18 loss:0.029 grdn:5.106 lr:1.0e-05 updt_s:0.234 data_s:0.021
|
| 243 |
+
INFO 2025-06-15 04:03:56 ts/train.py:232 step:40K smpl:320K ep:753 epch:752.94 loss:0.028 grdn:4.900 lr:1.0e-05 updt_s:0.236 data_s:0.016
|
| 244 |
+
INFO 2025-06-15 04:03:56 ts/train.py:241 Checkpoint policy after step 40000
|
| 245 |
+
INFO 2025-06-15 04:03:58 ts/train.py:283 End of training
|
wandb/run-20250615_011116-h5jstcmg/logs/debug-core.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-06-15T01:11:16.477799867Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpdzbg2bgw/port-4025658.txt","pid":4025658,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2025-06-15T01:11:16.485807272Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":4025658}
|
| 3 |
+
{"time":"2025-06-15T01:11:16.48580074Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":36179,"Zone":""}}
|
| 4 |
+
{"time":"2025-06-15T01:11:16.665292401Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:49126"}
|
| 5 |
+
{"time":"2025-06-15T01:11:16.672826229Z","level":"INFO","msg":"handleInformInit: received","streamId":"h5jstcmg","id":"127.0.0.1:49126"}
|
| 6 |
+
{"time":"2025-06-15T01:11:16.817599146Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"h5jstcmg","id":"127.0.0.1:49126"}
|
| 7 |
+
{"time":"2025-06-15T04:03:59.296699125Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:49126"}
|
| 8 |
+
{"time":"2025-06-15T04:03:59.296771241Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:49126"}
|
| 9 |
+
{"time":"2025-06-15T04:03:59.296797272Z","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2025-06-15T04:03:59.296868863Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:49126"}
|
| 11 |
+
{"time":"2025-06-15T04:04:00.632846199Z","level":"ERROR","msg":"processOutgoingData: flush error","error":"write tcp 127.0.0.1:36179->127.0.0.1:49126: use of closed network connection","id":"127.0.0.1:49126"}
|
| 12 |
+
{"time":"2025-06-15T04:04:00.747361235Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:49126"}
|
| 13 |
+
{"time":"2025-06-15T04:04:00.747377539Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:49126"}
|
| 14 |
+
{"time":"2025-06-15T04:04:00.747388972Z","level":"INFO","msg":"server is closed"}
|
wandb/run-20250615_011116-h5jstcmg/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2025-06-15T01:11:16.672938296Z","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug-core.log"}
|
| 2 |
+
{"time":"2025-06-15T01:11:16.817564688Z","level":"INFO","msg":"created new stream","id":"h5jstcmg"}
|
| 3 |
+
{"time":"2025-06-15T01:11:16.817593605Z","level":"INFO","msg":"stream: started","id":"h5jstcmg"}
|
| 4 |
+
{"time":"2025-06-15T01:11:16.817622363Z","level":"INFO","msg":"writer: Do: started","stream_id":"h5jstcmg"}
|
| 5 |
+
{"time":"2025-06-15T01:11:16.817662835Z","level":"INFO","msg":"handler: started","stream_id":"h5jstcmg"}
|
| 6 |
+
{"time":"2025-06-15T01:11:16.817664338Z","level":"INFO","msg":"sender: started","stream_id":"h5jstcmg"}
|
| 7 |
+
{"time":"2025-06-15T01:11:16.908550251Z","level":"INFO","msg":"Starting system monitor"}
|
| 8 |
+
{"time":"2025-06-15T04:03:59.296805009Z","level":"INFO","msg":"stream: closing","id":"h5jstcmg"}
|
| 9 |
+
{"time":"2025-06-15T04:03:59.296861951Z","level":"INFO","msg":"Stopping system monitor"}
|
| 10 |
+
{"time":"2025-06-15T04:03:59.296897991Z","level":"INFO","msg":"Stopped system monitor"}
|
| 11 |
+
{"time":"2025-06-15T04:04:00.683232437Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 12 |
+
{"time":"2025-06-15T04:04:00.747268558Z","level":"INFO","msg":"handler: closed","stream_id":"h5jstcmg"}
|
| 13 |
+
{"time":"2025-06-15T04:04:00.74729338Z","level":"INFO","msg":"writer: Close: closed","stream_id":"h5jstcmg"}
|
| 14 |
+
{"time":"2025-06-15T04:04:00.747309168Z","level":"INFO","msg":"sender: closed","stream_id":"h5jstcmg"}
|
| 15 |
+
{"time":"2025-06-15T04:04:00.747351595Z","level":"INFO","msg":"stream: closed","id":"h5jstcmg"}
|
wandb/run-20250615_011116-h5jstcmg/logs/debug.log
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
|
| 2 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Configure stats pid to 4025658
|
| 3 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from /root/.config/wandb/settings
|
| 4 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from /content/lerobot/wandb/settings
|
| 5 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_setup.py:_flush():70] Loading settings from environment variables
|
| 6 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_init.py:setup_run_log_directory():724] Logging user logs to outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug.log
|
| 7 |
+
2025-06-15 01:11:16,667 INFO MainThread:4025658 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to outputs/train/eval_act_grab-test-2/wandb/run-20250615_011116-h5jstcmg/logs/debug-internal.log
|
| 8 |
+
2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():852] calling init triggers
|
| 9 |
+
2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
|
| 10 |
+
config: {'dataset': {'repo_id': 'CombatRoad/grab-test-2', 'root': None, 'episodes': None, 'image_transforms': {'enable': False, 'max_num_transforms': 3, 'random_order': False, 'tfs': {'brightness': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'brightness': [0.8, 1.2]}}, 'contrast': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'contrast': [0.8, 1.2]}}, 'saturation': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'saturation': [0.5, 1.5]}}, 'hue': {'weight': 1.0, 'type': 'ColorJitter', 'kwargs': {'hue': [-0.05, 0.05]}}, 'sharpness': {'weight': 1.0, 'type': 'SharpnessJitter', 'kwargs': {'sharpness': [0.5, 1.5]}}}}, 'revision': None, 'use_imagenet_stats': True, 'video_backend': 'pyav'}, 'env': None, 'policy': {'type': 'act', 'n_obs_steps': 1, 'normalization_mapping': {'VISUAL': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'STATE': <NormalizationMode.MEAN_STD: 'MEAN_STD'>, 'ACTION': <NormalizationMode.MEAN_STD: 'MEAN_STD'>}, 'input_features': {}, 'output_features': {}, 'device': 'cuda', 'use_amp': False, 'chunk_size': 100, 'n_action_steps': 100, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': 'ResNet18_Weights.IMAGENET1K_V1', 'replace_final_stride_with_dilation': False, 'pre_norm': False, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'temporal_ensemble_coeff': None, 'dropout': 0.1, 'kl_weight': 10.0, 'optimizer_lr': 1e-05, 'optimizer_weight_decay': 0.0001, 'optimizer_lr_backbone': 1e-05}, 'output_dir': 'outputs/train/eval_act_grab-test-2', 'job_name': 'eval_act_grab-test-2', 'resume': False, 'seed': 1000, 'num_workers': 4, 'batch_size': 8, 'steps': 40000, 'eval_freq': 20000, 'log_freq': 200, 'save_checkpoint': True, 'save_freq': 2000, 'use_policy_training_preset': True, 'optimizer': {'type': 'adamw', 'lr': 1e-05, 'weight_decay': 0.0001, 'grad_clip_norm': 10.0, 'betas': [0.9, 0.999], 'eps': 1e-08}, 'scheduler': None, 'eval': {'n_episodes': 50, 'batch_size': 50, 'use_async_envs': False}, 'wandb': {'enable': True, 'disable_artifact': False, 'project': 'lerobot', 'entity': None, 'notes': None, 'run_id': None, 'mode': None}, '_wandb': {}}
|
| 11 |
+
2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():893] starting backend
|
| 12 |
+
2025-06-15 01:11:16,668 INFO MainThread:4025658 [wandb_init.py:init():897] sending inform_init request
|
| 13 |
+
2025-06-15 01:11:16,671 INFO MainThread:4025658 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
| 14 |
+
2025-06-15 01:11:16,671 INFO MainThread:4025658 [wandb_init.py:init():907] backend started and connected
|
| 15 |
+
2025-06-15 01:11:16,673 INFO MainThread:4025658 [wandb_init.py:init():1005] updated telemetry
|
| 16 |
+
2025-06-15 01:11:16,678 INFO MainThread:4025658 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
|
| 17 |
+
2025-06-15 01:11:16,906 INFO MainThread:4025658 [wandb_init.py:init():1104] starting run threads in backend
|
| 18 |
+
2025-06-15 01:11:17,324 INFO MainThread:4025658 [wandb_run.py:_console_start():2573] atexit reg
|
| 19 |
+
2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2421] redirect: wrap_raw
|
| 20 |
+
2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2490] Wrapping output streams.
|
| 21 |
+
2025-06-15 01:11:17,325 INFO MainThread:4025658 [wandb_run.py:_redirect():2513] Redirects installed.
|
| 22 |
+
2025-06-15 01:11:17,326 INFO MainThread:4025658 [wandb_init.py:init():1150] run started, returning control to user process
|
| 23 |
+
2025-06-15 04:03:59,296 INFO MsgRouterThr:4025658 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 2 handles.
|
wandb/run-20250615_011116-h5jstcmg/run-h5jstcmg.wandb
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:67783920a1168c8e55bd00fab5800b413f1af342cf4f8a585123ee8d7fb7534a
|
| 3 |
+
size 815916
|