backfill batch 14 (62)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- wandb/run-20260423_044249-zs7qqoy8/logs/debug-internal.log +18 -0
- wandb/run-20260423_044249-zs7qqoy8/logs/debug.log +21 -0
- wandb/run-20260423_045157-714fdnxz/files/output.log +92 -0
- wandb/run-20260423_045157-714fdnxz/logs/debug-core.log +13 -0
- wandb/run-20260423_045157-714fdnxz/logs/debug-internal.log +190 -0
- wandb/run-20260423_045157-714fdnxz/logs/debug.log +21 -0
- wandb/run-20260423_051457-2j4bd54k/files/output.log +206 -0
- wandb/run-20260423_051457-2j4bd54k/logs/debug-core.log +13 -0
- wandb/run-20260423_051457-2j4bd54k/logs/debug-internal.log +378 -0
- wandb/run-20260423_051457-2j4bd54k/logs/debug.log +21 -0
- wandb/run-20260423_060125-7vxzycy6/files/output.log +505 -0
- wandb/run-20260423_060125-7vxzycy6/logs/debug-core.log +13 -0
- wandb/run-20260423_060125-7vxzycy6/logs/debug-internal.log +0 -0
- wandb/run-20260423_060125-7vxzycy6/logs/debug.log +21 -0
- wandb/run-20260423_093030-7hhwxxjb/files/output.log +517 -0
- wandb/run-20260423_093030-7hhwxxjb/logs/debug-core.log +15 -0
- wandb/run-20260423_093030-7hhwxxjb/logs/debug-internal.log +0 -0
- wandb/run-20260423_093030-7hhwxxjb/logs/debug.log +21 -0
- wandb/run-20260423_151526-rkikhrj0/files/output.log +6 -0
- wandb/run-20260423_151526-rkikhrj0/logs/debug-core.log +15 -0
- wandb/run-20260423_151526-rkikhrj0/logs/debug-internal.log +28 -0
- wandb/run-20260423_151526-rkikhrj0/logs/debug.log +21 -0
- wandb/run-20260423_152554-78ryrosu/files/output.log +0 -0
- wandb/run-20260423_152554-78ryrosu/logs/debug-core.log +13 -0
- wandb/run-20260423_152554-78ryrosu/logs/debug-internal.log +0 -0
- wandb/run-20260423_152554-78ryrosu/logs/debug.log +21 -0
- wandb/run-20260424_060920-vdilsesj/files/output.log +0 -0
- wandb/run-20260424_060920-vdilsesj/logs/debug-core.log +15 -0
- wandb/run-20260424_060920-vdilsesj/logs/debug-internal.log +0 -0
- wandb/run-20260424_060920-vdilsesj/logs/debug.log +21 -0
- wandb/run-20260426_173211-4zyjccoe/files/output.log +21 -0
- wandb/run-20260426_173211-4zyjccoe/logs/debug-core.log +13 -0
- wandb/run-20260426_173211-4zyjccoe/logs/debug-internal.log +24 -0
- wandb/run-20260426_173211-4zyjccoe/logs/debug.log +21 -0
- wandb/run-20260426_173430-jy4xvva6/files/output.log +0 -0
- wandb/run-20260426_173430-jy4xvva6/logs/debug-core.log +8 -0
- wandb/run-20260426_173430-jy4xvva6/logs/debug-internal.log +0 -0
- wandb/run-20260426_173430-jy4xvva6/logs/debug.log +19 -0
- wandb/run-20260428_172322-yizklreh/files/output.log +17 -0
- wandb/run-20260428_172322-yizklreh/logs/debug-core.log +8 -0
- wandb/run-20260428_172322-yizklreh/logs/debug-internal.log +93 -0
- wandb/run-20260428_172322-yizklreh/logs/debug.log +19 -0
- wandb/run-20260428_173540-bbc05m6k/files/output.log +930 -0
- wandb/run-20260428_173540-bbc05m6k/logs/debug-core.log +15 -0
- wandb/run-20260428_173540-bbc05m6k/logs/debug-internal.log +0 -0
- wandb/run-20260428_173540-bbc05m6k/logs/debug.log +21 -0
- wandb/run-20260428_183729-tv4p94fz/files/output.log +3 -0
- wandb/run-20260428_183729-tv4p94fz/logs/debug-core.log +14 -0
- wandb/run-20260428_183729-tv4p94fz/logs/debug-internal.log +16 -0
- wandb/run-20260428_183729-tv4p94fz/logs/debug.log +21 -0
wandb/run-20260423_044249-zs7qqoy8/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T04:42:49.35918078Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-23T04:42:49.360578769Z","level":"INFO","msg":"stream: starting","core version":"0.25.1"}
|
| 3 |
+
{"time":"2026-04-23T04:42:49.934754225Z","level":"INFO","msg":"stream: created new stream","id":"zs7qqoy8"}
|
| 4 |
+
{"time":"2026-04-23T04:42:49.934927083Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-23T04:42:49.9349765Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-23T04:42:49.93500259Z","level":"INFO","msg":"writer: started","stream_id":"zs7qqoy8"}
|
| 7 |
+
{"time":"2026-04-23T04:42:49.935012686Z","level":"INFO","msg":"sender: started"}
|
| 8 |
+
{"time":"2026-04-23T04:42:50.655439359Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-04-23T04:42:51.32632955Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-23T04:43:05.655645062Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":1,"console_offset":0,"console_lines":6,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-04-23T04:43:06.045031138Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-23T04:43:11.704078047Z","level":"INFO","msg":"stream: closing"}
|
| 13 |
+
{"time":"2026-04-23T04:43:12.472649817Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 14 |
+
{"time":"2026-04-23T04:43:12.473179548Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":1,"events_lines":1,"console_offset":6,"console_lines":32,"uploaded_len":3,"complete":true,"exit_code":1}
|
| 15 |
+
{"time":"2026-04-23T04:43:12.935507392Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-04-23T04:43:12.935664044Z","level":"INFO","msg":"handler: closed"}
|
| 17 |
+
{"time":"2026-04-23T04:43:12.93574588Z","level":"INFO","msg":"sender: closed"}
|
| 18 |
+
{"time":"2026-04-23T04:43:12.93576119Z","level":"INFO","msg":"stream: closed"}
|
wandb/run-20260423_044249-zs7qqoy8/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 04:42:49,131 INFO MainThread:889786 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-23 04:42:49,132 INFO MainThread:889786 [wandb_setup.py:_flush():81] Configure stats pid to 889786
|
| 3 |
+
2026-04-23 04:42:49,132 INFO MainThread:889786 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 04:42:49,132 INFO MainThread:889786 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260423_044249-zs7qqoy8/logs/debug.log
|
| 5 |
+
2026-04-23 04:42:49,132 INFO MainThread:889786 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260423_044249-zs7qqoy8/logs/debug-internal.log
|
| 6 |
+
2026-04-23 04:42:49,132 INFO MainThread:889786 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-23 04:42:49,132 INFO MainThread:889786 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 8, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top3_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0005, 'lr_min': 5e-05, 'warmup_steps': 10000, 'weight_decay': 0.02, 'grad_clip': 1.0, 'adam_betas': [0.9, 0.99], 'adam_eps': 1e-09, 'dtype': 'bf16', 'compile': 'default', 'curriculum_t_max_init': 0.3, 'curriculum_steps': 20000, 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_new_owt', 'save_dir': 'outputs/sad_new'}, '_wandb': {}}
|
| 9 |
+
2026-04-23 04:42:49,132 INFO MainThread:889786 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-23 04:42:49,351 INFO MainThread:889786 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-23 04:42:49,355 INFO MainThread:889786 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-23 04:42:49,357 INFO MainThread:889786 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-23 04:42:49,361 INFO MainThread:889786 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 04:42:50,587 INFO MainThread:889786 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-23 04:42:50,648 INFO MainThread:889786 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-23 04:42:50,648 INFO MainThread:889786 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-23 04:42:50,648 INFO MainThread:889786 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-23 04:42:50,648 INFO MainThread:889786 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-23 04:42:50,651 INFO MainThread:889786 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-23 04:43:11,704 INFO wandb-AsyncioManager-main:889786 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-23 04:43:11,704 INFO wandb-AsyncioManager-main:889786 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260423_045157-714fdnxz/files/output.log
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
full-denoise training: 0%| | 0/500000 [00:00<?, ?it/s][rank0]:W0423 04:51:59.612000 909865 site-packages/torch/_logging/_internal.py:1089] [0/0] Profiler function <class 'torch.autograd.profiler.record_function'> will be ignored
|
| 2 |
+
[rank0]:W0423 04:52:05.306000 909865 site-packages/torch/_dynamo/convert_frame.py:906] [3/8] torch._dynamo hit config.cache_size_limit (8)
|
| 3 |
+
[rank0]:W0423 04:52:05.306000 909865 site-packages/torch/_dynamo/convert_frame.py:906] [3/8] function: 'torch_dynamo_resume_in_hook_at_319' (/home/sunhc/diffusion/sad/src/utils/debug_utils.py:319)
|
| 4 |
+
[rank0]:W0423 04:52:05.306000 909865 site-packages/torch/_dynamo/convert_frame.py:906] [3/8] last reason: 3/1: tensor 'L['output']' dtype mismatch. expected Float, actual BFloat16
|
| 5 |
+
[rank0]:W0423 04:52:05.306000 909865 site-packages/torch/_dynamo/convert_frame.py:906] [3/8] To log all recompilation reasons, use TORCH_LOGS="recompiles".
|
| 6 |
+
[rank0]:W0423 04:52:05.306000 909865 site-packages/torch/_dynamo/convert_frame.py:906] [3/8] To diagnose recompilation issues, see https://pytorch.org/docs/main/torch.compiler_troubleshooting.html.
|
| 7 |
+
[step 0] [GRAD STATS] total_norm=3.8375e-01 max_single=2.2608e-01 (vocab_embed.embedding)
|
| 8 |
+
full-denoise training: 0%|▏ | 1600/500000 [12:13<62:06:55, 2.23it/s, leaf=0.6429, lr=8.0e-05]
|
| 9 |
+
[step 0] step= 0 | total=1.6813 | leaf=1.6813 | ancestor=0.0000 | lr=0.00e+00 | gnorm=3.8375e-01 | t=7.1s
|
| 10 |
+
[step 100] [GRAD STATS] total_norm=5.7111e-01 max_single=2.4027e-01 (vocab_embed.embedding)
|
| 11 |
+
[step 100] step= 100 | total=1.5913 | leaf=1.5913 | ancestor=0.0000 | lr=5.00e-06 | gnorm=5.7111e-01 | t=44.7s
|
| 12 |
+
[step 200] [GRAD STATS] total_norm=2.8939e-01 max_single=2.6784e-01 (vocab_embed.embedding)
|
| 13 |
+
[step 200] step= 200 | total=1.3939 | leaf=1.3939 | ancestor=0.0000 | lr=1.00e-05 | gnorm=2.8939e-01 | t=44.7s
|
| 14 |
+
[step 300] [GRAD STATS] total_norm=1.9621e-01 max_single=1.8121e-01 (vocab_embed.embedding)
|
| 15 |
+
[step 300] step= 300 | total=1.2702 | leaf=1.2702 | ancestor=0.0000 | lr=1.50e-05 | gnorm=1.9621e-01 | t=45.8s
|
| 16 |
+
[step 400] [GRAD STATS] total_norm=1.2198e-01 max_single=1.0431e-01 (vocab_embed.embedding)
|
| 17 |
+
[step 400] step= 400 | total=1.0965 | leaf=1.0965 | ancestor=0.0000 | lr=2.00e-05 | gnorm=1.2198e-01 | t=44.8s
|
| 18 |
+
[step 500] [GRAD STATS] total_norm=2.0822e-01 max_single=1.1388e-01 (segment_embed.weight)
|
| 19 |
+
[step 500] step= 500 | total=1.0640 | leaf=1.0640 | ancestor=0.0000 | lr=2.50e-05 | gnorm=2.0822e-01 | t=44.8s
|
| 20 |
+
[step 600] [GRAD STATS] total_norm=3.1063e-01 max_single=1.7680e-01 (segment_embed.weight)
|
| 21 |
+
[step 600] step= 600 | total=0.9859 | leaf=0.9859 | ancestor=0.0000 | lr=3.00e-05 | gnorm=3.1063e-01 | t=47.3s
|
| 22 |
+
[step 700] [GRAD STATS] total_norm=4.0455e-01 max_single=1.9365e-01 (segment_embed.weight)
|
| 23 |
+
[step 700] step= 700 | total=0.9559 | leaf=0.9559 | ancestor=0.0000 | lr=3.50e-05 | gnorm=4.0455e-01 | t=44.9s
|
| 24 |
+
[step 800] [GRAD STATS] total_norm=3.7235e-01 max_single=2.1468e-01 (segment_embed.weight)
|
| 25 |
+
[step 800] step= 800 | total=0.8873 | leaf=0.8873 | ancestor=0.0000 | lr=4.00e-05 | gnorm=3.7235e-01 | t=47.9s
|
| 26 |
+
[step 900] [GRAD STATS] total_norm=5.0979e-01 max_single=2.7573e-01 (segment_embed.weight)
|
| 27 |
+
[step 900] step= 900 | total=0.8170 | leaf=0.8170 | ancestor=0.0000 | lr=4.50e-05 | gnorm=5.0979e-01 | t=44.9s
|
| 28 |
+
[step 1000] [GRAD STATS] total_norm=4.5055e-01 max_single=2.0007e-01 (segment_embed.weight)
|
| 29 |
+
[step 1000] step= 1000 | total=0.7875 | leaf=0.7875 | ancestor=0.0000 | lr=5.00e-05 | gnorm=4.5055e-01 | t=45.0s
|
| 30 |
+
[step 1100] [GRAD STATS] total_norm=5.0730e-01 max_single=2.5448e-01 (segment_embed.weight)
|
| 31 |
+
[step 1100] step= 1100 | total=0.7391 | leaf=0.7391 | ancestor=0.0000 | lr=5.50e-05 | gnorm=5.0730e-01 | t=45.2s
|
| 32 |
+
[step 1200] [GRAD STATS] total_norm=6.5659e-01 max_single=3.6576e-01 (segment_embed.weight)
|
| 33 |
+
[step 1200] step= 1200 | total=0.6970 | leaf=0.6970 | ancestor=0.0000 | lr=6.00e-05 | gnorm=6.5659e-01 | t=45.1s
|
| 34 |
+
[step 1300] [GRAD STATS] total_norm=4.1640e-01 max_single=2.2803e-01 (segment_embed.weight)
|
| 35 |
+
[step 1300] step= 1300 | total=0.7038 | leaf=0.7038 | ancestor=0.0000 | lr=6.50e-05 | gnorm=4.1640e-01 | t=45.2s
|
| 36 |
+
[step 1400] [GRAD STATS] total_norm=5.2224e-01 max_single=3.0123e-01 (segment_embed.weight)
|
| 37 |
+
[step 1400] step= 1400 | total=0.6784 | leaf=0.6784 | ancestor=0.0000 | lr=7.00e-05 | gnorm=5.2224e-01 | t=45.6s
|
| 38 |
+
[step 1500] [GRAD STATS] total_norm=4.4046e-01 max_single=2.4135e-01 (segment_embed.weight)
|
| 39 |
+
[step 1500] step= 1500 | total=0.6848 | leaf=0.6848 | ancestor=0.0000 | lr=7.50e-05 | gnorm=4.4046e-01 | t=45.9s
|
| 40 |
+
[step 1600] [GRAD STATS] total_norm=5.1771e-01 max_single=2.9584e-01 (segment_embed.weight)
|
| 41 |
+
[step 1600] step= 1600 | total=0.6811 | leaf=0.6811 | ancestor=0.0000 | lr=8.00e-05 | gnorm=5.1771e-01 | t=44.9s
|
| 42 |
+
[step 1700] [GRAD STATS] total_norm=4.4087e-01 max_single=2.5131e-01 (segment_embed.weight)
|
| 43 |
+
[step 1700] step= 1700 | total=0.6348 | leaf=0.6348 | ancestor=0.0000 | lr=8.50e-05 | gnorm=4.4087e-01 | t=45.9s
|
| 44 |
+
[step 1800] [GRAD STATS] total_norm=4.6572e-01 max_single=2.7563e-01 (segment_embed.weight)
|
| 45 |
+
[step 1800] step= 1800 | total=0.5820 | leaf=0.5820 | ancestor=0.0000 | lr=9.00e-05 | gnorm=4.6572e-01 | t=45.4s
|
| 46 |
+
[step 1900] [GRAD STATS] total_norm=5.6717e-01 max_single=3.6148e-01 (segment_embed.weight)
|
| 47 |
+
[step 1900] step= 1900 | total=0.6022 | leaf=0.6022 | ancestor=0.0000 | lr=9.50e-05 | gnorm=5.6717e-01 | t=44.9s
|
| 48 |
+
[step 2000] [GRAD STATS] total_norm=4.1023e-01 max_single=2.4933e-01 (segment_embed.weight)
|
| 49 |
+
[step 2000] step= 2000 | total=0.6011 | leaf=0.6011 | ancestor=0.0000 | lr=1.00e-04 | gnorm=4.1023e-01 | t=45.1s
|
| 50 |
+
[step 2100] [GRAD STATS] total_norm=4.8075e-01 max_single=2.9903e-01 (segment_embed.weight)
|
| 51 |
+
[step 2100] step= 2100 | total=0.5738 | leaf=0.5738 | ancestor=0.0000 | lr=1.05e-04 | gnorm=4.8075e-01 | t=45.2s
|
| 52 |
+
[step 2200] [GRAD STATS] total_norm=4.0934e-01 max_single=2.3357e-01 (segment_embed.weight)
|
| 53 |
+
[step 2200] step= 2200 | total=0.5447 | leaf=0.5447 | ancestor=0.0000 | lr=1.10e-04 | gnorm=4.0934e-01 | t=44.9s
|
| 54 |
+
[step 2300] [GRAD STATS] total_norm=4.2533e-01 max_single=2.5993e-01 (segment_embed.weight)
|
| 55 |
+
[step 2300] step= 2300 | total=0.5140 | leaf=0.5140 | ancestor=0.0000 | lr=1.15e-04 | gnorm=4.2533e-01 | t=44.9s
|
| 56 |
+
[step 2400] [GRAD STATS] total_norm=3.8359e-01 max_single=2.2847e-01 (segment_embed.weight)
|
| 57 |
+
[step 2400] step= 2400 | total=0.5202 | leaf=0.5202 | ancestor=0.0000 | lr=1.20e-04 | gnorm=3.8359e-01 | t=44.9s
|
| 58 |
+
[step 2500] [GRAD STATS] total_norm=3.4170e-01 max_single=2.0452e-01 (segment_embed.weight)
|
| 59 |
+
[step 2500] step= 2500 | total=0.5314 | leaf=0.5314 | ancestor=0.0000 | lr=1.25e-04 | gnorm=3.4170e-01 | t=45.6s
|
| 60 |
+
[step 2600] [GRAD STATS] total_norm=3.2129e-01 max_single=1.8665e-01 (segment_embed.weight)
|
| 61 |
+
[step 2600] step= 2600 | total=0.5016 | leaf=0.5016 | ancestor=0.0000 | lr=1.30e-04 | gnorm=3.2129e-01 | t=45.3s
|
| 62 |
+
[step 2700] [GRAD STATS] total_norm=2.7893e-01 max_single=1.5089e-01 (segment_embed.weight)
|
| 63 |
+
[step 2700] step= 2700 | total=0.5099 | leaf=0.5099 | ancestor=0.0000 | lr=1.35e-04 | gnorm=2.7893e-01 | t=44.9s
|
| 64 |
+
[step 2800] [GRAD STATS] total_norm=3.2715e-01 max_single=2.0284e-01 (segment_embed.weight)
|
| 65 |
+
[step 2800] step= 2800 | total=0.5001 | leaf=0.5001 | ancestor=0.0000 | lr=1.40e-04 | gnorm=3.2715e-01 | t=45.1s
|
| 66 |
+
[step 2900] [GRAD STATS] total_norm=3.5657e-01 max_single=2.2850e-01 (segment_embed.weight)
|
| 67 |
+
[step 2900] step= 2900 | total=0.5003 | leaf=0.5003 | ancestor=0.0000 | lr=1.45e-04 | gnorm=3.5657e-01 | t=44.9s
|
| 68 |
+
File "/home/sunhc/diffusion/sad/scripts/train.py", line 707, in <module>
|
| 69 |
+
main()
|
| 70 |
+
File "/home/sunhc/diffusion/sad/scripts/train.py", line 610, in main
|
| 71 |
+
loss.backward()
|
| 72 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/_tensor.py", line 626, in backward
|
| 73 |
+
torch.autograd.backward(
|
| 74 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 75 |
+
_engine_run_backward(
|
| 76 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/graph.py", line 823, in _engine_run_backward
|
| 77 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 78 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 79 |
+
KeyboardInterrupt
|
| 80 |
+
[rank0]: Traceback (most recent call last):
|
| 81 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train.py", line 707, in <module>
|
| 82 |
+
[rank0]: main()
|
| 83 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train.py", line 610, in main
|
| 84 |
+
[rank0]: loss.backward()
|
| 85 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/_tensor.py", line 626, in backward
|
| 86 |
+
[rank0]: torch.autograd.backward(
|
| 87 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 88 |
+
[rank0]: _engine_run_backward(
|
| 89 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/graph.py", line 823, in _engine_run_backward
|
| 90 |
+
[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 91 |
+
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 92 |
+
[rank0]: KeyboardInterrupt
|
wandb/run-20260423_045157-714fdnxz/logs/debug-core.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T04:51:57.459337527Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_8zbx9lq/port-909865.txt","pid":909865,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T04:51:57.459865832Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":909865}
|
| 3 |
+
{"time":"2026-04-23T04:51:57.45986366Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-909865-915261-3324972582/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T04:51:57.640929171Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T04:51:57.648322906Z","level":"INFO","msg":"handleInformInit: received","streamId":"714fdnxz","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T04:51:58.503714998Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"714fdnxz","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T04:52:04.378940364Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"h25q404n7uy4"}
|
| 8 |
+
{"time":"2026-04-23T05:14:20.239651306Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-23T05:14:20.239698972Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-04-23T05:14:20.239725574Z","level":"INFO","msg":"server is shutting down"}
|
| 11 |
+
{"time":"2026-04-23T05:14:20.239761216Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-23T05:14:20.239940086Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-909865-915261-3324972582/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-23T05:14:35.108632564Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
wandb/run-20260423_045157-714fdnxz/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T04:51:57.648541989Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-23T04:51:57.651282684Z","level":"INFO","msg":"stream: starting","core version":"0.25.1"}
|
| 3 |
+
{"time":"2026-04-23T04:51:58.503594145Z","level":"INFO","msg":"stream: created new stream","id":"714fdnxz"}
|
| 4 |
+
{"time":"2026-04-23T04:51:58.503709017Z","level":"INFO","msg":"stream: started"}
|
| 5 |
+
{"time":"2026-04-23T04:51:58.503752979Z","level":"INFO","msg":"sender: started"}
|
| 6 |
+
{"time":"2026-04-23T04:51:58.503770708Z","level":"INFO","msg":"handler: started"}
|
| 7 |
+
{"time":"2026-04-23T04:51:58.50377889Z","level":"INFO","msg":"writer: started","stream_id":"714fdnxz"}
|
| 8 |
+
{"time":"2026-04-23T04:51:59.358135076Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-04-23T04:51:59.875420663Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-23T04:52:14.358905418Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":9}
|
| 11 |
+
{"time":"2026-04-23T04:52:14.961267899Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-23T04:52:29.358065313Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":2,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 13 |
+
{"time":"2026-04-23T04:52:29.790649579Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-04-23T04:52:44.358550607Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 15 |
+
{"time":"2026-04-23T04:52:44.970628083Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-04-23T04:52:59.358740751Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":6,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 17 |
+
{"time":"2026-04-23T04:52:59.789632684Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-04-23T04:53:14.358747658Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 19 |
+
{"time":"2026-04-23T04:53:14.790689098Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 20 |
+
{"time":"2026-04-23T04:53:29.358768295Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":10,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 21 |
+
{"time":"2026-04-23T04:53:29.896539212Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 22 |
+
{"time":"2026-04-23T04:53:44.358492423Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":12,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 23 |
+
{"time":"2026-04-23T04:53:44.805556891Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 24 |
+
{"time":"2026-04-23T04:53:59.358227287Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 25 |
+
{"time":"2026-04-23T04:53:59.870478229Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 26 |
+
{"time":"2026-04-23T04:54:14.358330328Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 27 |
+
{"time":"2026-04-23T04:54:14.889838789Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 28 |
+
{"time":"2026-04-23T04:54:29.358457594Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":7,"console_lines":1,"uploaded_len":2}
|
| 29 |
+
{"time":"2026-04-23T04:54:29.964511604Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 30 |
+
{"time":"2026-04-23T04:54:44.358180583Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 31 |
+
{"time":"2026-04-23T04:54:44.744647289Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 32 |
+
{"time":"2026-04-23T04:54:59.358700445Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 33 |
+
{"time":"2026-04-23T04:54:59.702292405Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 34 |
+
{"time":"2026-04-23T04:55:14.358181064Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":24,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 35 |
+
{"time":"2026-04-23T04:55:14.863494886Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 36 |
+
{"time":"2026-04-23T04:55:29.358264282Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":26,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 37 |
+
{"time":"2026-04-23T04:55:29.849304621Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 38 |
+
{"time":"2026-04-23T04:55:44.358091231Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":28,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 39 |
+
{"time":"2026-04-23T04:55:44.860114145Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 40 |
+
{"time":"2026-04-23T04:55:59.358475479Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":30,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 41 |
+
{"time":"2026-04-23T04:55:59.784390869Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 42 |
+
{"time":"2026-04-23T04:56:14.358207835Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":32,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 43 |
+
{"time":"2026-04-23T04:56:14.792284283Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 44 |
+
{"time":"2026-04-23T04:56:29.35804993Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 45 |
+
{"time":"2026-04-23T04:56:29.862147511Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 46 |
+
{"time":"2026-04-23T04:56:44.358891707Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":36,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 47 |
+
{"time":"2026-04-23T04:56:45.066502873Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 48 |
+
{"time":"2026-04-23T04:56:59.358812246Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":38,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 49 |
+
{"time":"2026-04-23T04:56:59.854386095Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 50 |
+
{"time":"2026-04-23T04:57:14.358553414Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":40,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 51 |
+
{"time":"2026-04-23T04:57:14.870346943Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 52 |
+
{"time":"2026-04-23T04:57:29.358275226Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 53 |
+
{"time":"2026-04-23T04:57:29.898982754Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 54 |
+
{"time":"2026-04-23T04:57:44.358107345Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":44,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 55 |
+
{"time":"2026-04-23T04:57:44.916736436Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 56 |
+
{"time":"2026-04-23T04:57:59.358417423Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":46,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 57 |
+
{"time":"2026-04-23T04:57:59.787273492Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 58 |
+
{"time":"2026-04-23T04:58:14.358255089Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":48,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 59 |
+
{"time":"2026-04-23T04:58:14.896792579Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 60 |
+
{"time":"2026-04-23T04:58:29.358103807Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":50,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 61 |
+
{"time":"2026-04-23T04:58:29.778318534Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 62 |
+
{"time":"2026-04-23T04:58:44.358290121Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":52,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 63 |
+
{"time":"2026-04-23T04:58:44.759310223Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 64 |
+
{"time":"2026-04-23T04:58:59.358268489Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":54,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 65 |
+
{"time":"2026-04-23T04:58:59.923783145Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 66 |
+
{"time":"2026-04-23T04:59:14.358537459Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":56,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 67 |
+
{"time":"2026-04-23T04:59:14.791905198Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 68 |
+
{"time":"2026-04-23T04:59:29.358262824Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":58,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 69 |
+
{"time":"2026-04-23T04:59:29.781407367Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 70 |
+
{"time":"2026-04-23T04:59:44.358605254Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":60,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 71 |
+
{"time":"2026-04-23T04:59:44.933252565Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 72 |
+
{"time":"2026-04-23T04:59:59.358607791Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":62,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 73 |
+
{"time":"2026-04-23T04:59:59.892848856Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 74 |
+
{"time":"2026-04-23T05:00:14.358213516Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":64,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 75 |
+
{"time":"2026-04-23T05:00:14.829088899Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 76 |
+
{"time":"2026-04-23T05:00:29.35847279Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 77 |
+
{"time":"2026-04-23T05:00:30.176949615Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 78 |
+
{"time":"2026-04-23T05:00:44.358670608Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":68,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 79 |
+
{"time":"2026-04-23T05:00:44.843471497Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 80 |
+
{"time":"2026-04-23T05:00:59.358531473Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":70,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 81 |
+
{"time":"2026-04-23T05:00:59.770458893Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 82 |
+
{"time":"2026-04-23T05:01:14.358378564Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":72,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 83 |
+
{"time":"2026-04-23T05:01:14.763677859Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 84 |
+
{"time":"2026-04-23T05:01:29.358231626Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":74,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 85 |
+
{"time":"2026-04-23T05:01:29.830549321Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 86 |
+
{"time":"2026-04-23T05:01:44.358248499Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":76,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 87 |
+
{"time":"2026-04-23T05:01:44.776156691Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 88 |
+
{"time":"2026-04-23T05:01:59.359078537Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":78,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 89 |
+
{"time":"2026-04-23T05:01:59.812193481Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 90 |
+
{"time":"2026-04-23T05:02:14.358339086Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":80,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 91 |
+
{"time":"2026-04-23T05:02:14.784128112Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 92 |
+
{"time":"2026-04-23T05:02:29.358603021Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":82,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 93 |
+
{"time":"2026-04-23T05:02:29.86082627Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 94 |
+
{"time":"2026-04-23T05:02:44.358658364Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":84,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 95 |
+
{"time":"2026-04-23T05:02:44.978133504Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 96 |
+
{"time":"2026-04-23T05:02:59.358593669Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":86,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 97 |
+
{"time":"2026-04-23T05:02:59.801658544Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 98 |
+
{"time":"2026-04-23T05:03:14.358717674Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":88,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 99 |
+
{"time":"2026-04-23T05:03:14.757453208Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 100 |
+
{"time":"2026-04-23T05:03:29.358201366Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":90,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 101 |
+
{"time":"2026-04-23T05:03:29.858044023Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 102 |
+
{"time":"2026-04-23T05:03:44.35833559Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":92,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 103 |
+
{"time":"2026-04-23T05:03:44.779561006Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 104 |
+
{"time":"2026-04-23T05:03:59.358291393Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":94,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 105 |
+
{"time":"2026-04-23T05:03:59.736565984Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 106 |
+
{"time":"2026-04-23T05:04:14.358484909Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":96,"events_lines":2,"console_offset":7,"console_lines":1}
|
| 107 |
+
{"time":"2026-04-23T05:04:14.760597991Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 108 |
+
{"time":"2026-04-23T05:04:29.358517945Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":98,"events_lines":2,"console_offset":9,"console_lines":32}
|
| 109 |
+
{"time":"2026-04-23T05:04:29.765741059Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 110 |
+
{"time":"2026-04-23T05:04:44.358667755Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":100,"events_lines":2}
|
| 111 |
+
{"time":"2026-04-23T05:04:44.759001867Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 112 |
+
{"time":"2026-04-23T05:04:59.358682838Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":102,"events_lines":2,"console_offset":41,"console_lines":2}
|
| 113 |
+
{"time":"2026-04-23T05:04:59.824037082Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 114 |
+
{"time":"2026-04-23T05:05:14.358826302Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":104,"events_lines":2}
|
| 115 |
+
{"time":"2026-04-23T05:05:14.779888406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 116 |
+
{"time":"2026-04-23T05:05:29.358993156Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":106,"events_lines":2}
|
| 117 |
+
{"time":"2026-04-23T05:05:29.826493986Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 118 |
+
{"time":"2026-04-23T05:05:44.358299506Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":108,"events_lines":2,"console_offset":43,"console_lines":2}
|
| 119 |
+
{"time":"2026-04-23T05:05:44.756728493Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 120 |
+
{"time":"2026-04-23T05:05:59.35878821Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":110,"events_lines":2}
|
| 121 |
+
{"time":"2026-04-23T05:05:59.758819715Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 122 |
+
{"time":"2026-04-23T05:06:14.358300191Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":112,"events_lines":2}
|
| 123 |
+
{"time":"2026-04-23T05:06:14.949705291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 124 |
+
{"time":"2026-04-23T05:06:29.358695565Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":114,"events_lines":2,"console_offset":45,"console_lines":2}
|
| 125 |
+
{"time":"2026-04-23T05:06:29.874307467Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 126 |
+
{"time":"2026-04-23T05:06:44.358609259Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":116,"events_lines":2}
|
| 127 |
+
{"time":"2026-04-23T05:06:44.838865507Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 128 |
+
{"time":"2026-04-23T05:06:59.358596932Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":118,"events_lines":2}
|
| 129 |
+
{"time":"2026-04-23T05:06:59.742305112Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 130 |
+
{"time":"2026-04-23T05:07:14.359066225Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":120,"events_lines":2,"console_offset":47,"console_lines":2}
|
| 131 |
+
{"time":"2026-04-23T05:07:14.815042271Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 132 |
+
{"time":"2026-04-23T05:07:29.358740992Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":122,"events_lines":2}
|
| 133 |
+
{"time":"2026-04-23T05:07:29.764239675Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 134 |
+
{"time":"2026-04-23T05:07:44.358662667Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":124,"events_lines":2}
|
| 135 |
+
{"time":"2026-04-23T05:07:44.808747069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 136 |
+
{"time":"2026-04-23T05:07:59.358277087Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":126,"events_lines":2}
|
| 137 |
+
{"time":"2026-04-23T05:07:59.795286254Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 138 |
+
{"time":"2026-04-23T05:08:14.359052869Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":128,"events_lines":2,"console_offset":49,"console_lines":2}
|
| 139 |
+
{"time":"2026-04-23T05:08:14.810802378Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 140 |
+
{"time":"2026-04-23T05:08:29.358476368Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":130,"events_lines":2}
|
| 141 |
+
{"time":"2026-04-23T05:08:29.776030334Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 142 |
+
{"time":"2026-04-23T05:08:44.357973765Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":132,"events_lines":2}
|
| 143 |
+
{"time":"2026-04-23T05:08:44.779781314Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 144 |
+
{"time":"2026-04-23T05:08:59.358827872Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":134,"events_lines":2,"console_offset":51,"console_lines":2}
|
| 145 |
+
{"time":"2026-04-23T05:08:59.723068407Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 146 |
+
{"time":"2026-04-23T05:09:14.358843818Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":136,"events_lines":2}
|
| 147 |
+
{"time":"2026-04-23T05:09:14.701281837Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 148 |
+
{"time":"2026-04-23T05:09:29.358414343Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":138,"events_lines":2,"console_offset":53,"console_lines":2}
|
| 149 |
+
{"time":"2026-04-23T05:09:29.769783967Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 150 |
+
{"time":"2026-04-23T05:09:44.359054173Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":140,"events_lines":2}
|
| 151 |
+
{"time":"2026-04-23T05:09:44.813830799Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 152 |
+
{"time":"2026-04-23T05:09:59.358129661Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":142,"events_lines":2}
|
| 153 |
+
{"time":"2026-04-23T05:09:59.751416014Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 154 |
+
{"time":"2026-04-23T05:10:14.358626556Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":144,"events_lines":2,"console_offset":55,"console_lines":2}
|
| 155 |
+
{"time":"2026-04-23T05:10:14.825707067Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 156 |
+
{"time":"2026-04-23T05:10:29.358153045Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":146,"events_lines":2}
|
| 157 |
+
{"time":"2026-04-23T05:10:29.771601341Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 158 |
+
{"time":"2026-04-23T05:10:44.358933652Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":148,"events_lines":2}
|
| 159 |
+
{"time":"2026-04-23T05:10:44.717816728Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 160 |
+
{"time":"2026-04-23T05:10:59.358620135Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":150,"events_lines":2}
|
| 161 |
+
{"time":"2026-04-23T05:10:59.845267831Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 162 |
+
{"time":"2026-04-23T05:11:14.358755327Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":152,"events_lines":2,"console_offset":57,"console_lines":2}
|
| 163 |
+
{"time":"2026-04-23T05:11:14.807502625Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 164 |
+
{"time":"2026-04-23T05:11:29.359127505Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":154,"events_lines":2}
|
| 165 |
+
{"time":"2026-04-23T05:11:29.733381832Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 166 |
+
{"time":"2026-04-23T05:11:44.358062261Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":156,"events_lines":2}
|
| 167 |
+
{"time":"2026-04-23T05:11:44.770126267Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 168 |
+
{"time":"2026-04-23T05:11:59.358559511Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":158,"events_lines":2,"console_offset":59,"console_lines":2}
|
| 169 |
+
{"time":"2026-04-23T05:11:59.779121312Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 170 |
+
{"time":"2026-04-23T05:12:14.358130603Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":160,"events_lines":2}
|
| 171 |
+
{"time":"2026-04-23T05:12:14.812686762Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 172 |
+
{"time":"2026-04-23T05:12:29.358138169Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":162,"events_lines":2}
|
| 173 |
+
{"time":"2026-04-23T05:12:29.79415807Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 174 |
+
{"time":"2026-04-23T05:12:44.358216353Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":164,"events_lines":2,"console_offset":61,"console_lines":2}
|
| 175 |
+
{"time":"2026-04-23T05:12:44.830169058Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 176 |
+
{"time":"2026-04-23T05:12:59.358459464Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":166,"events_lines":2}
|
| 177 |
+
{"time":"2026-04-23T05:12:59.753391332Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 178 |
+
{"time":"2026-04-23T05:13:14.358402261Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":168,"events_lines":2}
|
| 179 |
+
{"time":"2026-04-23T05:13:14.775506941Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 180 |
+
{"time":"2026-04-23T05:13:29.35858134Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":170,"events_lines":2,"console_offset":63,"console_lines":2}
|
| 181 |
+
{"time":"2026-04-23T05:13:29.736597894Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 182 |
+
{"time":"2026-04-23T05:13:44.358463184Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":172,"events_lines":2}
|
| 183 |
+
{"time":"2026-04-23T05:13:44.769237018Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 184 |
+
{"time":"2026-04-23T05:13:59.358161633Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":174,"events_lines":2}
|
| 185 |
+
{"time":"2026-04-23T05:13:59.737856447Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 186 |
+
{"time":"2026-04-23T05:14:14.358864293Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":176,"events_lines":2,"console_offset":65,"console_lines":2}
|
| 187 |
+
{"time":"2026-04-23T05:14:14.764608099Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 188 |
+
{"time":"2026-04-23T05:14:20.239705503Z","level":"INFO","msg":"stream: closing"}
|
| 189 |
+
{"time":"2026-04-23T05:14:29.358462022Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":67,"console_lines":25}
|
| 190 |
+
{"time":"2026-04-23T05:14:29.767730802Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
wandb/run-20260423_045157-714fdnxz/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 04:51:57,423 INFO MainThread:909865 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-23 04:51:57,423 INFO MainThread:909865 [wandb_setup.py:_flush():81] Configure stats pid to 909865
|
| 3 |
+
2026-04-23 04:51:57,423 INFO MainThread:909865 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 04:51:57,423 INFO MainThread:909865 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260423_045157-714fdnxz/logs/debug.log
|
| 5 |
+
2026-04-23 04:51:57,423 INFO MainThread:909865 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260423_045157-714fdnxz/logs/debug-internal.log
|
| 6 |
+
2026-04-23 04:51:57,423 INFO MainThread:909865 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-23 04:51:57,423 INFO MainThread:909865 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 8, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top3_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0005, 'lr_min': 5e-05, 'warmup_steps': 10000, 'weight_decay': 0.02, 'grad_clip': 1.0, 'adam_betas': [0.9, 0.99], 'adam_eps': 1e-09, 'dtype': 'bf16', 'compile': 'default', 'curriculum_t_max_init': 0.3, 'curriculum_steps': 20000, 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_new_owt', 'save_dir': 'outputs/sad_new'}, '_wandb': {}}
|
| 9 |
+
2026-04-23 04:51:57,423 INFO MainThread:909865 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-23 04:51:57,641 INFO MainThread:909865 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-23 04:51:57,645 INFO MainThread:909865 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-23 04:51:57,647 INFO MainThread:909865 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-23 04:51:57,650 INFO MainThread:909865 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 04:51:59,226 INFO MainThread:909865 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-23 04:51:59,348 INFO MainThread:909865 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-23 04:51:59,348 INFO MainThread:909865 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-23 04:51:59,348 INFO MainThread:909865 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-23 04:51:59,348 INFO MainThread:909865 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-23 04:51:59,353 INFO MainThread:909865 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-23 05:14:20,239 INFO wandb-AsyncioManager-main:909865 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-23 05:14:20,239 INFO wandb-AsyncioManager-main:909865 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260423_051457-2j4bd54k/files/output.log
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
full-denoise training: 0%| | 0/500000 [00:00<?, ?it/s][rank0]:W0423 05:14:59.733000 952189 site-packages/torch/_logging/_internal.py:1089] [0/0] Profiler function <class 'torch.autograd.profiler.record_function'> will be ignored
|
| 2 |
+
[step 0] [GRAD STATS] total_norm=3.8375e-01 max_single=2.2608e-01 (vocab_embed.embedding)
|
| 3 |
+
full-denoise training: 0%|▏ | 1600/500000 [08:49<43:43:09, 3.17it/s, leaf=0.6629, lr=8.0e-05]
|
| 4 |
+
[step 0] step= 0 | total=1.6813 | leaf=1.6813 | ancestor=0.0000 | lr=0.00e+00 | gnorm=3.8375e-01 | t=22.8s
|
| 5 |
+
[step 100] [GRAD STATS] total_norm=5.7325e-01 max_single=2.4064e-01 (vocab_embed.embedding)
|
| 6 |
+
[step 100] step= 100 | total=1.5914 | leaf=1.5914 | ancestor=0.0000 | lr=5.00e-06 | gnorm=5.7325e-01 | t=31.4s
|
| 7 |
+
[step 200] [GRAD STATS] total_norm=2.9175e-01 max_single=2.6790e-01 (vocab_embed.embedding)
|
| 8 |
+
[step 200] step= 200 | total=1.3936 | leaf=1.3936 | ancestor=0.0000 | lr=1.00e-05 | gnorm=2.9175e-01 | t=31.5s
|
| 9 |
+
[step 300] [GRAD STATS] total_norm=1.9524e-01 max_single=1.8196e-01 (vocab_embed.embedding)
|
| 10 |
+
[step 300] step= 300 | total=1.2734 | leaf=1.2734 | ancestor=0.0000 | lr=1.50e-05 | gnorm=1.9524e-01 | t=31.5s
|
| 11 |
+
[step 400] [GRAD STATS] total_norm=1.2036e-01 max_single=1.0447e-01 (vocab_embed.embedding)
|
| 12 |
+
[step 400] step= 400 | total=1.0994 | leaf=1.0994 | ancestor=0.0000 | lr=2.00e-05 | gnorm=1.2036e-01 | t=31.5s
|
| 13 |
+
[step 500] [GRAD STATS] total_norm=9.7457e-02 max_single=5.5912e-02 (vocab_embed.embedding)
|
| 14 |
+
[step 500] step= 500 | total=1.0670 | leaf=1.0670 | ancestor=0.0000 | lr=2.50e-05 | gnorm=9.7457e-02 | t=31.5s
|
| 15 |
+
[step 600] [GRAD STATS] total_norm=2.2821e-01 max_single=1.3099e-01 (segment_embed.weight)
|
| 16 |
+
[step 600] step= 600 | total=0.9898 | leaf=0.9898 | ancestor=0.0000 | lr=3.00e-05 | gnorm=2.2821e-01 | t=31.8s
|
| 17 |
+
[step 700] [GRAD STATS] total_norm=1.1330e+00 max_single=7.0410e-01 (segment_embed.weight)
|
| 18 |
+
[step 700] step= 700 | total=0.9614 | leaf=0.9614 | ancestor=0.0000 | lr=3.50e-05 | gnorm=1.1330e+00 | t=31.6s
|
| 19 |
+
[step 800] [GRAD STATS] total_norm=2.0460e-01 max_single=9.0138e-02 (cond_bias)
|
| 20 |
+
[step 800] step= 800 | total=0.9005 | leaf=0.9005 | ancestor=0.0000 | lr=4.00e-05 | gnorm=2.0460e-01 | t=32.3s
|
| 21 |
+
[step 900] [GRAD STATS] total_norm=3.3391e-01 max_single=2.0368e-01 (segment_embed.weight)
|
| 22 |
+
[step 900] step= 900 | total=0.8354 | leaf=0.8354 | ancestor=0.0000 | lr=4.50e-05 | gnorm=3.3391e-01 | t=32.0s
|
| 23 |
+
[step 1000] [GRAD STATS] total_norm=9.0231e-01 max_single=5.7044e-01 (segment_embed.weight)
|
| 24 |
+
[step 1000] step= 1000 | total=0.8049 | leaf=0.8049 | ancestor=0.0000 | lr=5.00e-05 | gnorm=9.0231e-01 | t=31.7s
|
| 25 |
+
[step 1100] [GRAD STATS] total_norm=5.9149e-01 max_single=3.3861e-01 (segment_embed.weight)
|
| 26 |
+
[step 1100] step= 1100 | total=0.7553 | leaf=0.7553 | ancestor=0.0000 | lr=5.50e-05 | gnorm=5.9149e-01 | t=31.6s
|
| 27 |
+
[step 1200] [GRAD STATS] total_norm=5.4244e-01 max_single=3.1106e-01 (segment_embed.weight)
|
| 28 |
+
[step 1200] step= 1200 | total=0.7127 | leaf=0.7127 | ancestor=0.0000 | lr=6.00e-05 | gnorm=5.4244e-01 | t=31.7s
|
| 29 |
+
[step 1300] [GRAD STATS] total_norm=5.3178e-01 max_single=2.8593e-01 (segment_embed.weight)
|
| 30 |
+
[step 1300] step= 1300 | total=0.7212 | leaf=0.7212 | ancestor=0.0000 | lr=6.50e-05 | gnorm=5.3178e-01 | t=31.6s
|
| 31 |
+
[step 1400] [GRAD STATS] total_norm=4.2365e-01 max_single=2.3590e-01 (segment_embed.weight)
|
| 32 |
+
[step 1400] step= 1400 | total=0.6981 | leaf=0.6981 | ancestor=0.0000 | lr=7.00e-05 | gnorm=4.2365e-01 | t=31.6s
|
| 33 |
+
[step 1500] [GRAD STATS] total_norm=7.9471e-01 max_single=4.8411e-01 (segment_embed.weight)
|
| 34 |
+
[step 1500] step= 1500 | total=0.7044 | leaf=0.7044 | ancestor=0.0000 | lr=7.50e-05 | gnorm=7.9471e-01 | t=31.6s
|
| 35 |
+
[step 1600] [GRAD STATS] total_norm=4.9281e-01 max_single=2.9299e-01 (segment_embed.weight)
|
| 36 |
+
[step 1600] step= 1600 | total=0.7030 | leaf=0.7030 | ancestor=0.0000 | lr=8.00e-05 | gnorm=4.9281e-01 | t=32.1s
|
| 37 |
+
[step 1700] [GRAD STATS] total_norm=3.9567e-01 max_single=2.2474e-01 (segment_embed.weight)
|
| 38 |
+
[step 1700] step= 1700 | total=0.6545 | leaf=0.6545 | ancestor=0.0000 | lr=8.50e-05 | gnorm=3.9567e-01 | t=31.9s
|
| 39 |
+
[step 1800] [GRAD STATS] total_norm=3.8288e-01 max_single=2.1744e-01 (segment_embed.weight)
|
| 40 |
+
[step 1800] step= 1800 | total=0.6025 | leaf=0.6025 | ancestor=0.0000 | lr=9.00e-05 | gnorm=3.8288e-01 | t=31.6s
|
| 41 |
+
[step 1900] [GRAD STATS] total_norm=5.0733e-01 max_single=3.2102e-01 (segment_embed.weight)
|
| 42 |
+
[step 1900] step= 1900 | total=0.6266 | leaf=0.6266 | ancestor=0.0000 | lr=9.50e-05 | gnorm=5.0733e-01 | t=31.6s
|
| 43 |
+
[step 2000] [GRAD STATS] total_norm=4.5187e-01 max_single=2.8643e-01 (segment_embed.weight)
|
| 44 |
+
[step 2000] step= 2000 | total=0.6317 | leaf=0.6317 | ancestor=0.0000 | lr=1.00e-04 | gnorm=4.5187e-01 | t=31.6s
|
| 45 |
+
[step 2100] [GRAD STATS] total_norm=4.0677e-01 max_single=2.3428e-01 (segment_embed.weight)
|
| 46 |
+
[step 2100] step= 2100 | total=0.6077 | leaf=0.6077 | ancestor=0.0000 | lr=1.05e-04 | gnorm=4.0677e-01 | t=31.6s
|
| 47 |
+
[step 2200] [GRAD STATS] total_norm=4.4824e-01 max_single=2.8548e-01 (segment_embed.weight)
|
| 48 |
+
[step 2200] step= 2200 | total=0.5827 | leaf=0.5827 | ancestor=0.0000 | lr=1.10e-04 | gnorm=4.4824e-01 | t=31.6s
|
| 49 |
+
[step 2300] [GRAD STATS] total_norm=3.9318e-01 max_single=2.3701e-01 (segment_embed.weight)
|
| 50 |
+
[step 2300] step= 2300 | total=0.5550 | leaf=0.5550 | ancestor=0.0000 | lr=1.15e-04 | gnorm=3.9318e-01 | t=32.7s
|
| 51 |
+
[step 2400] [GRAD STATS] total_norm=3.4369e-01 max_single=1.9185e-01 (segment_embed.weight)
|
| 52 |
+
[step 2400] step= 2400 | total=0.5611 | leaf=0.5611 | ancestor=0.0000 | lr=1.20e-04 | gnorm=3.4369e-01 | t=31.9s
|
| 53 |
+
[step 2500] [GRAD STATS] total_norm=5.1486e-01 max_single=3.4635e-01 (segment_embed.weight)
|
| 54 |
+
[step 2500] step= 2500 | total=0.5751 | leaf=0.5751 | ancestor=0.0000 | lr=1.25e-04 | gnorm=5.1486e-01 | t=31.6s
|
| 55 |
+
[step 2600] [GRAD STATS] total_norm=4.5317e-01 max_single=2.9054e-01 (segment_embed.weight)
|
| 56 |
+
[step 2600] step= 2600 | total=0.5397 | leaf=0.5397 | ancestor=0.0000 | lr=1.30e-04 | gnorm=4.5317e-01 | t=31.6s
|
| 57 |
+
[step 2700] [GRAD STATS] total_norm=3.8298e-01 max_single=2.2755e-01 (segment_embed.weight)
|
| 58 |
+
[step 2700] step= 2700 | total=0.5411 | leaf=0.5411 | ancestor=0.0000 | lr=1.35e-04 | gnorm=3.8298e-01 | t=31.7s
|
| 59 |
+
[step 2800] [GRAD STATS] total_norm=3.7867e-01 max_single=2.2780e-01 (segment_embed.weight)
|
| 60 |
+
[step 2800] step= 2800 | total=0.5302 | leaf=0.5302 | ancestor=0.0000 | lr=1.40e-04 | gnorm=3.7867e-01 | t=32.0s
|
| 61 |
+
[step 2900] [GRAD STATS] total_norm=3.8120e-01 max_single=2.3717e-01 (segment_embed.weight)
|
| 62 |
+
[step 2900] step= 2900 | total=0.5202 | leaf=0.5202 | ancestor=0.0000 | lr=1.45e-04 | gnorm=3.8120e-01 | t=31.6s
|
| 63 |
+
[step 3000] [GRAD STATS] total_norm=4.1739e-01 max_single=2.7847e-01 (segment_embed.weight)
|
| 64 |
+
[step 3000] step= 3000 | total=0.5416 | leaf=0.5416 | ancestor=0.0000 | lr=1.50e-04 | gnorm=4.1739e-01 | t=31.7s
|
| 65 |
+
[step 3100] [GRAD STATS] total_norm=3.0974e-01 max_single=1.9299e-01 (segment_embed.weight)
|
| 66 |
+
[step 3100] step= 3100 | total=0.5075 | leaf=0.5075 | ancestor=0.0000 | lr=1.55e-04 | gnorm=3.0974e-01 | t=31.7s
|
| 67 |
+
[step 3200] [GRAD STATS] total_norm=4.1345e-01 max_single=2.9809e-01 (segment_embed.weight)
|
| 68 |
+
[step 3200] step= 3200 | total=0.5135 | leaf=0.5135 | ancestor=0.0000 | lr=1.60e-04 | gnorm=4.1345e-01 | t=32.0s
|
| 69 |
+
[step 3300] [GRAD STATS] total_norm=3.4848e-01 max_single=2.3292e-01 (segment_embed.weight)
|
| 70 |
+
[step 3300] step= 3300 | total=0.5216 | leaf=0.5216 | ancestor=0.0000 | lr=1.65e-04 | gnorm=3.4848e-01 | t=31.7s
|
| 71 |
+
[step 3400] [GRAD STATS] total_norm=2.6366e-01 max_single=1.4751e-01 (segment_embed.weight)
|
| 72 |
+
[step 3400] step= 3400 | total=0.5294 | leaf=0.5294 | ancestor=0.0000 | lr=1.70e-04 | gnorm=2.6366e-01 | t=31.6s
|
| 73 |
+
[step 3500] [GRAD STATS] total_norm=3.1788e-01 max_single=2.1557e-01 (segment_embed.weight)
|
| 74 |
+
[step 3500] step= 3500 | total=0.5041 | leaf=0.5041 | ancestor=0.0000 | lr=1.75e-04 | gnorm=3.1788e-01 | t=31.6s
|
| 75 |
+
[step 3600] [GRAD STATS] total_norm=2.4893e-01 max_single=1.4053e-01 (segment_embed.weight)
|
| 76 |
+
[step 3600] step= 3600 | total=0.5457 | leaf=0.5457 | ancestor=0.0000 | lr=1.80e-04 | gnorm=2.4893e-01 | t=31.6s
|
| 77 |
+
[step 3700] [GRAD STATS] total_norm=2.8741e-01 max_single=1.8059e-01 (segment_embed.weight)
|
| 78 |
+
[step 3700] step= 3700 | total=0.5292 | leaf=0.5292 | ancestor=0.0000 | lr=1.85e-04 | gnorm=2.8741e-01 | t=31.6s
|
| 79 |
+
[step 3800] [GRAD STATS] total_norm=2.5177e-01 max_single=1.5015e-01 (segment_embed.weight)
|
| 80 |
+
[step 3800] step= 3800 | total=0.5326 | leaf=0.5326 | ancestor=0.0000 | lr=1.90e-04 | gnorm=2.5177e-01 | t=32.0s
|
| 81 |
+
[step 3900] [GRAD STATS] total_norm=2.9774e-01 max_single=2.0344e-01 (segment_embed.weight)
|
| 82 |
+
[step 3900] step= 3900 | total=0.5106 | leaf=0.5106 | ancestor=0.0000 | lr=1.95e-04 | gnorm=2.9774e-01 | t=31.7s
|
| 83 |
+
[step 4000] [GRAD STATS] total_norm=2.7286e-01 max_single=1.6606e-01 (segment_embed.weight)
|
| 84 |
+
[step 4000] step= 4000 | total=0.5307 | leaf=0.5307 | ancestor=0.0000 | lr=2.00e-04 | gnorm=2.7286e-01 | t=31.9s
|
| 85 |
+
[step 4100] [GRAD STATS] total_norm=2.4372e-01 max_single=1.4403e-01 (segment_embed.weight)
|
| 86 |
+
[step 4100] step= 4100 | total=0.5138 | leaf=0.5138 | ancestor=0.0000 | lr=2.05e-04 | gnorm=2.4372e-01 | t=31.7s
|
| 87 |
+
[step 4200] [GRAD STATS] total_norm=2.2622e-01 max_single=1.2350e-01 (segment_embed.weight)
|
| 88 |
+
[step 4200] step= 4200 | total=0.4899 | leaf=0.4899 | ancestor=0.0000 | lr=2.10e-04 | gnorm=2.2622e-01 | t=33.1s
|
| 89 |
+
[step 4300] [GRAD STATS] total_norm=2.2942e-01 max_single=1.2473e-01 (segment_embed.weight)
|
| 90 |
+
[step 4300] step= 4300 | total=0.5253 | leaf=0.5253 | ancestor=0.0000 | lr=2.15e-04 | gnorm=2.2942e-01 | t=32.0s
|
| 91 |
+
[step 4400] [GRAD STATS] total_norm=3.0138e-01 max_single=2.0224e-01 (segment_embed.weight)
|
| 92 |
+
[step 4400] step= 4400 | total=0.5312 | leaf=0.5312 | ancestor=0.0000 | lr=2.20e-04 | gnorm=3.0138e-01 | t=31.7s
|
| 93 |
+
[step 4500] [GRAD STATS] total_norm=2.3245e-01 max_single=1.3552e-01 (segment_embed.weight)
|
| 94 |
+
[step 4500] step= 4500 | total=0.5582 | leaf=0.5582 | ancestor=0.0000 | lr=2.25e-04 | gnorm=2.3245e-01 | t=31.8s
|
| 95 |
+
[step 4600] [GRAD STATS] total_norm=2.3066e-01 max_single=1.3418e-01 (segment_embed.weight)
|
| 96 |
+
[step 4600] step= 4600 | total=0.5631 | leaf=0.5631 | ancestor=0.0000 | lr=2.30e-04 | gnorm=2.3066e-01 | t=31.7s
|
| 97 |
+
[step 4700] [GRAD STATS] total_norm=2.3045e-01 max_single=1.3182e-01 (segment_embed.weight)
|
| 98 |
+
[step 4700] step= 4700 | total=0.5261 | leaf=0.5261 | ancestor=0.0000 | lr=2.35e-04 | gnorm=2.3045e-01 | t=31.9s
|
| 99 |
+
[step 4800] [GRAD STATS] total_norm=2.9945e-01 max_single=2.0506e-01 (segment_embed.weight)
|
| 100 |
+
[step 4800] step= 4800 | total=0.5471 | leaf=0.5471 | ancestor=0.0000 | lr=2.40e-04 | gnorm=2.9945e-01 | t=31.7s
|
| 101 |
+
[step 4900] [GRAD STATS] total_norm=3.3554e-01 max_single=2.4435e-01 (segment_embed.weight)
|
| 102 |
+
[step 4900] step= 4900 | total=0.5706 | leaf=0.5706 | ancestor=0.0000 | lr=2.45e-04 | gnorm=3.3554e-01 | t=31.7s
|
| 103 |
+
[step 5000] [GRAD STATS] total_norm=2.4071e-01 max_single=1.4057e-01 (segment_embed.weight)
|
| 104 |
+
[step 5000] step= 5000 | total=0.5435 | leaf=0.5435 | ancestor=0.0000 | lr=2.50e-04 | gnorm=2.4071e-01 | t=32.7s
|
| 105 |
+
torch._dynamo.utils.warn_once(msg)
|
| 106 |
+
VAL | loss_leaf=2.1396 | loss_ancestor=0.0000 | loss_total=2.1396
|
| 107 |
+
full-denoise training: 1%|▋ | 6601/500000 [35:45<43:26:27, 3.15it/s, leaf=0.5676, lr=3.3e-04]
|
| 108 |
+
[step 5100] [GRAD STATS] total_norm=2.5772e-01 max_single=1.6444e-01 (segment_embed.weight)
|
| 109 |
+
[step 5100] step= 5100 | total=0.5735 | leaf=0.5735 | ancestor=0.0000 | lr=2.55e-04 | gnorm=2.5772e-01 | t=48.1s
|
| 110 |
+
[step 5200] [GRAD STATS] total_norm=1.9102e-01 max_single=1.0036e-01 (segment_embed.weight)
|
| 111 |
+
[step 5200] step= 5200 | total=0.5337 | leaf=0.5337 | ancestor=0.0000 | lr=2.60e-04 | gnorm=1.9102e-01 | t=31.7s
|
| 112 |
+
[step 5300] [GRAD STATS] total_norm=2.2061e-01 max_single=1.3324e-01 (segment_embed.weight)
|
| 113 |
+
[step 5300] step= 5300 | total=0.5272 | leaf=0.5272 | ancestor=0.0000 | lr=2.65e-04 | gnorm=2.2061e-01 | t=32.9s
|
| 114 |
+
[step 5400] [GRAD STATS] total_norm=2.0995e-01 max_single=1.2244e-01 (segment_embed.weight)
|
| 115 |
+
[step 5400] step= 5400 | total=0.5722 | leaf=0.5722 | ancestor=0.0000 | lr=2.70e-04 | gnorm=2.0995e-01 | t=31.7s
|
| 116 |
+
[step 5500] [GRAD STATS] total_norm=1.8527e-01 max_single=9.3596e-02 (segment_embed.weight)
|
| 117 |
+
[step 5500] step= 5500 | total=0.5826 | leaf=0.5826 | ancestor=0.0000 | lr=2.75e-04 | gnorm=1.8527e-01 | t=31.7s
|
| 118 |
+
[step 5600] [GRAD STATS] total_norm=1.9657e-01 max_single=1.0957e-01 (segment_embed.weight)
|
| 119 |
+
[step 5600] step= 5600 | total=0.5362 | leaf=0.5362 | ancestor=0.0000 | lr=2.80e-04 | gnorm=1.9657e-01 | t=32.1s
|
| 120 |
+
[step 5700] [GRAD STATS] total_norm=1.8602e-01 max_single=9.9302e-02 (segment_embed.weight)
|
| 121 |
+
[step 5700] step= 5700 | total=0.5668 | leaf=0.5668 | ancestor=0.0000 | lr=2.85e-04 | gnorm=1.8602e-01 | t=33.1s
|
| 122 |
+
[step 5800] [GRAD STATS] total_norm=2.0725e-01 max_single=1.1564e-01 (segment_embed.weight)
|
| 123 |
+
[step 5800] step= 5800 | total=0.5892 | leaf=0.5892 | ancestor=0.0000 | lr=2.90e-04 | gnorm=2.0725e-01 | t=31.7s
|
| 124 |
+
[step 5900] [GRAD STATS] total_norm=1.8535e-01 max_single=9.2532e-02 (segment_embed.weight)
|
| 125 |
+
[step 5900] step= 5900 | total=0.5578 | leaf=0.5578 | ancestor=0.0000 | lr=2.95e-04 | gnorm=1.8535e-01 | t=31.8s
|
| 126 |
+
[step 6000] [GRAD STATS] total_norm=1.7251e-01 max_single=7.6364e-02 (segment_embed.weight)
|
| 127 |
+
[step 6000] step= 6000 | total=0.5609 | leaf=0.5609 | ancestor=0.0000 | lr=3.00e-04 | gnorm=1.7251e-01 | t=31.7s
|
| 128 |
+
[step 6100] [GRAD STATS] total_norm=2.1286e-01 max_single=1.3338e-01 (segment_embed.weight)
|
| 129 |
+
[step 6100] step= 6100 | total=0.5596 | leaf=0.5596 | ancestor=0.0000 | lr=3.05e-04 | gnorm=2.1286e-01 | t=31.7s
|
| 130 |
+
[step 6200] [GRAD STATS] total_norm=1.7902e-01 max_single=9.1203e-02 (segment_embed.weight)
|
| 131 |
+
[step 6200] step= 6200 | total=0.5846 | leaf=0.5846 | ancestor=0.0000 | lr=3.10e-04 | gnorm=1.7902e-01 | t=31.7s
|
| 132 |
+
[step 6300] [GRAD STATS] total_norm=2.1555e-01 max_single=1.4103e-01 (segment_embed.weight)
|
| 133 |
+
[step 6300] step= 6300 | total=0.5612 | leaf=0.5612 | ancestor=0.0000 | lr=3.15e-04 | gnorm=2.1555e-01 | t=31.7s
|
| 134 |
+
[step 6400] [GRAD STATS] total_norm=1.9477e-01 max_single=1.1795e-01 (segment_embed.weight)
|
| 135 |
+
[step 6400] step= 6400 | total=0.5972 | leaf=0.5972 | ancestor=0.0000 | lr=3.20e-04 | gnorm=1.9477e-01 | t=36.2s
|
| 136 |
+
[step 6500] [GRAD STATS] total_norm=1.6090e-01 max_single=7.1319e-02 (segment_embed.weight)
|
| 137 |
+
[step 6500] step= 6500 | total=0.6105 | leaf=0.6105 | ancestor=0.0000 | lr=3.25e-04 | gnorm=1.6090e-01 | t=33.9s
|
| 138 |
+
[step 6600] [GRAD STATS] total_norm=2.1018e-01 max_single=1.3145e-01 (segment_embed.weight)
|
| 139 |
+
[step 6600] step= 6600 | total=0.5676 | leaf=0.5676 | ancestor=0.0000 | lr=3.30e-04 | gnorm=2.1018e-01 | t=32.3s
|
| 140 |
+
[step 6700] [GRAD STATS] total_norm=1.6667e-01 max_single=8.2727e-02 (segment_embed.weight)
|
| 141 |
+
[step 6700] step= 6700 | total=0.5937 | leaf=0.5937 | ancestor=0.0000 | lr=3.35e-04 | gnorm=1.6667e-01 | t=31.7s
|
| 142 |
+
[step 6800] [GRAD STATS] total_norm=1.7295e-01 max_single=8.7372e-02 (segment_embed.weight)
|
| 143 |
+
[step 6800] step= 6800 | total=0.5936 | leaf=0.5936 | ancestor=0.0000 | lr=3.40e-04 | gnorm=1.7295e-01 | t=34.9s
|
| 144 |
+
[step 6900] [GRAD STATS] total_norm=1.8082e-01 max_single=1.0605e-01 (segment_embed.weight)
|
| 145 |
+
[step 6900] step= 6900 | total=0.5847 | leaf=0.5847 | ancestor=0.0000 | lr=3.45e-04 | gnorm=1.8082e-01 | t=31.7s
|
| 146 |
+
[step 7000] [GRAD STATS] total_norm=1.6916e-01 max_single=8.2139e-02 (segment_embed.weight)
|
| 147 |
+
[step 7000] step= 7000 | total=0.6063 | leaf=0.6063 | ancestor=0.0000 | lr=3.50e-04 | gnorm=1.6916e-01 | t=31.7s
|
| 148 |
+
[step 7100] [GRAD STATS] total_norm=1.9658e-01 max_single=1.2897e-01 (segment_embed.weight)
|
| 149 |
+
[step 7100] step= 7100 | total=0.6055 | leaf=0.6055 | ancestor=0.0000 | lr=3.55e-04 | gnorm=1.9658e-01 | t=31.7s
|
| 150 |
+
[step 7200] [GRAD STATS] total_norm=1.7571e-01 max_single=9.8518e-02 (segment_embed.weight)
|
| 151 |
+
[step 7200] step= 7200 | total=0.5911 | leaf=0.5911 | ancestor=0.0000 | lr=3.60e-04 | gnorm=1.7571e-01 | t=38.6s
|
| 152 |
+
[step 7300] [GRAD STATS] total_norm=1.6891e-01 max_single=9.7972e-02 (segment_embed.weight)
|
| 153 |
+
[step 7300] step= 7300 | total=0.6255 | leaf=0.6255 | ancestor=0.0000 | lr=3.65e-04 | gnorm=1.6891e-01 | t=31.7s
|
| 154 |
+
[step 7400] [GRAD STATS] total_norm=1.4308e-01 max_single=7.0317e-02 (vocab_embed.embedding)
|
| 155 |
+
[step 7400] step= 7400 | total=0.5810 | leaf=0.5810 | ancestor=0.0000 | lr=3.70e-04 | gnorm=1.4308e-01 | t=31.7s
|
| 156 |
+
[step 7500] [GRAD STATS] total_norm=2.2660e-01 max_single=1.5884e-01 (segment_embed.weight)
|
| 157 |
+
[step 7500] step= 7500 | total=0.6055 | leaf=0.6055 | ancestor=0.0000 | lr=3.75e-04 | gnorm=2.2660e-01 | t=31.7s
|
| 158 |
+
[step 7600] [GRAD STATS] total_norm=1.6671e-01 max_single=8.9658e-02 (segment_embed.weight)
|
| 159 |
+
[step 7600] step= 7600 | total=0.6632 | leaf=0.6632 | ancestor=0.0000 | lr=3.80e-04 | gnorm=1.6671e-01 | t=31.7s
|
| 160 |
+
[step 7700] [GRAD STATS] total_norm=2.8842e-01 max_single=2.2237e-01 (segment_embed.weight)
|
| 161 |
+
[step 7700] step= 7700 | total=0.6442 | leaf=0.6442 | ancestor=0.0000 | lr=3.85e-04 | gnorm=2.8842e-01 | t=31.7s
|
| 162 |
+
[step 7800] [GRAD STATS] total_norm=1.4522e-01 max_single=6.6484e-02 (vocab_embed.embedding)
|
| 163 |
+
[step 7800] step= 7800 | total=0.5893 | leaf=0.5893 | ancestor=0.0000 | lr=3.90e-04 | gnorm=1.4522e-01 | t=31.7s
|
| 164 |
+
[step 7900] [GRAD STATS] total_norm=1.5193e-01 max_single=7.7448e-02 (segment_embed.weight)
|
| 165 |
+
[step 7900] step= 7900 | total=0.6192 | leaf=0.6192 | ancestor=0.0000 | lr=3.95e-04 | gnorm=1.5193e-01 | t=31.7s
|
| 166 |
+
[step 8000] [GRAD STATS] total_norm=1.8643e-01 max_single=1.2290e-01 (segment_embed.weight)
|
| 167 |
+
[step 8000] step= 8000 | total=0.5806 | leaf=0.5806 | ancestor=0.0000 | lr=4.00e-04 | gnorm=1.8643e-01 | t=31.7s
|
| 168 |
+
[step 8100] [GRAD STATS] total_norm=1.7063e-01 max_single=9.7380e-02 (segment_embed.weight)
|
| 169 |
+
[step 8100] step= 8100 | total=0.6631 | leaf=0.6631 | ancestor=0.0000 | lr=4.05e-04 | gnorm=1.7063e-01 | t=31.7s
|
| 170 |
+
[step 8200] [GRAD STATS] total_norm=1.3556e-01 max_single=6.3052e-02 (vocab_embed.embedding)
|
| 171 |
+
[step 8200] step= 8200 | total=0.6352 | leaf=0.6352 | ancestor=0.0000 | lr=4.10e-04 | gnorm=1.3556e-01 | t=31.7s
|
| 172 |
+
[step 8300] [GRAD STATS] total_norm=1.4400e-01 max_single=6.5865e-02 (vocab_embed.embedding)
|
| 173 |
+
[step 8300] step= 8300 | total=0.6678 | leaf=0.6678 | ancestor=0.0000 | lr=4.15e-04 | gnorm=1.4400e-01 | t=31.7s
|
| 174 |
+
[step 8400] [GRAD STATS] total_norm=1.3551e-01 max_single=6.2218e-02 (vocab_embed.embedding)
|
| 175 |
+
[step 8400] step= 8400 | total=0.6580 | leaf=0.6580 | ancestor=0.0000 | lr=4.20e-04 | gnorm=1.3551e-01 | t=31.7s
|
| 176 |
+
[step 8500] [GRAD STATS] total_norm=1.5300e-01 max_single=7.5763e-02 (segment_embed.weight)
|
| 177 |
+
[step 8500] step= 8500 | total=0.6423 | leaf=0.6423 | ancestor=0.0000 | lr=4.25e-04 | gnorm=1.5300e-01 | t=31.7s
|
| 178 |
+
File "/home/sunhc/diffusion/sad/scripts/train.py", line 707, in <module>
|
| 179 |
+
if pbar is not None:
|
| 180 |
+
^^^^^^
|
| 181 |
+
File "/home/sunhc/diffusion/sad/scripts/train.py", line 610, in main
|
| 182 |
+
if pbar is not None:
|
| 183 |
+
^^^^^^^^^^^^^^^
|
| 184 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/_tensor.py", line 626, in backward
|
| 185 |
+
torch.autograd.backward(
|
| 186 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 187 |
+
_engine_run_backward(
|
| 188 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/graph.py", line 823, in _engine_run_backward
|
| 189 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 190 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 191 |
+
KeyboardInterrupt
|
| 192 |
+
[rank0]: Traceback (most recent call last):
|
| 193 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train.py", line 707, in <module>
|
| 194 |
+
[rank0]: if pbar is not None:
|
| 195 |
+
[rank0]: ^^^^^^
|
| 196 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train.py", line 610, in main
|
| 197 |
+
[rank0]: if pbar is not None:
|
| 198 |
+
[rank0]: ^^^^^^^^^^^^^^^
|
| 199 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/_tensor.py", line 626, in backward
|
| 200 |
+
[rank0]: torch.autograd.backward(
|
| 201 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 202 |
+
[rank0]: _engine_run_backward(
|
| 203 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/graph.py", line 823, in _engine_run_backward
|
| 204 |
+
[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 205 |
+
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 206 |
+
[rank0]: KeyboardInterrupt
|
wandb/run-20260423_051457-2j4bd54k/logs/debug-core.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T05:14:57.5063942Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmplweblm79/port-952189.txt","pid":952189,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T05:14:57.507046386Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":952189}
|
| 3 |
+
{"time":"2026-04-23T05:14:57.506983479Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-952189-958224-915749100/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T05:14:57.686694506Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T05:14:57.696914669Z","level":"INFO","msg":"handleInformInit: received","streamId":"2j4bd54k","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T05:14:58.500839329Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2j4bd54k","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T05:15:04.49922953Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6y442mpn28t3"}
|
| 8 |
+
{"time":"2026-04-23T06:01:01.610709473Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-23T06:01:01.610823017Z","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2026-04-23T06:01:01.61081217Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-23T06:01:01.610932188Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-23T06:01:01.610960913Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-952189-958224-915749100/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-23T06:01:05.445065818Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
wandb/run-20260423_051457-2j4bd54k/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,378 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T05:14:57.697007568Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-23T05:14:57.699667348Z","level":"INFO","msg":"stream: starting","core version":"0.25.1"}
|
| 3 |
+
{"time":"2026-04-23T05:14:58.500431116Z","level":"INFO","msg":"stream: created new stream","id":"2j4bd54k"}
|
| 4 |
+
{"time":"2026-04-23T05:14:58.500559711Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-23T05:14:58.500717726Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-23T05:14:58.500765845Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-04-23T05:14:58.500760853Z","level":"INFO","msg":"writer: started","stream_id":"2j4bd54k"}
|
| 8 |
+
{"time":"2026-04-23T05:14:59.494740727Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-04-23T05:15:00.028332386Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-23T05:15:14.494628674Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":1}
|
| 11 |
+
{"time":"2026-04-23T05:15:14.89043625Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-23T05:15:29.495318751Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":1,"console_lines":3}
|
| 13 |
+
{"time":"2026-04-23T05:15:29.940555933Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-04-23T05:15:44.494621115Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 15 |
+
{"time":"2026-04-23T05:15:44.890073298Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-04-23T05:15:59.495068064Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":6,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 17 |
+
{"time":"2026-04-23T05:15:59.982894704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-04-23T05:16:14.494875948Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 19 |
+
{"time":"2026-04-23T05:16:14.904085432Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 20 |
+
{"time":"2026-04-23T05:16:29.495080973Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 21 |
+
{"time":"2026-04-23T05:16:30.515257284Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 22 |
+
{"time":"2026-04-23T05:16:44.49543911Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":12,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 23 |
+
{"time":"2026-04-23T05:16:44.916635829Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 24 |
+
{"time":"2026-04-23T05:16:59.495510488Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":14,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 25 |
+
{"time":"2026-04-23T05:16:59.960471182Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 26 |
+
{"time":"2026-04-23T05:17:14.49472264Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":16,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 27 |
+
{"time":"2026-04-23T05:17:14.954138321Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 28 |
+
{"time":"2026-04-23T05:17:29.494858561Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":18,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 29 |
+
{"time":"2026-04-23T05:17:29.950444979Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 30 |
+
{"time":"2026-04-23T05:17:44.495679758Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 31 |
+
{"time":"2026-04-23T05:17:44.857309362Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 32 |
+
{"time":"2026-04-23T05:17:59.49526802Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 33 |
+
{"time":"2026-04-23T05:17:59.884031157Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 34 |
+
{"time":"2026-04-23T05:18:14.495153729Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":24,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 35 |
+
{"time":"2026-04-23T05:18:14.949762735Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 36 |
+
{"time":"2026-04-23T05:18:29.495585698Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":26,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 37 |
+
{"time":"2026-04-23T05:18:29.957092662Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 38 |
+
{"time":"2026-04-23T05:18:44.495918464Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":28,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 39 |
+
{"time":"2026-04-23T05:18:44.893696294Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 40 |
+
{"time":"2026-04-23T05:18:59.49470474Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":30,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 41 |
+
{"time":"2026-04-23T05:18:59.94444976Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 42 |
+
{"time":"2026-04-23T05:19:14.495465075Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":32,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 43 |
+
{"time":"2026-04-23T05:19:14.927047514Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 44 |
+
{"time":"2026-04-23T05:19:29.495000912Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 45 |
+
{"time":"2026-04-23T05:19:29.917452377Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 46 |
+
{"time":"2026-04-23T05:19:44.495607822Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":36,"events_lines":2,"console_offset":2,"console_lines":1,"uploaded_len":2}
|
| 47 |
+
{"time":"2026-04-23T05:19:45.046810126Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 48 |
+
{"time":"2026-04-23T05:19:59.495515008Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":38,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 49 |
+
{"time":"2026-04-23T05:19:59.88091251Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 50 |
+
{"time":"2026-04-23T05:20:14.494989241Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":40,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 51 |
+
{"time":"2026-04-23T05:20:14.931811083Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 52 |
+
{"time":"2026-04-23T05:20:29.495428626Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":42,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 53 |
+
{"time":"2026-04-23T05:20:29.972508263Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 54 |
+
{"time":"2026-04-23T05:20:44.494845987Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":44,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 55 |
+
{"time":"2026-04-23T05:20:44.964637134Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 56 |
+
{"time":"2026-04-23T05:20:59.495657574Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":46,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 57 |
+
{"time":"2026-04-23T05:20:59.910394549Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 58 |
+
{"time":"2026-04-23T05:21:14.495015875Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":48,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 59 |
+
{"time":"2026-04-23T05:21:14.969992562Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 60 |
+
{"time":"2026-04-23T05:21:29.49523007Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":50,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 61 |
+
{"time":"2026-04-23T05:21:29.937433227Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 62 |
+
{"time":"2026-04-23T05:21:44.495808682Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 63 |
+
{"time":"2026-04-23T05:21:44.957306323Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 64 |
+
{"time":"2026-04-23T05:21:59.495297249Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":54,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 65 |
+
{"time":"2026-04-23T05:21:59.859400437Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 66 |
+
{"time":"2026-04-23T05:22:14.494967037Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 67 |
+
{"time":"2026-04-23T05:22:14.917667047Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 68 |
+
{"time":"2026-04-23T05:22:29.494787424Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":58,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 69 |
+
{"time":"2026-04-23T05:22:29.956747915Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 70 |
+
{"time":"2026-04-23T05:22:44.495231571Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":60,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 71 |
+
{"time":"2026-04-23T05:22:44.857975659Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 72 |
+
{"time":"2026-04-23T05:22:59.494820772Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":62,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 73 |
+
{"time":"2026-04-23T05:23:00.001887552Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 74 |
+
{"time":"2026-04-23T05:23:14.49534075Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":64,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 75 |
+
{"time":"2026-04-23T05:23:14.971379492Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 76 |
+
{"time":"2026-04-23T05:23:29.495433308Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 77 |
+
{"time":"2026-04-23T05:23:30.006481637Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 78 |
+
{"time":"2026-04-23T05:23:44.495763939Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":68,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 79 |
+
{"time":"2026-04-23T05:23:44.92573312Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 80 |
+
{"time":"2026-04-23T05:23:59.495655808Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":1,"events_offset":70,"events_lines":2,"console_offset":2,"console_lines":1}
|
| 81 |
+
{"time":"2026-04-23T05:23:59.940561936Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 82 |
+
{"time":"2026-04-23T05:24:14.494679809Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":72,"events_lines":2,"console_offset":4,"console_lines":32}
|
| 83 |
+
{"time":"2026-04-23T05:24:14.912975477Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 84 |
+
{"time":"2026-04-23T05:24:29.495102586Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":17,"history_lines":1,"events_offset":74,"events_lines":2,"console_offset":36,"console_lines":2}
|
| 85 |
+
{"time":"2026-04-23T05:24:29.882192754Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 86 |
+
{"time":"2026-04-23T05:24:44.495006066Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
|
| 87 |
+
{"time":"2026-04-23T05:24:44.937192443Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 88 |
+
{"time":"2026-04-23T05:24:59.495249526Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":78,"events_lines":2,"console_offset":38,"console_lines":2}
|
| 89 |
+
{"time":"2026-04-23T05:24:59.897041464Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 90 |
+
{"time":"2026-04-23T05:25:14.495203615Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":80,"events_lines":2}
|
| 91 |
+
{"time":"2026-04-23T05:25:15.012341578Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 92 |
+
{"time":"2026-04-23T05:25:29.495660691Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":82,"events_lines":2,"console_offset":40,"console_lines":2}
|
| 93 |
+
{"time":"2026-04-23T05:25:29.939627463Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 94 |
+
{"time":"2026-04-23T05:25:44.49530254Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":84,"events_lines":2}
|
| 95 |
+
{"time":"2026-04-23T05:25:44.995588169Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 96 |
+
{"time":"2026-04-23T05:25:59.495892949Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":86,"events_lines":2,"console_offset":42,"console_lines":2}
|
| 97 |
+
{"time":"2026-04-23T05:25:59.963906864Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 98 |
+
{"time":"2026-04-23T05:26:14.495574843Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":88,"events_lines":2}
|
| 99 |
+
{"time":"2026-04-23T05:26:14.877691055Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 100 |
+
{"time":"2026-04-23T05:26:29.495649909Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":1,"events_offset":90,"events_lines":2,"console_offset":44,"console_lines":2}
|
| 101 |
+
{"time":"2026-04-23T05:26:30.024384941Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 102 |
+
{"time":"2026-04-23T05:26:44.495739046Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":92,"events_lines":2}
|
| 103 |
+
{"time":"2026-04-23T05:26:44.96668635Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 104 |
+
{"time":"2026-04-23T05:26:59.49495833Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":22,"history_lines":1,"events_offset":94,"events_lines":2,"console_offset":46,"console_lines":2}
|
| 105 |
+
{"time":"2026-04-23T05:26:59.941380333Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 106 |
+
{"time":"2026-04-23T05:27:14.494949232Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":96,"events_lines":2}
|
| 107 |
+
{"time":"2026-04-23T05:27:14.941556921Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 108 |
+
{"time":"2026-04-23T05:27:29.495620256Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":98,"events_lines":2}
|
| 109 |
+
{"time":"2026-04-23T05:27:30.189828384Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 110 |
+
{"time":"2026-04-23T05:27:44.495786011Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":1,"events_offset":100,"events_lines":2,"console_offset":48,"console_lines":2}
|
| 111 |
+
{"time":"2026-04-23T05:27:44.986693485Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 112 |
+
{"time":"2026-04-23T05:27:59.495653261Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":102,"events_lines":2}
|
| 113 |
+
{"time":"2026-04-23T05:28:00.032011063Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 114 |
+
{"time":"2026-04-23T05:28:14.494877747Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":24,"history_lines":1,"events_offset":104,"events_lines":2,"console_offset":50,"console_lines":2}
|
| 115 |
+
{"time":"2026-04-23T05:28:14.961833672Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 116 |
+
{"time":"2026-04-23T05:28:29.494642236Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":106,"events_lines":2}
|
| 117 |
+
{"time":"2026-04-23T05:28:29.937405937Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 118 |
+
{"time":"2026-04-23T05:28:44.494784035Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":1,"events_offset":108,"events_lines":2,"console_offset":52,"console_lines":2}
|
| 119 |
+
{"time":"2026-04-23T05:28:44.976245032Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 120 |
+
{"time":"2026-04-23T05:28:59.494629757Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":110,"events_lines":2}
|
| 121 |
+
{"time":"2026-04-23T05:28:59.974777916Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 122 |
+
{"time":"2026-04-23T05:29:14.495117206Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":26,"history_lines":1,"events_offset":112,"events_lines":2,"console_offset":54,"console_lines":2}
|
| 123 |
+
{"time":"2026-04-23T05:29:14.943209717Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 124 |
+
{"time":"2026-04-23T05:29:29.494894171Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":114,"events_lines":2}
|
| 125 |
+
{"time":"2026-04-23T05:29:29.959002844Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 126 |
+
{"time":"2026-04-23T05:29:44.494983237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":116,"events_lines":2,"console_offset":56,"console_lines":2}
|
| 127 |
+
{"time":"2026-04-23T05:29:45.012643715Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 128 |
+
{"time":"2026-04-23T05:29:59.494739855Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":118,"events_lines":2}
|
| 129 |
+
{"time":"2026-04-23T05:29:59.895521378Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 130 |
+
{"time":"2026-04-23T05:30:14.495302779Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":120,"events_lines":2,"console_offset":58,"console_lines":2}
|
| 131 |
+
{"time":"2026-04-23T05:30:15.037285444Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 132 |
+
{"time":"2026-04-23T05:30:29.495640784Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":122,"events_lines":2}
|
| 133 |
+
{"time":"2026-04-23T05:30:29.953708051Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 134 |
+
{"time":"2026-04-23T05:30:44.495662898Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":124,"events_lines":2,"console_offset":60,"console_lines":2}
|
| 135 |
+
{"time":"2026-04-23T05:30:45.018999233Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 136 |
+
{"time":"2026-04-23T05:30:59.495023278Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":126,"events_lines":2}
|
| 137 |
+
{"time":"2026-04-23T05:30:59.98080866Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 138 |
+
{"time":"2026-04-23T05:31:14.495655024Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":1,"events_offset":128,"events_lines":2,"console_offset":62,"console_lines":2}
|
| 139 |
+
{"time":"2026-04-23T05:31:14.888577204Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 140 |
+
{"time":"2026-04-23T05:31:29.495140143Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":130,"events_lines":2}
|
| 141 |
+
{"time":"2026-04-23T05:31:29.948181939Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 142 |
+
{"time":"2026-04-23T05:31:44.495472514Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":132,"events_lines":2}
|
| 143 |
+
{"time":"2026-04-23T05:31:44.918554698Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 144 |
+
{"time":"2026-04-23T05:31:59.495537335Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":31,"history_lines":1,"events_offset":134,"events_lines":2,"console_offset":64,"console_lines":2}
|
| 145 |
+
{"time":"2026-04-23T05:31:59.91213015Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 146 |
+
{"time":"2026-04-23T05:32:14.494985324Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":136,"events_lines":2}
|
| 147 |
+
{"time":"2026-04-23T05:32:14.942118901Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 148 |
+
{"time":"2026-04-23T05:32:29.495728933Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":1,"events_offset":138,"events_lines":2,"console_offset":66,"console_lines":2}
|
| 149 |
+
{"time":"2026-04-23T05:32:29.98383634Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 150 |
+
{"time":"2026-04-23T05:32:44.495204847Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":140,"events_lines":2}
|
| 151 |
+
{"time":"2026-04-23T05:32:45.005565347Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 152 |
+
{"time":"2026-04-23T05:32:59.496004497Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":33,"history_lines":1,"events_offset":142,"events_lines":2,"console_offset":68,"console_lines":2}
|
| 153 |
+
{"time":"2026-04-23T05:32:59.950895644Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 154 |
+
{"time":"2026-04-23T05:33:14.495196531Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":144,"events_lines":2}
|
| 155 |
+
{"time":"2026-04-23T05:33:14.931043084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 156 |
+
{"time":"2026-04-23T05:33:29.495216484Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":34,"history_lines":1,"events_offset":146,"events_lines":2,"console_offset":70,"console_lines":2}
|
| 157 |
+
{"time":"2026-04-23T05:33:29.964545689Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 158 |
+
{"time":"2026-04-23T05:33:44.494826825Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":148,"events_lines":2}
|
| 159 |
+
{"time":"2026-04-23T05:33:44.945517343Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 160 |
+
{"time":"2026-04-23T05:33:59.495143704Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":35,"history_lines":1,"events_offset":150,"events_lines":2,"console_offset":72,"console_lines":2}
|
| 161 |
+
{"time":"2026-04-23T05:33:59.930579019Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 162 |
+
{"time":"2026-04-23T05:34:14.495348378Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":152,"events_lines":2}
|
| 163 |
+
{"time":"2026-04-23T05:34:14.980751215Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 164 |
+
{"time":"2026-04-23T05:34:29.495289409Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":36,"history_lines":1,"events_offset":154,"events_lines":2,"console_offset":74,"console_lines":2}
|
| 165 |
+
{"time":"2026-04-23T05:34:29.93326643Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 166 |
+
{"time":"2026-04-23T05:34:44.495583994Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":156,"events_lines":2}
|
| 167 |
+
{"time":"2026-04-23T05:34:44.984507486Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 168 |
+
{"time":"2026-04-23T05:34:59.495298633Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":37,"history_lines":1,"events_offset":158,"events_lines":2,"console_offset":76,"console_lines":2}
|
| 169 |
+
{"time":"2026-04-23T05:34:59.927566939Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 170 |
+
{"time":"2026-04-23T05:35:14.495263658Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":160,"events_lines":2}
|
| 171 |
+
{"time":"2026-04-23T05:35:15.004562056Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 172 |
+
{"time":"2026-04-23T05:35:29.495525548Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":38,"history_lines":1,"events_offset":162,"events_lines":2,"console_offset":78,"console_lines":2}
|
| 173 |
+
{"time":"2026-04-23T05:35:30.070165971Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 174 |
+
{"time":"2026-04-23T05:35:44.495036247Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":164,"events_lines":2}
|
| 175 |
+
{"time":"2026-04-23T05:35:44.966947475Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 176 |
+
{"time":"2026-04-23T05:35:59.495768611Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":39,"history_lines":1,"events_offset":166,"events_lines":2,"console_offset":80,"console_lines":2}
|
| 177 |
+
{"time":"2026-04-23T05:35:59.970710466Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 178 |
+
{"time":"2026-04-23T05:36:14.495586126Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":168,"events_lines":2}
|
| 179 |
+
{"time":"2026-04-23T05:36:15.131009803Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 180 |
+
{"time":"2026-04-23T05:36:29.495154516Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":170,"events_lines":2}
|
| 181 |
+
{"time":"2026-04-23T05:36:29.976790746Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 182 |
+
{"time":"2026-04-23T05:36:44.495105972Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":40,"history_lines":1,"events_offset":172,"events_lines":2,"console_offset":82,"console_lines":2}
|
| 183 |
+
{"time":"2026-04-23T05:36:44.970875414Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 184 |
+
{"time":"2026-04-23T05:36:59.494749529Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":174,"events_lines":2}
|
| 185 |
+
{"time":"2026-04-23T05:36:59.994288934Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 186 |
+
{"time":"2026-04-23T05:37:14.494636917Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":1,"events_offset":176,"events_lines":2,"console_offset":84,"console_lines":2}
|
| 187 |
+
{"time":"2026-04-23T05:37:14.974946998Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 188 |
+
{"time":"2026-04-23T05:37:29.494897634Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":178,"events_lines":2}
|
| 189 |
+
{"time":"2026-04-23T05:37:29.974680856Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 190 |
+
{"time":"2026-04-23T05:37:44.495704935Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":42,"history_lines":1,"events_offset":180,"events_lines":2,"console_offset":86,"console_lines":2}
|
| 191 |
+
{"time":"2026-04-23T05:37:44.949805357Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 192 |
+
{"time":"2026-04-23T05:37:59.495232089Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":182,"events_lines":2}
|
| 193 |
+
{"time":"2026-04-23T05:37:59.912307669Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 194 |
+
{"time":"2026-04-23T05:38:14.495737239Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":43,"history_lines":1,"events_offset":184,"events_lines":2,"console_offset":88,"console_lines":2}
|
| 195 |
+
{"time":"2026-04-23T05:38:14.948230952Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 196 |
+
{"time":"2026-04-23T05:38:29.494767395Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":186,"events_lines":2}
|
| 197 |
+
{"time":"2026-04-23T05:38:29.876022206Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 198 |
+
{"time":"2026-04-23T05:38:44.495253968Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":44,"history_lines":1,"events_offset":188,"events_lines":2,"console_offset":90,"console_lines":2}
|
| 199 |
+
{"time":"2026-04-23T05:38:45.06384293Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 200 |
+
{"time":"2026-04-23T05:38:59.495314264Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":190,"events_lines":2}
|
| 201 |
+
{"time":"2026-04-23T05:38:59.907288685Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 202 |
+
{"time":"2026-04-23T05:39:14.495197968Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":45,"history_lines":1,"events_offset":192,"events_lines":2,"console_offset":92,"console_lines":2}
|
| 203 |
+
{"time":"2026-04-23T05:39:14.955547519Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 204 |
+
{"time":"2026-04-23T05:39:29.495146274Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":194,"events_lines":2}
|
| 205 |
+
{"time":"2026-04-23T05:39:30.025509562Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 206 |
+
{"time":"2026-04-23T05:39:44.495707973Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":46,"history_lines":1,"events_offset":196,"events_lines":2,"console_offset":94,"console_lines":2}
|
| 207 |
+
{"time":"2026-04-23T05:39:44.914575387Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 208 |
+
{"time":"2026-04-23T05:39:59.495532917Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":198,"events_lines":2}
|
| 209 |
+
{"time":"2026-04-23T05:40:00.031369431Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 210 |
+
{"time":"2026-04-23T05:40:14.495282559Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":200,"events_lines":2}
|
| 211 |
+
{"time":"2026-04-23T05:40:14.894805738Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 212 |
+
{"time":"2026-04-23T05:40:29.495651245Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":1,"events_offset":202,"events_lines":2,"console_offset":96,"console_lines":2}
|
| 213 |
+
{"time":"2026-04-23T05:40:29.972249436Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 214 |
+
{"time":"2026-04-23T05:40:44.495172678Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":204,"events_lines":2}
|
| 215 |
+
{"time":"2026-04-23T05:40:45.00970851Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 216 |
+
{"time":"2026-04-23T05:40:59.495861214Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":1,"events_offset":206,"events_lines":2,"console_offset":98,"console_lines":2}
|
| 217 |
+
{"time":"2026-04-23T05:41:00.13033112Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 218 |
+
{"time":"2026-04-23T05:41:14.495464626Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":208,"events_lines":2}
|
| 219 |
+
{"time":"2026-04-23T05:41:15.013265694Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 220 |
+
{"time":"2026-04-23T05:41:29.494875398Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":49,"history_lines":1,"events_offset":210,"events_lines":2,"console_offset":100,"console_lines":2}
|
| 221 |
+
{"time":"2026-04-23T05:41:29.966888503Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 222 |
+
{"time":"2026-04-23T05:41:44.495553096Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":212,"events_lines":2}
|
| 223 |
+
{"time":"2026-04-23T05:41:44.964460314Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 224 |
+
{"time":"2026-04-23T05:41:59.495577443Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":50,"history_lines":1,"events_offset":214,"events_lines":2,"console_offset":102,"console_lines":3}
|
| 225 |
+
{"time":"2026-04-23T05:41:59.928156949Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 226 |
+
{"time":"2026-04-23T05:42:14.495470558Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":51,"history_lines":1,"events_offset":216,"events_lines":2,"console_offset":105,"console_lines":2}
|
| 227 |
+
{"time":"2026-04-23T05:42:14.934534849Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 228 |
+
{"time":"2026-04-23T05:42:29.495542028Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":218,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 229 |
+
{"time":"2026-04-23T05:42:29.913905609Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 230 |
+
{"time":"2026-04-23T05:42:44.494970783Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":52,"history_lines":1,"events_offset":220,"events_lines":2,"console_offset":106,"console_lines":3}
|
| 231 |
+
{"time":"2026-04-23T05:42:44.94711417Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 232 |
+
{"time":"2026-04-23T05:42:59.494814902Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":222,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 233 |
+
{"time":"2026-04-23T05:42:59.938165987Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 234 |
+
{"time":"2026-04-23T05:43:14.494964035Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":53,"history_lines":1,"events_offset":224,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 235 |
+
{"time":"2026-04-23T05:43:15.030774747Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 236 |
+
{"time":"2026-04-23T05:43:29.495725201Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":226,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 237 |
+
{"time":"2026-04-23T05:43:29.993247356Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 238 |
+
{"time":"2026-04-23T05:43:44.494950771Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":54,"history_lines":1,"events_offset":228,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 239 |
+
{"time":"2026-04-23T05:43:44.954669672Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 240 |
+
{"time":"2026-04-23T05:43:59.495212175Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":230,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 241 |
+
{"time":"2026-04-23T05:43:59.907393523Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 242 |
+
{"time":"2026-04-23T05:44:14.494770668Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":232,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 243 |
+
{"time":"2026-04-23T05:44:14.927082487Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 244 |
+
{"time":"2026-04-23T05:44:29.495583237Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":55,"history_lines":1,"events_offset":234,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 245 |
+
{"time":"2026-04-23T05:44:29.954627362Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 246 |
+
{"time":"2026-04-23T05:44:44.495504005Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":236,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 247 |
+
{"time":"2026-04-23T05:44:44.925723495Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 248 |
+
{"time":"2026-04-23T05:44:59.495114665Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":56,"history_lines":1,"events_offset":238,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 249 |
+
{"time":"2026-04-23T05:45:00.144595981Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 250 |
+
{"time":"2026-04-23T05:45:14.495262436Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":240,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 251 |
+
{"time":"2026-04-23T05:45:15.053936578Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 252 |
+
{"time":"2026-04-23T05:45:29.495410509Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":57,"history_lines":1,"events_offset":242,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 253 |
+
{"time":"2026-04-23T05:45:29.946053378Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 254 |
+
{"time":"2026-04-23T05:45:44.495394072Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":244,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 255 |
+
{"time":"2026-04-23T05:45:44.984595037Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 256 |
+
{"time":"2026-04-23T05:45:59.494911124Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":58,"history_lines":1,"events_offset":246,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 257 |
+
{"time":"2026-04-23T05:45:59.904979162Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 258 |
+
{"time":"2026-04-23T05:46:14.494990767Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":248,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 259 |
+
{"time":"2026-04-23T05:46:14.906490629Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 260 |
+
{"time":"2026-04-23T05:46:29.495739938Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":59,"history_lines":1,"events_offset":250,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 261 |
+
{"time":"2026-04-23T05:46:29.958509052Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 262 |
+
{"time":"2026-04-23T05:46:44.495631888Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":252,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 263 |
+
{"time":"2026-04-23T05:46:45.17056758Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 264 |
+
{"time":"2026-04-23T05:46:59.495449441Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":60,"history_lines":1,"events_offset":254,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 265 |
+
{"time":"2026-04-23T05:47:00.064495876Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 266 |
+
{"time":"2026-04-23T05:47:14.495760072Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":256,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 267 |
+
{"time":"2026-04-23T05:47:14.960191977Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 268 |
+
{"time":"2026-04-23T05:47:29.49550688Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":61,"history_lines":1,"events_offset":258,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 269 |
+
{"time":"2026-04-23T05:47:29.917775898Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 270 |
+
{"time":"2026-04-23T05:47:44.495594273Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":260,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 271 |
+
{"time":"2026-04-23T05:47:44.962702518Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 272 |
+
{"time":"2026-04-23T05:47:59.495495306Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":262,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 273 |
+
{"time":"2026-04-23T05:48:00.075112067Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 274 |
+
{"time":"2026-04-23T05:48:14.495043082Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":62,"history_lines":1,"events_offset":264,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 275 |
+
{"time":"2026-04-23T05:48:15.120396687Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 276 |
+
{"time":"2026-04-23T05:48:29.49574005Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":266,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 277 |
+
{"time":"2026-04-23T05:48:29.981933184Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 278 |
+
{"time":"2026-04-23T05:48:44.495770864Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":63,"history_lines":1,"events_offset":268,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 279 |
+
{"time":"2026-04-23T05:48:45.021602797Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 280 |
+
{"time":"2026-04-23T05:48:59.495107947Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":270,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 281 |
+
{"time":"2026-04-23T05:48:59.97755596Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 282 |
+
{"time":"2026-04-23T05:49:14.495487491Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":64,"history_lines":1,"events_offset":272,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 283 |
+
{"time":"2026-04-23T05:49:14.948366576Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 284 |
+
{"time":"2026-04-23T05:49:29.495517387Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":274,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 285 |
+
{"time":"2026-04-23T05:49:29.989269813Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 286 |
+
{"time":"2026-04-23T05:49:44.495704422Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":65,"history_lines":1,"events_offset":276,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 287 |
+
{"time":"2026-04-23T05:49:44.934480326Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 288 |
+
{"time":"2026-04-23T05:49:59.494936924Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":278,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 289 |
+
{"time":"2026-04-23T05:50:00.039125994Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 290 |
+
{"time":"2026-04-23T05:50:14.495787156Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":66,"history_lines":1,"events_offset":280,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 291 |
+
{"time":"2026-04-23T05:50:15.125233508Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 292 |
+
{"time":"2026-04-23T05:50:29.494751603Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":282,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 293 |
+
{"time":"2026-04-23T05:50:29.977794873Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 294 |
+
{"time":"2026-04-23T05:50:44.49538402Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":284,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 295 |
+
{"time":"2026-04-23T05:50:45.325293848Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 296 |
+
{"time":"2026-04-23T05:50:59.495574683Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":67,"history_lines":1,"events_offset":286,"events_lines":2,"console_offset":106,"console_lines":1}
|
| 297 |
+
{"time":"2026-04-23T05:51:00.027735963Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 298 |
+
{"time":"2026-04-23T05:51:14.495680404Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":288,"events_lines":2,"console_offset":109,"console_lines":30}
|
| 299 |
+
{"time":"2026-04-23T05:51:14.990802711Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 300 |
+
{"time":"2026-04-23T05:51:29.495904787Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":68,"history_lines":1,"events_offset":290,"events_lines":2,"console_offset":139,"console_lines":2}
|
| 301 |
+
{"time":"2026-04-23T05:51:29.976252664Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 302 |
+
{"time":"2026-04-23T05:51:44.495326541Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":292,"events_lines":2}
|
| 303 |
+
{"time":"2026-04-23T05:51:45.071595898Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 304 |
+
{"time":"2026-04-23T05:51:59.495630384Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":69,"history_lines":1,"events_offset":294,"events_lines":2,"console_offset":141,"console_lines":2}
|
| 305 |
+
{"time":"2026-04-23T05:51:59.952830923Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 306 |
+
{"time":"2026-04-23T05:52:14.495589184Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":296,"events_lines":2}
|
| 307 |
+
{"time":"2026-04-23T05:52:15.093153798Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 308 |
+
{"time":"2026-04-23T05:52:29.495920004Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":70,"history_lines":1,"events_offset":298,"events_lines":2,"console_offset":143,"console_lines":2}
|
| 309 |
+
{"time":"2026-04-23T05:52:29.890438308Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 310 |
+
{"time":"2026-04-23T05:52:44.495501485Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":300,"events_lines":2}
|
| 311 |
+
{"time":"2026-04-23T05:52:45.217043628Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 312 |
+
{"time":"2026-04-23T05:52:59.495391423Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":71,"history_lines":1,"events_offset":302,"events_lines":2,"console_offset":145,"console_lines":2}
|
| 313 |
+
{"time":"2026-04-23T05:52:59.995011663Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 314 |
+
{"time":"2026-04-23T05:53:14.495486227Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":304,"events_lines":2}
|
| 315 |
+
{"time":"2026-04-23T05:53:14.970267864Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 316 |
+
{"time":"2026-04-23T05:53:29.495567709Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":72,"history_lines":1,"events_offset":306,"events_lines":2,"console_offset":147,"console_lines":2}
|
| 317 |
+
{"time":"2026-04-23T05:53:29.95195553Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 318 |
+
{"time":"2026-04-23T05:53:44.495269745Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":308,"events_lines":2}
|
| 319 |
+
{"time":"2026-04-23T05:53:44.889125793Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 320 |
+
{"time":"2026-04-23T05:53:59.495014016Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":310,"events_lines":2}
|
| 321 |
+
{"time":"2026-04-23T05:54:00.082902712Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 322 |
+
{"time":"2026-04-23T05:54:14.49485323Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":73,"history_lines":1,"events_offset":312,"events_lines":2,"console_offset":149,"console_lines":2}
|
| 323 |
+
{"time":"2026-04-23T05:54:14.987817437Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 324 |
+
{"time":"2026-04-23T05:54:29.4954327Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":314,"events_lines":2}
|
| 325 |
+
{"time":"2026-04-23T05:54:29.989867726Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 326 |
+
{"time":"2026-04-23T05:54:44.495258005Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":74,"history_lines":1,"events_offset":316,"events_lines":2,"console_offset":151,"console_lines":2}
|
| 327 |
+
{"time":"2026-04-23T05:54:44.963875133Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 328 |
+
{"time":"2026-04-23T05:54:59.494656981Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":318,"events_lines":2}
|
| 329 |
+
{"time":"2026-04-23T05:54:59.904087704Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 330 |
+
{"time":"2026-04-23T05:55:14.495065782Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":75,"history_lines":1,"events_offset":320,"events_lines":2,"console_offset":153,"console_lines":2}
|
| 331 |
+
{"time":"2026-04-23T05:55:14.939504951Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 332 |
+
{"time":"2026-04-23T05:55:29.495226283Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":322,"events_lines":2}
|
| 333 |
+
{"time":"2026-04-23T05:55:29.986062556Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 334 |
+
{"time":"2026-04-23T05:55:44.495512599Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":76,"history_lines":1,"events_offset":324,"events_lines":2,"console_offset":155,"console_lines":2}
|
| 335 |
+
{"time":"2026-04-23T05:55:44.923683909Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 336 |
+
{"time":"2026-04-23T05:55:59.495800802Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":326,"events_lines":2}
|
| 337 |
+
{"time":"2026-04-23T05:56:00.018591008Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 338 |
+
{"time":"2026-04-23T05:56:14.495662468Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":77,"history_lines":1,"events_offset":328,"events_lines":2,"console_offset":157,"console_lines":2}
|
| 339 |
+
{"time":"2026-04-23T05:56:15.090346856Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 340 |
+
{"time":"2026-04-23T05:56:29.4948333Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":330,"events_lines":2}
|
| 341 |
+
{"time":"2026-04-23T05:56:30.017349397Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 342 |
+
{"time":"2026-04-23T05:56:44.495158538Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":78,"history_lines":1,"events_offset":332,"events_lines":2,"console_offset":159,"console_lines":2}
|
| 343 |
+
{"time":"2026-04-23T05:56:44.887638742Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 344 |
+
{"time":"2026-04-23T05:56:59.495521919Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":334,"events_lines":2}
|
| 345 |
+
{"time":"2026-04-23T05:57:00.033237003Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 346 |
+
{"time":"2026-04-23T05:57:14.495740848Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":336,"events_lines":2}
|
| 347 |
+
{"time":"2026-04-23T05:57:14.90664108Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 348 |
+
{"time":"2026-04-23T05:57:29.495284369Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":79,"history_lines":1,"events_offset":338,"events_lines":2,"console_offset":161,"console_lines":2}
|
| 349 |
+
{"time":"2026-04-23T05:57:29.968045648Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 350 |
+
{"time":"2026-04-23T05:57:44.49524633Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":340,"events_lines":2}
|
| 351 |
+
{"time":"2026-04-23T05:57:44.9109177Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 352 |
+
{"time":"2026-04-23T05:57:59.494986662Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":80,"history_lines":1,"events_offset":342,"events_lines":2,"console_offset":163,"console_lines":2}
|
| 353 |
+
{"time":"2026-04-23T05:57:59.954225192Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 354 |
+
{"time":"2026-04-23T05:58:14.495333525Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":344,"events_lines":2}
|
| 355 |
+
{"time":"2026-04-23T05:58:14.958812729Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 356 |
+
{"time":"2026-04-23T05:58:29.495331245Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":81,"history_lines":1,"events_offset":346,"events_lines":2,"console_offset":165,"console_lines":2}
|
| 357 |
+
{"time":"2026-04-23T05:58:30.045046604Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 358 |
+
{"time":"2026-04-23T05:58:44.495086762Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":348,"events_lines":2}
|
| 359 |
+
{"time":"2026-04-23T05:58:44.99518831Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 360 |
+
{"time":"2026-04-23T05:58:59.49513913Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":82,"history_lines":1,"events_offset":350,"events_lines":2,"console_offset":167,"console_lines":2}
|
| 361 |
+
{"time":"2026-04-23T05:59:00.079628616Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 362 |
+
{"time":"2026-04-23T05:59:14.495231019Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":352,"events_lines":2}
|
| 363 |
+
{"time":"2026-04-23T05:59:14.887510951Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 364 |
+
{"time":"2026-04-23T05:59:29.495577104Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":83,"history_lines":1,"events_offset":354,"events_lines":2,"console_offset":169,"console_lines":2}
|
| 365 |
+
{"time":"2026-04-23T05:59:29.889476156Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 366 |
+
{"time":"2026-04-23T05:59:44.495348357Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":356,"events_lines":2}
|
| 367 |
+
{"time":"2026-04-23T05:59:44.916039327Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 368 |
+
{"time":"2026-04-23T05:59:59.494976894Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":84,"history_lines":1,"events_offset":358,"events_lines":2,"console_offset":171,"console_lines":2}
|
| 369 |
+
{"time":"2026-04-23T05:59:59.960339272Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 370 |
+
{"time":"2026-04-23T06:00:14.495143392Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":360,"events_lines":2}
|
| 371 |
+
{"time":"2026-04-23T06:00:15.072348617Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 372 |
+
{"time":"2026-04-23T06:00:29.495543554Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":85,"history_lines":1,"events_offset":362,"events_lines":2,"console_offset":173,"console_lines":2}
|
| 373 |
+
{"time":"2026-04-23T06:00:30.040838801Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 374 |
+
{"time":"2026-04-23T06:00:44.495433434Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":364,"events_lines":2}
|
| 375 |
+
{"time":"2026-04-23T06:00:45.278552912Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 376 |
+
{"time":"2026-04-23T06:00:59.495085169Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":86,"history_lines":1,"events_offset":366,"events_lines":2,"console_offset":175,"console_lines":2}
|
| 377 |
+
{"time":"2026-04-23T06:01:00.019179631Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 378 |
+
{"time":"2026-04-23T06:01:01.610819102Z","level":"INFO","msg":"stream: closing"}
|
wandb/run-20260423_051457-2j4bd54k/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 05:14:57,469 INFO MainThread:952189 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-23 05:14:57,469 INFO MainThread:952189 [wandb_setup.py:_flush():81] Configure stats pid to 952189
|
| 3 |
+
2026-04-23 05:14:57,469 INFO MainThread:952189 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 05:14:57,470 INFO MainThread:952189 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260423_051457-2j4bd54k/logs/debug.log
|
| 5 |
+
2026-04-23 05:14:57,470 INFO MainThread:952189 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260423_051457-2j4bd54k/logs/debug-internal.log
|
| 6 |
+
2026-04-23 05:14:57,470 INFO MainThread:952189 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-23 05:14:57,470 INFO MainThread:952189 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 8, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top3_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0005, 'lr_min': 5e-05, 'warmup_steps': 10000, 'weight_decay': 0.02, 'grad_clip': 1.0, 'adam_betas': [0.9, 0.99], 'adam_eps': 1e-09, 'dtype': 'bf16', 'compile': 'default', 'curriculum_t_max_init': 0.3, 'curriculum_steps': 20000, 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_new_owt', 'save_dir': 'outputs/sad_new'}, '_wandb': {}}
|
| 9 |
+
2026-04-23 05:14:57,470 INFO MainThread:952189 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-23 05:14:57,686 INFO MainThread:952189 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-23 05:14:57,692 INFO MainThread:952189 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-23 05:14:57,694 INFO MainThread:952189 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-23 05:14:57,698 INFO MainThread:952189 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 05:14:59,367 INFO MainThread:952189 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-23 05:14:59,488 INFO MainThread:952189 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-23 05:14:59,488 INFO MainThread:952189 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-23 05:14:59,488 INFO MainThread:952189 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-23 05:14:59,488 INFO MainThread:952189 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-23 05:14:59,491 INFO MainThread:952189 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-23 06:01:01,610 INFO wandb-AsyncioManager-main:952189 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-23 06:01:01,611 INFO wandb-AsyncioManager-main:952189 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260423_060125-7vxzycy6/files/output.log
ADDED
|
@@ -0,0 +1,505 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
full-denoise training: 0%| | 0/500000 [00:00<?, ?it/s][rank0]:W0423 06:01:27.574000 1026041 site-packages/torch/_logging/_internal.py:1089] [0/0] Profiler function <class 'torch.autograd.profiler.record_function'> will be ignored
|
| 2 |
+
[step 0] [GRAD STATS] total_norm=3.8375e-01 max_single=2.2608e-01 (vocab_embed.embedding)
|
| 3 |
+
full-denoise training: 1%|▎ | 2901/500000 [14:31<41:18:19, 3.34it/s, leaf=0.5207, lr=1.5e-04]
|
| 4 |
+
[step 0] step= 0 | total=1.6813 | leaf=1.6813 | ancestor=0.0000 | lr=0.00e+00 | gnorm=3.8375e-01 | t=22.7s
|
| 5 |
+
[step 100] step= 100 | total=1.5914 | leaf=1.5914 | ancestor=0.0000 | lr=5.00e-06 | gnorm=5.7325e-01 | t=29.1s
|
| 6 |
+
[step 200] step= 200 | total=1.3936 | leaf=1.3936 | ancestor=0.0000 | lr=1.00e-05 | gnorm=2.9174e-01 | t=29.2s
|
| 7 |
+
[step 300] step= 300 | total=1.2734 | leaf=1.2734 | ancestor=0.0000 | lr=1.50e-05 | gnorm=1.9524e-01 | t=29.1s
|
| 8 |
+
[step 400] step= 400 | total=1.0994 | leaf=1.0994 | ancestor=0.0000 | lr=2.00e-05 | gnorm=1.2036e-01 | t=29.2s
|
| 9 |
+
[step 500] step= 500 | total=1.0670 | leaf=1.0670 | ancestor=0.0000 | lr=2.50e-05 | gnorm=1.0768e-01 | t=29.2s
|
| 10 |
+
[step 600] step= 600 | total=0.9905 | leaf=0.9905 | ancestor=0.0000 | lr=3.00e-05 | gnorm=3.2683e-01 | t=29.5s
|
| 11 |
+
[step 700] step= 700 | total=0.9605 | leaf=0.9605 | ancestor=0.0000 | lr=3.50e-05 | gnorm=5.9415e-01 | t=29.2s
|
| 12 |
+
[step 800] step= 800 | total=0.9025 | leaf=0.9025 | ancestor=0.0000 | lr=4.00e-05 | gnorm=9.1643e-01 | t=29.2s
|
| 13 |
+
[step 900] step= 900 | total=0.8343 | leaf=0.8343 | ancestor=0.0000 | lr=4.50e-05 | gnorm=3.9106e-01 | t=29.7s
|
| 14 |
+
[step 1000] [GRAD STATS] total_norm=7.1645e-01 max_single=4.2301e-01 (segment_embed.weight)
|
| 15 |
+
[step 1000] step= 1000 | total=0.8051 | leaf=0.8051 | ancestor=0.0000 | lr=5.00e-05 | gnorm=7.1645e-01 | t=29.2s
|
| 16 |
+
[step 1100] step= 1100 | total=0.7562 | leaf=0.7562 | ancestor=0.0000 | lr=5.50e-05 | gnorm=6.0970e-01 | t=29.2s
|
| 17 |
+
[step 1200] step= 1200 | total=0.7140 | leaf=0.7140 | ancestor=0.0000 | lr=6.00e-05 | gnorm=5.3551e-01 | t=29.2s
|
| 18 |
+
[step 1300] step= 1300 | total=0.7230 | leaf=0.7230 | ancestor=0.0000 | lr=6.50e-05 | gnorm=3.9475e-01 | t=29.2s
|
| 19 |
+
[step 1400] step= 1400 | total=0.7013 | leaf=0.7013 | ancestor=0.0000 | lr=7.00e-05 | gnorm=4.9973e-01 | t=29.2s
|
| 20 |
+
[step 1500] step= 1500 | total=0.7053 | leaf=0.7053 | ancestor=0.0000 | lr=7.50e-05 | gnorm=5.6499e-01 | t=29.2s
|
| 21 |
+
[step 1600] step= 1600 | total=0.7055 | leaf=0.7055 | ancestor=0.0000 | lr=8.00e-05 | gnorm=4.9817e-01 | t=29.5s
|
| 22 |
+
[step 1700] step= 1700 | total=0.6587 | leaf=0.6587 | ancestor=0.0000 | lr=8.50e-05 | gnorm=5.6639e-01 | t=29.5s
|
| 23 |
+
[step 1800] step= 1800 | total=0.6049 | leaf=0.6049 | ancestor=0.0000 | lr=9.00e-05 | gnorm=6.0167e-01 | t=29.2s
|
| 24 |
+
[step 1900] step= 1900 | total=0.6286 | leaf=0.6286 | ancestor=0.0000 | lr=9.50e-05 | gnorm=4.7993e-01 | t=29.2s
|
| 25 |
+
[step 2000] [GRAD STATS] total_norm=3.4630e-01 max_single=2.1110e-01 (segment_embed.weight)
|
| 26 |
+
[step 2000] step= 2000 | total=0.6332 | leaf=0.6332 | ancestor=0.0000 | lr=1.00e-04 | gnorm=3.4630e-01 | t=29.2s
|
| 27 |
+
[step 2100] step= 2100 | total=0.6094 | leaf=0.6094 | ancestor=0.0000 | lr=1.05e-04 | gnorm=4.0417e-01 | t=29.2s
|
| 28 |
+
[step 2200] step= 2200 | total=0.5840 | leaf=0.5840 | ancestor=0.0000 | lr=1.10e-04 | gnorm=4.4450e-01 | t=29.3s
|
| 29 |
+
[step 2300] step= 2300 | total=0.5579 | leaf=0.5579 | ancestor=0.0000 | lr=1.15e-04 | gnorm=4.7962e-01 | t=29.3s
|
| 30 |
+
[step 2400] step= 2400 | total=0.5623 | leaf=0.5623 | ancestor=0.0000 | lr=1.20e-04 | gnorm=4.0218e-01 | t=29.3s
|
| 31 |
+
[step 2500] step= 2500 | total=0.5763 | leaf=0.5763 | ancestor=0.0000 | lr=1.25e-04 | gnorm=4.6291e-01 | t=29.3s
|
| 32 |
+
[step 2600] step= 2600 | total=0.5422 | leaf=0.5422 | ancestor=0.0000 | lr=1.30e-04 | gnorm=4.1282e-01 | t=29.3s
|
| 33 |
+
[step 2700] step= 2700 | total=0.5406 | leaf=0.5406 | ancestor=0.0000 | lr=1.35e-04 | gnorm=3.2424e-01 | t=29.3s
|
| 34 |
+
[step 2800] step= 2800 | total=0.5319 | leaf=0.5319 | ancestor=0.0000 | lr=1.40e-04 | gnorm=3.6698e-01 | t=29.3s
|
| 35 |
+
[step 2900] step= 2900 | total=0.5207 | leaf=0.5207 | ancestor=0.0000 | lr=1.45e-04 | gnorm=4.3980e-01 | t=29.3s
|
| 36 |
+
[step 3000] [GRAD STATS] total_norm=4.6929e-01 max_single=3.2606e-01 (segment_embed.weight)
|
| 37 |
+
[step 3000] step= 3000 | total=0.5438 | leaf=0.5438 | ancestor=0.0000 | lr=1.50e-04 | gnorm=4.6929e-01 | t=29.3s
|
| 38 |
+
[step 3100] step= 3100 | total=0.5090 | leaf=0.5090 | ancestor=0.0000 | lr=1.55e-04 | gnorm=3.1783e-01 | t=29.3s
|
| 39 |
+
[step 3200] step= 3200 | total=0.5128 | leaf=0.5128 | ancestor=0.0000 | lr=1.60e-04 | gnorm=3.1766e-01 | t=29.3s
|
| 40 |
+
[step 3300] step= 3300 | total=0.5243 | leaf=0.5243 | ancestor=0.0000 | lr=1.65e-04 | gnorm=3.6358e-01 | t=29.3s
|
| 41 |
+
[step 3400] step= 3400 | total=0.5299 | leaf=0.5299 | ancestor=0.0000 | lr=1.70e-04 | gnorm=2.6806e-01 | t=29.3s
|
| 42 |
+
[step 3500] step= 3500 | total=0.5051 | leaf=0.5051 | ancestor=0.0000 | lr=1.75e-04 | gnorm=2.8595e-01 | t=29.3s
|
| 43 |
+
[step 3600] step= 3600 | total=0.5463 | leaf=0.5463 | ancestor=0.0000 | lr=1.80e-04 | gnorm=2.6060e-01 | t=29.3s
|
| 44 |
+
[step 3700] step= 3700 | total=0.5294 | leaf=0.5294 | ancestor=0.0000 | lr=1.85e-04 | gnorm=2.7364e-01 | t=29.3s
|
| 45 |
+
[step 3800] step= 3800 | total=0.5357 | leaf=0.5357 | ancestor=0.0000 | lr=1.90e-04 | gnorm=2.6996e-01 | t=29.3s
|
| 46 |
+
[step 3900] step= 3900 | total=0.5085 | leaf=0.5085 | ancestor=0.0000 | lr=1.95e-04 | gnorm=4.0343e-01 | t=29.3s
|
| 47 |
+
[step 4000] [GRAD STATS] total_norm=2.3813e-01 max_single=1.3049e-01 (segment_embed.weight)
|
| 48 |
+
[step 4000] step= 4000 | total=0.5289 | leaf=0.5289 | ancestor=0.0000 | lr=2.00e-04 | gnorm=2.3813e-01 | t=29.3s
|
| 49 |
+
[step 4100] step= 4100 | total=0.5130 | leaf=0.5130 | ancestor=0.0000 | lr=2.05e-04 | gnorm=2.6062e-01 | t=29.3s
|
| 50 |
+
[step 4200] step= 4200 | total=0.4895 | leaf=0.4895 | ancestor=0.0000 | lr=2.10e-04 | gnorm=2.2213e-01 | t=29.3s
|
| 51 |
+
[step 4300] step= 4300 | total=0.5240 | leaf=0.5240 | ancestor=0.0000 | lr=2.15e-04 | gnorm=2.4701e-01 | t=29.3s
|
| 52 |
+
[step 4400] step= 4400 | total=0.5291 | leaf=0.5291 | ancestor=0.0000 | lr=2.20e-04 | gnorm=2.5498e-01 | t=29.3s
|
| 53 |
+
[step 4500] step= 4500 | total=0.5562 | leaf=0.5562 | ancestor=0.0000 | lr=2.25e-04 | gnorm=2.3214e-01 | t=29.3s
|
| 54 |
+
[step 4600] step= 4600 | total=0.5664 | leaf=0.5664 | ancestor=0.0000 | lr=2.30e-04 | gnorm=3.0273e-01 | t=29.3s
|
| 55 |
+
[step 4700] step= 4700 | total=0.5264 | leaf=0.5264 | ancestor=0.0000 | lr=2.35e-04 | gnorm=2.7065e-01 | t=29.3s
|
| 56 |
+
[step 4800] step= 4800 | total=0.5463 | leaf=0.5463 | ancestor=0.0000 | lr=2.40e-04 | gnorm=2.3052e-01 | t=29.3s
|
| 57 |
+
[step 4900] step= 4900 | total=0.5710 | leaf=0.5710 | ancestor=0.0000 | lr=2.45e-04 | gnorm=2.5023e-01 | t=29.3s
|
| 58 |
+
[step 5000] [GRAD STATS] total_norm=2.0155e-01 max_single=9.7864e-02 (segment_embed.weight)
|
| 59 |
+
[step 5000] step= 5000 | total=0.5415 | leaf=0.5415 | ancestor=0.0000 | lr=2.50e-04 | gnorm=2.0155e-01 | t=29.3s
|
| 60 |
+
torch._dynamo.utils.warn_once(msg)
|
| 61 |
+
VAL | loss_leaf=2.1364 | loss_ancestor=0.0000 | loss_total=2.1364
|
| 62 |
+
full-denoise training: 2%|▉ | 8000/500000 [39:42<40:00:27, 3.42it/s, leaf=0.6747, lr=4.0e-04]
|
| 63 |
+
[step 5100] step= 5100 | total=0.5736 | leaf=0.5736 | ancestor=0.0000 | lr=2.55e-04 | gnorm=3.7978e-01 | t=45.7s
|
| 64 |
+
[step 5200] step= 5200 | total=0.5348 | leaf=0.5348 | ancestor=0.0000 | lr=2.60e-04 | gnorm=2.1980e-01 | t=29.3s
|
| 65 |
+
[step 5300] step= 5300 | total=0.5291 | leaf=0.5291 | ancestor=0.0000 | lr=2.65e-04 | gnorm=2.9401e-01 | t=29.3s
|
| 66 |
+
[step 5400] step= 5400 | total=0.5713 | leaf=0.5713 | ancestor=0.0000 | lr=2.70e-04 | gnorm=2.0217e-01 | t=29.3s
|
| 67 |
+
[step 5500] step= 5500 | total=0.5838 | leaf=0.5838 | ancestor=0.0000 | lr=2.75e-04 | gnorm=2.0907e-01 | t=29.3s
|
| 68 |
+
[step 5600] step= 5600 | total=0.5353 | leaf=0.5353 | ancestor=0.0000 | lr=2.80e-04 | gnorm=2.1685e-01 | t=29.3s
|
| 69 |
+
[step 5700] step= 5700 | total=0.5633 | leaf=0.5633 | ancestor=0.0000 | lr=2.85e-04 | gnorm=1.9464e-01 | t=29.3s
|
| 70 |
+
[step 5800] step= 5800 | total=0.5889 | leaf=0.5889 | ancestor=0.0000 | lr=2.90e-04 | gnorm=2.5097e-01 | t=29.3s
|
| 71 |
+
[step 5900] step= 5900 | total=0.5585 | leaf=0.5585 | ancestor=0.0000 | lr=2.95e-04 | gnorm=2.9532e-01 | t=29.3s
|
| 72 |
+
[step 6000] [GRAD STATS] total_norm=2.6553e-01 max_single=1.9400e-01 (segment_embed.weight)
|
| 73 |
+
[step 6000] step= 6000 | total=0.5588 | leaf=0.5588 | ancestor=0.0000 | lr=3.00e-04 | gnorm=2.6553e-01 | t=29.3s
|
| 74 |
+
[step 6100] step= 6100 | total=0.5620 | leaf=0.5620 | ancestor=0.0000 | lr=3.05e-04 | gnorm=2.5857e-01 | t=29.3s
|
| 75 |
+
[step 6200] step= 6200 | total=0.5830 | leaf=0.5830 | ancestor=0.0000 | lr=3.10e-04 | gnorm=1.8543e-01 | t=29.3s
|
| 76 |
+
[step 6300] step= 6300 | total=0.5608 | leaf=0.5608 | ancestor=0.0000 | lr=3.15e-04 | gnorm=2.9996e-01 | t=29.3s
|
| 77 |
+
[step 6400] step= 6400 | total=0.5964 | leaf=0.5964 | ancestor=0.0000 | lr=3.20e-04 | gnorm=1.9583e-01 | t=29.3s
|
| 78 |
+
[step 6500] step= 6500 | total=0.6098 | leaf=0.6098 | ancestor=0.0000 | lr=3.25e-04 | gnorm=1.8408e-01 | t=29.3s
|
| 79 |
+
[step 6600] step= 6600 | total=0.5706 | leaf=0.5706 | ancestor=0.0000 | lr=3.30e-04 | gnorm=1.9030e-01 | t=29.3s
|
| 80 |
+
[step 6700] step= 6700 | total=0.5957 | leaf=0.5957 | ancestor=0.0000 | lr=3.35e-04 | gnorm=2.1952e-01 | t=29.3s
|
| 81 |
+
[step 6800] step= 6800 | total=0.5957 | leaf=0.5957 | ancestor=0.0000 | lr=3.40e-04 | gnorm=2.1514e-01 | t=29.3s
|
| 82 |
+
[step 6900] step= 6900 | total=0.5857 | leaf=0.5857 | ancestor=0.0000 | lr=3.45e-04 | gnorm=2.5336e-01 | t=29.3s
|
| 83 |
+
[step 7000] [GRAD STATS] total_norm=1.7336e-01 max_single=7.8933e-02 (segment_embed.weight)
|
| 84 |
+
[step 7000] step= 7000 | total=0.6078 | leaf=0.6078 | ancestor=0.0000 | lr=3.50e-04 | gnorm=1.7336e-01 | t=29.3s
|
| 85 |
+
[step 7100] step= 7100 | total=0.6113 | leaf=0.6113 | ancestor=0.0000 | lr=3.55e-04 | gnorm=1.6350e-01 | t=29.3s
|
| 86 |
+
[step 7200] step= 7200 | total=0.5892 | leaf=0.5892 | ancestor=0.0000 | lr=3.60e-04 | gnorm=1.6004e-01 | t=29.3s
|
| 87 |
+
[step 7300] step= 7300 | total=0.6306 | leaf=0.6306 | ancestor=0.0000 | lr=3.65e-04 | gnorm=2.5173e-01 | t=29.3s
|
| 88 |
+
[step 7400] step= 7400 | total=0.5832 | leaf=0.5832 | ancestor=0.0000 | lr=3.70e-04 | gnorm=1.6664e-01 | t=29.3s
|
| 89 |
+
[step 7500] step= 7500 | total=0.6065 | leaf=0.6065 | ancestor=0.0000 | lr=3.75e-04 | gnorm=3.4408e-01 | t=29.3s
|
| 90 |
+
[step 7600] step= 7600 | total=0.6681 | leaf=0.6681 | ancestor=0.0000 | lr=3.80e-04 | gnorm=1.7291e-01 | t=29.3s
|
| 91 |
+
[step 7700] step= 7700 | total=0.6459 | leaf=0.6459 | ancestor=0.0000 | lr=3.85e-04 | gnorm=3.2149e-01 | t=29.3s
|
| 92 |
+
[step 7800] step= 7800 | total=0.5974 | leaf=0.5974 | ancestor=0.0000 | lr=3.90e-04 | gnorm=1.5246e-01 | t=29.3s
|
| 93 |
+
[step 7900] step= 7900 | total=0.6196 | leaf=0.6196 | ancestor=0.0000 | lr=3.95e-04 | gnorm=2.4295e-01 | t=29.3s
|
| 94 |
+
[step 8000] [GRAD STATS] total_norm=1.7910e-01 max_single=9.7511e-02 (segment_embed.weight)
|
| 95 |
+
[step 8000] step= 8000 | total=0.5858 | leaf=0.5858 | ancestor=0.0000 | lr=4.00e-04 | gnorm=1.7910e-01 | t=29.3s
|
| 96 |
+
[step 8100] step= 8100 | total=0.6700 | leaf=0.6700 | ancestor=0.0000 | lr=4.05e-04 | gnorm=1.7389e-01 | t=29.3s
|
| 97 |
+
[step 8200] step= 8200 | total=0.6395 | leaf=0.6395 | ancestor=0.0000 | lr=4.10e-04 | gnorm=1.6934e-01 | t=29.3s
|
| 98 |
+
[step 8300] step= 8300 | total=0.6677 | leaf=0.6677 | ancestor=0.0000 | lr=4.15e-04 | gnorm=1.8874e-01 | t=29.3s
|
| 99 |
+
[step 8400] step= 8400 | total=0.6632 | leaf=0.6632 | ancestor=0.0000 | lr=4.20e-04 | gnorm=1.9541e-01 | t=29.3s
|
| 100 |
+
[step 8500] step= 8500 | total=0.6453 | leaf=0.6453 | ancestor=0.0000 | lr=4.25e-04 | gnorm=1.6743e-01 | t=29.3s
|
| 101 |
+
[step 8600] step= 8600 | total=0.7116 | leaf=0.7116 | ancestor=0.0000 | lr=4.30e-04 | gnorm=1.4324e-01 | t=29.3s
|
| 102 |
+
[step 8700] step= 8700 | total=0.6398 | leaf=0.6398 | ancestor=0.0000 | lr=4.35e-04 | gnorm=2.1262e-01 | t=29.3s
|
| 103 |
+
[step 8800] step= 8800 | total=0.6961 | leaf=0.6961 | ancestor=0.0000 | lr=4.40e-04 | gnorm=1.8267e-01 | t=29.3s
|
| 104 |
+
[step 8900] step= 8900 | total=0.7149 | leaf=0.7149 | ancestor=0.0000 | lr=4.45e-04 | gnorm=1.7045e-01 | t=29.3s
|
| 105 |
+
[step 9000] [GRAD STATS] total_norm=1.5975e-01 max_single=7.0399e-02 (vocab_embed.embedding)
|
| 106 |
+
[step 9000] step= 9000 | total=0.7184 | leaf=0.7184 | ancestor=0.0000 | lr=4.50e-04 | gnorm=1.5975e-01 | t=29.3s
|
| 107 |
+
[step 9100] step= 9100 | total=0.6429 | leaf=0.6429 | ancestor=0.0000 | lr=4.55e-04 | gnorm=1.8638e-01 | t=29.3s
|
| 108 |
+
[step 9200] step= 9200 | total=0.6866 | leaf=0.6866 | ancestor=0.0000 | lr=4.60e-04 | gnorm=1.7601e-01 | t=29.3s
|
| 109 |
+
[step 9300] step= 9300 | total=0.6645 | leaf=0.6645 | ancestor=0.0000 | lr=4.65e-04 | gnorm=2.2818e-01 | t=29.3s
|
| 110 |
+
[step 9400] step= 9400 | total=0.6431 | leaf=0.6431 | ancestor=0.0000 | lr=4.70e-04 | gnorm=2.0105e-01 | t=29.3s
|
| 111 |
+
[step 9500] step= 9500 | total=0.7153 | leaf=0.7153 | ancestor=0.0000 | lr=4.75e-04 | gnorm=2.8180e-01 | t=29.3s
|
| 112 |
+
[step 9600] step= 9600 | total=0.7546 | leaf=0.7546 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.6927e-01 | t=29.3s
|
| 113 |
+
[step 9700] step= 9700 | total=0.6992 | leaf=0.6992 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.6926e-01 | t=29.3s
|
| 114 |
+
[step 9800] step= 9800 | total=0.7244 | leaf=0.7244 | ancestor=0.0000 | lr=4.90e-04 | gnorm=3.7622e-01 | t=29.3s
|
| 115 |
+
[step 9900] step= 9900 | total=0.7547 | leaf=0.7547 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.0584e-01 | t=29.3s
|
| 116 |
+
[step 10000] [GRAD STATS] total_norm=1.5987e-01 max_single=7.1247e-02 (vocab_embed.embedding)
|
| 117 |
+
[step 10000] step= 10000 | total=0.7414 | leaf=0.7414 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5987e-01 | t=29.3s
|
| 118 |
+
VAL | loss_leaf=1.9421 | loss_ancestor=0.0000 | loss_total=1.9421
|
| 119 |
+
Saved checkpoint: outputs/sad_new/ckpt_10000.pt
|
| 120 |
+
[step 10100] step= 10100 | total=0.7739 | leaf=0.7739 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6926e-01 | t=45.4s
|
| 121 |
+
[step 10200] step= 10200 | total=0.6819 | leaf=0.6819 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7476e-01 | t=29.4s
|
| 122 |
+
[step 10300] step= 10300 | total=0.7581 | leaf=0.7581 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7507e-01 | t=29.4s
|
| 123 |
+
[step 10400] step= 10400 | total=0.7880 | leaf=0.7880 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.9516e-01 | t=29.4s
|
| 124 |
+
[step 10500] step= 10500 | total=0.7351 | leaf=0.7351 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6056e-01 | t=29.4s
|
| 125 |
+
[step 10600] step= 10600 | total=0.8150 | leaf=0.8150 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6791e-01 | t=29.4s
|
| 126 |
+
[step 10700] step= 10700 | total=0.8061 | leaf=0.8061 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5447e-01 | t=29.4s
|
| 127 |
+
[step 10800] step= 10800 | total=0.7727 | leaf=0.7727 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6158e-01 | t=29.4s
|
| 128 |
+
[step 10900] step= 10900 | total=0.8301 | leaf=0.8301 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.0469e-01 | t=29.4s
|
| 129 |
+
[step 11000] [GRAD STATS] total_norm=2.0829e-01 max_single=1.0048e-01 (segment_embed.weight)
|
| 130 |
+
[step 11000] step= 11000 | total=0.8940 | leaf=0.8940 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.0829e-01 | t=29.4s
|
| 131 |
+
[step 11100] step= 11100 | total=0.7927 | leaf=0.7927 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6163e-01 | t=29.4s
|
| 132 |
+
[step 11200] step= 11200 | total=0.8689 | leaf=0.8689 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.8882e-01 | t=29.4s
|
| 133 |
+
[step 11300] step= 11300 | total=0.7800 | leaf=0.7800 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.1572e-01 | t=29.4s
|
| 134 |
+
[step 11400] step= 11400 | total=0.8723 | leaf=0.8723 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4664e-01 | t=29.3s
|
| 135 |
+
[step 11500] step= 11500 | total=0.9078 | leaf=0.9078 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5177e-01 | t=29.3s
|
| 136 |
+
[step 11600] step= 11600 | total=0.8872 | leaf=0.8872 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5267e-01 | t=29.3s
|
| 137 |
+
[step 11700] step= 11700 | total=0.9044 | leaf=0.9044 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5529e-01 | t=29.3s
|
| 138 |
+
[step 11800] step= 11800 | total=0.8230 | leaf=0.8230 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3433e-01 | t=29.3s
|
| 139 |
+
[step 11900] step= 11900 | total=0.8731 | leaf=0.8731 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5325e-01 | t=29.3s
|
| 140 |
+
[step 12000] [GRAD STATS] total_norm=1.9457e-01 max_single=8.3195e-02 (segment_embed.weight)
|
| 141 |
+
[step 12000] step= 12000 | total=0.9163 | leaf=0.9163 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.9457e-01 | t=29.4s
|
| 142 |
+
[step 12100] step= 12100 | total=0.8981 | leaf=0.8981 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5608e-01 | t=29.4s
|
| 143 |
+
[step 12200] step= 12200 | total=0.8626 | leaf=0.8626 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.2983e-01 | t=29.4s
|
| 144 |
+
[step 12300] step= 12300 | total=0.9135 | leaf=0.9135 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5624e-01 | t=29.4s
|
| 145 |
+
[step 12400] step= 12400 | total=0.8660 | leaf=0.8660 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7920e-01 | t=29.4s
|
| 146 |
+
[step 12500] step= 12500 | total=0.9005 | leaf=0.9005 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5316e-01 | t=29.4s
|
| 147 |
+
[step 12600] step= 12600 | total=0.8778 | leaf=0.8778 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4907e-01 | t=29.4s
|
| 148 |
+
[step 12700] step= 12700 | total=0.9468 | leaf=0.9468 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4502e-01 | t=29.4s
|
| 149 |
+
[step 12800] step= 12800 | total=0.8915 | leaf=0.8915 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4995e-01 | t=29.4s
|
| 150 |
+
[step 12900] step= 12900 | total=0.8977 | leaf=0.8977 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.1588e-01 | t=29.4s
|
| 151 |
+
[step 13000] [GRAD STATS] total_norm=1.5089e-01 max_single=6.5135e-02 (vocab_embed.embedding)
|
| 152 |
+
[step 13000] step= 13000 | total=0.9614 | leaf=0.9614 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5089e-01 | t=29.4s
|
| 153 |
+
[step 13100] step= 13100 | total=0.9129 | leaf=0.9129 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4044e-01 | t=29.4s
|
| 154 |
+
[step 13200] step= 13200 | total=0.9574 | leaf=0.9574 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3886e-01 | t=29.4s
|
| 155 |
+
[step 13300] step= 13300 | total=0.9646 | leaf=0.9646 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3701e-01 | t=29.4s
|
| 156 |
+
[step 13400] step= 13400 | total=0.9875 | leaf=0.9875 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6192e-01 | t=29.4s
|
| 157 |
+
[step 13500] step= 13500 | total=0.9406 | leaf=0.9406 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.2551e-01 | t=29.4s
|
| 158 |
+
[step 13600] step= 13600 | total=1.0110 | leaf=1.0110 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6842e-01 | t=29.4s
|
| 159 |
+
[step 13700] step= 13700 | total=1.0266 | leaf=1.0266 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5130e-01 | t=29.4s
|
| 160 |
+
[step 13800] step= 13800 | total=1.0426 | leaf=1.0426 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7490e-01 | t=29.4s
|
| 161 |
+
[step 13900] step= 13900 | total=1.0150 | leaf=1.0150 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5256e-01 | t=29.4s
|
| 162 |
+
[step 14000] [GRAD STATS] total_norm=1.7383e-01 max_single=6.5421e-02 (vocab_embed.embedding)
|
| 163 |
+
[step 14000] step= 14000 | total=1.0668 | leaf=1.0668 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7383e-01 | t=29.4s
|
| 164 |
+
[step 14100] step= 14100 | total=0.9980 | leaf=0.9980 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4240e-01 | t=29.4s
|
| 165 |
+
[step 14200] step= 14200 | total=1.0411 | leaf=1.0411 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.0927e-01 | t=29.4s
|
| 166 |
+
[step 14300] step= 14300 | total=1.0437 | leaf=1.0437 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5478e-01 | t=29.4s
|
| 167 |
+
[step 14400] step= 14400 | total=0.9989 | leaf=0.9989 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5301e-01 | t=29.4s
|
| 168 |
+
[step 14500] step= 14500 | total=1.1024 | leaf=1.1024 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5488e-01 | t=29.4s
|
| 169 |
+
[step 14600] step= 14600 | total=0.9661 | leaf=0.9661 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4085e-01 | t=29.4s
|
| 170 |
+
[step 14700] step= 14700 | total=1.1135 | leaf=1.1135 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.1737e-01 | t=29.4s
|
| 171 |
+
[step 14800] step= 14800 | total=1.0516 | leaf=1.0516 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4489e-01 | t=29.4s
|
| 172 |
+
[step 14900] step= 14900 | total=1.0699 | leaf=1.0699 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6633e-01 | t=29.4s
|
| 173 |
+
[step 15000] [GRAD STATS] total_norm=1.4045e-01 max_single=6.2739e-02 (vocab_embed.embedding)
|
| 174 |
+
[step 15000] step= 15000 | total=1.1320 | leaf=1.1320 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4045e-01 | t=29.4s
|
| 175 |
+
VAL | loss_leaf=1.8367 | loss_ancestor=0.0000 | loss_total=1.8367
|
| 176 |
+
[step 15100] step= 15100 | total=1.0739 | leaf=1.0739 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.1349e-01 | t=35.2s
|
| 177 |
+
[step 15200] step= 15200 | total=1.1435 | leaf=1.1435 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.9734e-01 | t=29.4s
|
| 178 |
+
[step 15300] step= 15300 | total=1.1640 | leaf=1.1640 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.8529e-01 | t=29.4s
|
| 179 |
+
[step 15400] step= 15400 | total=1.1545 | leaf=1.1545 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7091e-01 | t=29.4s
|
| 180 |
+
[step 15500] step= 15500 | total=1.1831 | leaf=1.1831 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6683e-01 | t=55.2s
|
| 181 |
+
[step 15600] step= 15600 | total=1.1938 | leaf=1.1938 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5709e-01 | t=29.4s
|
| 182 |
+
[step 15700] step= 15700 | total=1.1706 | leaf=1.1706 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4788e-01 | t=29.4s
|
| 183 |
+
[step 15800] step= 15800 | total=1.1637 | leaf=1.1637 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5946e-01 | t=29.4s
|
| 184 |
+
[step 15900] step= 15900 | total=1.1433 | leaf=1.1433 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7585e-01 | t=29.4s
|
| 185 |
+
[step 16000] [GRAD STATS] total_norm=1.6789e-01 max_single=9.2900e-02 (cond_bias)
|
| 186 |
+
[step 16000] step= 16000 | total=1.2219 | leaf=1.2219 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6789e-01 | t=29.4s
|
| 187 |
+
[step 16100] step= 16100 | total=1.2127 | leaf=1.2127 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6676e-01 | t=29.4s
|
| 188 |
+
[step 16200] step= 16200 | total=1.2011 | leaf=1.2011 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7028e-01 | t=29.4s
|
| 189 |
+
[step 16300] step= 16300 | total=1.3346 | leaf=1.3346 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4629e-01 | t=29.4s
|
| 190 |
+
[step 16400] step= 16400 | total=1.2603 | leaf=1.2603 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7873e-01 | t=29.4s
|
| 191 |
+
[step 16500] step= 16500 | total=1.2919 | leaf=1.2919 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3446e-01 | t=29.4s
|
| 192 |
+
[step 16600] step= 16600 | total=1.1463 | leaf=1.1463 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4392e-01 | t=29.4s
|
| 193 |
+
[step 16700] step= 16700 | total=1.2981 | leaf=1.2981 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.3489e-01 | t=29.4s
|
| 194 |
+
[step 16800] step= 16800 | total=1.2435 | leaf=1.2435 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4204e-01 | t=29.4s
|
| 195 |
+
[step 16900] step= 16900 | total=1.3194 | leaf=1.3194 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3536e-01 | t=29.4s
|
| 196 |
+
[step 17000] [GRAD STATS] total_norm=1.3760e-01 max_single=6.4880e-02 (vocab_embed.embedding)
|
| 197 |
+
[step 17000] step= 17000 | total=1.2461 | leaf=1.2461 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3760e-01 | t=29.4s
|
| 198 |
+
[step 17100] step= 17100 | total=1.2677 | leaf=1.2677 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4079e-01 | t=29.4s
|
| 199 |
+
[step 17200] step= 17200 | total=1.3583 | leaf=1.3583 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4817e-01 | t=29.4s
|
| 200 |
+
[step 17300] step= 17300 | total=1.4137 | leaf=1.4137 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.1299e-01 | t=29.4s
|
| 201 |
+
[step 17400] step= 17400 | total=1.4111 | leaf=1.4111 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.1329e-01 | t=29.4s
|
| 202 |
+
[step 17500] step= 17500 | total=1.3274 | leaf=1.3274 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3216e-01 | t=29.4s
|
| 203 |
+
[step 17600] step= 17600 | total=1.3046 | leaf=1.3046 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6520e-01 | t=29.4s
|
| 204 |
+
[step 17700] step= 17700 | total=1.4228 | leaf=1.4228 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4407e-01 | t=29.4s
|
| 205 |
+
[step 17800] step= 17800 | total=1.5297 | leaf=1.5297 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.5785e-01 | t=29.4s
|
| 206 |
+
[step 17900] step= 17900 | total=1.4230 | leaf=1.4230 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3392e-01 | t=29.4s
|
| 207 |
+
[step 18000] [GRAD STATS] total_norm=1.3124e-01 max_single=5.9642e-02 (vocab_embed.embedding)
|
| 208 |
+
[step 18000] step= 18000 | total=1.3841 | leaf=1.3841 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3124e-01 | t=29.4s
|
| 209 |
+
[step 18100] step= 18100 | total=1.4962 | leaf=1.4962 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3159e-01 | t=29.4s
|
| 210 |
+
[step 18200] step= 18200 | total=1.5236 | leaf=1.5236 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4609e-01 | t=29.4s
|
| 211 |
+
[step 18300] step= 18300 | total=1.5150 | leaf=1.5150 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5446e-01 | t=29.4s
|
| 212 |
+
[step 18400] step= 18400 | total=1.4210 | leaf=1.4210 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3999e-01 | t=29.4s
|
| 213 |
+
[step 18500] step= 18500 | total=1.5169 | leaf=1.5169 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.8729e-01 | t=29.4s
|
| 214 |
+
[step 18600] step= 18600 | total=1.5769 | leaf=1.5769 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6038e-01 | t=29.4s
|
| 215 |
+
[step 18700] step= 18700 | total=1.5137 | leaf=1.5137 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4560e-01 | t=29.4s
|
| 216 |
+
[step 18800] step= 18800 | total=1.5902 | leaf=1.5902 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.6095e-01 | t=29.4s
|
| 217 |
+
[step 18900] step= 18900 | total=1.5947 | leaf=1.5947 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.7888e-01 | t=29.4s
|
| 218 |
+
[step 19000] [GRAD STATS] total_norm=1.7146e-01 max_single=9.3498e-02 (cond_bias)
|
| 219 |
+
[step 19000] step= 19000 | total=1.6192 | leaf=1.6192 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7146e-01 | t=29.4s
|
| 220 |
+
[step 19100] step= 19100 | total=1.5205 | leaf=1.5205 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.8163e-01 | t=29.4s
|
| 221 |
+
[step 19200] step= 19200 | total=1.6280 | leaf=1.6280 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4784e-01 | t=29.4s
|
| 222 |
+
[step 19300] step= 19300 | total=1.6067 | leaf=1.6067 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4360e-01 | t=29.4s
|
| 223 |
+
[step 19400] step= 19400 | total=1.7316 | leaf=1.7316 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3859e-01 | t=29.4s
|
| 224 |
+
[step 19500] step= 19500 | total=1.6483 | leaf=1.6483 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4121e-01 | t=29.4s
|
| 225 |
+
[step 19600] step= 19600 | total=1.5720 | leaf=1.5720 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.3391e-01 | t=29.4s
|
| 226 |
+
[step 19700] step= 19700 | total=1.7926 | leaf=1.7926 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.8951e-01 | t=29.4s
|
| 227 |
+
[step 19800] step= 19800 | total=1.7450 | leaf=1.7450 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5311e-01 | t=29.4s
|
| 228 |
+
[step 19900] step= 19900 | total=1.7351 | leaf=1.7351 | ancestor=0.0000 | lr=5.00e-04 | gnorm=2.0989e-01 | t=29.4s
|
| 229 |
+
[step 20000] [GRAD STATS] total_norm=1.7989e-01 max_single=9.8951e-02 (cond_bias)
|
| 230 |
+
[step 20000] step= 20000 | total=1.7623 | leaf=1.7623 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7989e-01 | t=29.4s
|
| 231 |
+
VAL | loss_leaf=1.7525 | loss_ancestor=0.0000 | loss_total=1.7525
|
| 232 |
+
Saved checkpoint: outputs/sad_new/ckpt_20000.pt
|
| 233 |
+
[step 20100] step= 20100 | total=1.6886 | leaf=1.6886 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.7254e-01 | t=45.6s
|
| 234 |
+
[step 20200] step= 20200 | total=1.7715 | leaf=1.7715 | ancestor=0.0000 | lr=5.00e-04 | gnorm=3.0210e-01 | t=29.4s
|
| 235 |
+
[step 20300] step= 20300 | total=1.8338 | leaf=1.8338 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.5611e-01 | t=29.4s
|
| 236 |
+
[step 20400] step= 20400 | total=1.6831 | leaf=1.6831 | ancestor=0.0000 | lr=5.00e-04 | gnorm=1.4743e-01 | t=29.4s
|
| 237 |
+
[step 20500] step= 20500 | total=1.7504 | leaf=1.7504 | ancestor=0.0000 | lr=4.99e-04 | gnorm=2.2556e-01 | t=29.4s
|
| 238 |
+
[step 20600] step= 20600 | total=1.7600 | leaf=1.7600 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.5341e-01 | t=29.4s
|
| 239 |
+
[step 20700] step= 20700 | total=1.8589 | leaf=1.8589 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.5460e-01 | t=29.4s
|
| 240 |
+
[step 20800] step= 20800 | total=1.7869 | leaf=1.7869 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3988e-01 | t=29.4s
|
| 241 |
+
[step 20900] step= 20900 | total=1.5746 | leaf=1.5746 | ancestor=0.0000 | lr=4.99e-04 | gnorm=2.3450e-01 | t=29.4s
|
| 242 |
+
[step 21000] [GRAD STATS] total_norm=1.4652e-01 max_single=6.0327e-02 (vocab_embed.embedding)
|
| 243 |
+
[step 21000] step= 21000 | total=1.7434 | leaf=1.7434 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.4652e-01 | t=29.4s
|
| 244 |
+
[step 21100] step= 21100 | total=1.7357 | leaf=1.7357 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.9452e-01 | t=29.4s
|
| 245 |
+
[step 21200] step= 21200 | total=1.7299 | leaf=1.7299 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.7895e-01 | t=29.4s
|
| 246 |
+
[step 21300] step= 21300 | total=1.6739 | leaf=1.6739 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.9393e-01 | t=29.4s
|
| 247 |
+
[step 21400] step= 21400 | total=1.7975 | leaf=1.7975 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3822e-01 | t=29.4s
|
| 248 |
+
[step 21500] step= 21500 | total=1.7673 | leaf=1.7673 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6881e-01 | t=29.4s
|
| 249 |
+
[step 21600] step= 21600 | total=1.7797 | leaf=1.7797 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.8687e-01 | t=29.4s
|
| 250 |
+
[step 21700] step= 21700 | total=1.7434 | leaf=1.7434 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6299e-01 | t=29.4s
|
| 251 |
+
[step 21800] step= 21800 | total=1.7318 | leaf=1.7318 | ancestor=0.0000 | lr=4.99e-04 | gnorm=3.4018e-01 | t=29.4s
|
| 252 |
+
[step 21900] step= 21900 | total=1.7717 | leaf=1.7717 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3910e-01 | t=29.4s
|
| 253 |
+
[step 22000] [GRAD STATS] total_norm=1.2967e-01 max_single=5.8508e-02 (vocab_embed.embedding)
|
| 254 |
+
[step 22000] step= 22000 | total=1.7006 | leaf=1.7006 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.2967e-01 | t=29.4s
|
| 255 |
+
[step 22100] step= 22100 | total=1.7341 | leaf=1.7341 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3603e-01 | t=29.4s
|
| 256 |
+
[step 22200] step= 22200 | total=1.7150 | leaf=1.7150 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3306e-01 | t=29.4s
|
| 257 |
+
[step 22300] step= 22300 | total=1.8648 | leaf=1.8648 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6646e-01 | t=29.4s
|
| 258 |
+
[step 22400] step= 22400 | total=1.7778 | leaf=1.7778 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.4134e-01 | t=29.4s
|
| 259 |
+
[step 22500] step= 22500 | total=1.6478 | leaf=1.6478 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.5063e-01 | t=29.4s
|
| 260 |
+
[step 22600] step= 22600 | total=1.8154 | leaf=1.8154 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.2957e-01 | t=29.4s
|
| 261 |
+
[step 22700] step= 22700 | total=1.7328 | leaf=1.7328 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3625e-01 | t=29.4s
|
| 262 |
+
[step 22800] step= 22800 | total=1.7221 | leaf=1.7221 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.5817e-01 | t=29.4s
|
| 263 |
+
[step 22900] step= 22900 | total=1.6522 | leaf=1.6522 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.9462e-01 | t=29.4s
|
| 264 |
+
[step 23000] [GRAD STATS] total_norm=2.5928e-01 max_single=2.1065e-01 (cond_bias)
|
| 265 |
+
[step 23000] step= 23000 | total=1.8731 | leaf=1.8731 | ancestor=0.0000 | lr=4.99e-04 | gnorm=2.5928e-01 | t=29.4s
|
| 266 |
+
[step 23100] step= 23100 | total=1.7911 | leaf=1.7911 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6765e-01 | t=29.4s
|
| 267 |
+
[step 23200] step= 23200 | total=1.8432 | leaf=1.8432 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.4398e-01 | t=29.4s
|
| 268 |
+
[step 23300] step= 23300 | total=1.7290 | leaf=1.7290 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.4240e-01 | t=29.4s
|
| 269 |
+
[step 23400] step= 23400 | total=1.8196 | leaf=1.8196 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.5482e-01 | t=29.4s
|
| 270 |
+
[step 23500] step= 23500 | total=1.7221 | leaf=1.7221 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3920e-01 | t=29.4s
|
| 271 |
+
[step 23600] step= 23600 | total=1.7370 | leaf=1.7370 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.7009e-01 | t=29.4s
|
| 272 |
+
[step 23700] step= 23700 | total=1.7568 | leaf=1.7568 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.7777e-01 | t=29.4s
|
| 273 |
+
[step 23800] step= 23800 | total=1.6793 | leaf=1.6793 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.2617e-01 | t=29.4s
|
| 274 |
+
[step 23900] step= 23900 | total=1.8216 | leaf=1.8216 | ancestor=0.0000 | lr=4.99e-04 | gnorm=2.1555e-01 | t=29.4s
|
| 275 |
+
[step 24000] [GRAD STATS] total_norm=1.2327e-01 max_single=5.2885e-02 (vocab_embed.embedding)
|
| 276 |
+
[step 24000] step= 24000 | total=1.7934 | leaf=1.7934 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.2327e-01 | t=29.4s
|
| 277 |
+
[step 24100] step= 24100 | total=1.7111 | leaf=1.7111 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6104e-01 | t=29.4s
|
| 278 |
+
[step 24200] step= 24200 | total=1.6929 | leaf=1.6929 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3571e-01 | t=29.4s
|
| 279 |
+
[step 24300] step= 24300 | total=1.7155 | leaf=1.7155 | ancestor=0.0000 | lr=4.99e-04 | gnorm=2.0750e-01 | t=29.4s
|
| 280 |
+
[step 24400] step= 24400 | total=1.7121 | leaf=1.7121 | ancestor=0.0000 | lr=4.99e-04 | gnorm=2.0442e-01 | t=29.4s
|
| 281 |
+
[step 24500] step= 24500 | total=1.6285 | leaf=1.6285 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.4038e-01 | t=29.4s
|
| 282 |
+
[step 24600] step= 24600 | total=1.7254 | leaf=1.7254 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6927e-01 | t=29.4s
|
| 283 |
+
[step 24700] step= 24700 | total=1.8487 | leaf=1.8487 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.9439e-01 | t=29.4s
|
| 284 |
+
[step 24800] step= 24800 | total=1.7096 | leaf=1.7096 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.5923e-01 | t=29.4s
|
| 285 |
+
[step 24900] step= 24900 | total=1.6016 | leaf=1.6016 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.8388e-01 | t=29.4s
|
| 286 |
+
[step 25000] [GRAD STATS] total_norm=1.3537e-01 max_single=5.8594e-02 (vocab_embed.embedding)
|
| 287 |
+
[step 25000] step= 25000 | total=1.6545 | leaf=1.6545 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3537e-01 | t=29.4s
|
| 288 |
+
VAL | loss_leaf=1.7505 | loss_ancestor=0.0000 | loss_total=1.7505
|
| 289 |
+
[step 25100] step= 25100 | total=1.6914 | leaf=1.6914 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6450e-01 | t=35.2s
|
| 290 |
+
[step 25200] step= 25200 | total=1.6766 | leaf=1.6766 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6239e-01 | t=29.4s
|
| 291 |
+
[step 25300] step= 25300 | total=1.7789 | leaf=1.7789 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.7874e-01 | t=29.4s
|
| 292 |
+
[step 25400] step= 25400 | total=1.6049 | leaf=1.6049 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.4512e-01 | t=29.4s
|
| 293 |
+
[step 25500] step= 25500 | total=1.7757 | leaf=1.7757 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3510e-01 | t=29.4s
|
| 294 |
+
[step 25600] step= 25600 | total=1.7050 | leaf=1.7050 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.7927e-01 | t=29.4s
|
| 295 |
+
[step 25700] step= 25700 | total=1.7098 | leaf=1.7098 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.8071e-01 | t=29.4s
|
| 296 |
+
[step 25800] step= 25800 | total=1.7394 | leaf=1.7394 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6597e-01 | t=29.4s
|
| 297 |
+
[step 25900] step= 25900 | total=1.6429 | leaf=1.6429 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.5123e-01 | t=29.4s
|
| 298 |
+
[step 26000] [GRAD STATS] total_norm=2.7414e-01 max_single=1.8435e-01 (cond_bias)
|
| 299 |
+
[step 26000] step= 26000 | total=1.7766 | leaf=1.7766 | ancestor=0.0000 | lr=4.99e-04 | gnorm=2.7414e-01 | t=29.4s
|
| 300 |
+
[step 26100] step= 26100 | total=1.7383 | leaf=1.7383 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.8458e-01 | t=29.4s
|
| 301 |
+
[step 26200] step= 26200 | total=1.7333 | leaf=1.7333 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3934e-01 | t=29.4s
|
| 302 |
+
[step 26300] step= 26300 | total=1.8642 | leaf=1.8642 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6058e-01 | t=29.4s
|
| 303 |
+
[step 26400] step= 26400 | total=1.7132 | leaf=1.7132 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3431e-01 | t=29.4s
|
| 304 |
+
[step 26500] step= 26500 | total=1.8217 | leaf=1.8217 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3581e-01 | t=29.4s
|
| 305 |
+
[step 26600] step= 26600 | total=1.8115 | leaf=1.8115 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.7155e-01 | t=29.4s
|
| 306 |
+
[step 26700] step= 26700 | total=1.6873 | leaf=1.6873 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.7521e-01 | t=29.4s
|
| 307 |
+
[step 26800] step= 26800 | total=1.7895 | leaf=1.7895 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.4502e-01 | t=29.4s
|
| 308 |
+
[step 26900] step= 26900 | total=1.6810 | leaf=1.6810 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.2852e-01 | t=29.4s
|
| 309 |
+
[step 27000] [GRAD STATS] total_norm=1.4420e-01 max_single=5.6350e-02 (vocab_embed.embedding)
|
| 310 |
+
[step 27000] step= 27000 | total=1.7762 | leaf=1.7762 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.4420e-01 | t=29.4s
|
| 311 |
+
[step 27100] step= 27100 | total=1.7902 | leaf=1.7902 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.4661e-01 | t=29.4s
|
| 312 |
+
[step 27200] step= 27200 | total=1.8323 | leaf=1.8323 | ancestor=0.0000 | lr=4.99e-04 | gnorm=2.2128e-01 | t=29.4s
|
| 313 |
+
[step 27300] step= 27300 | total=1.7060 | leaf=1.7060 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3079e-01 | t=29.4s
|
| 314 |
+
[step 27400] step= 27400 | total=1.7173 | leaf=1.7173 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3965e-01 | t=29.4s
|
| 315 |
+
[step 27500] step= 27500 | total=1.8239 | leaf=1.8239 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.6139e-01 | t=29.4s
|
| 316 |
+
[step 27600] step= 27600 | total=1.7434 | leaf=1.7434 | ancestor=0.0000 | lr=4.99e-04 | gnorm=2.9336e-01 | t=29.4s
|
| 317 |
+
[step 27700] step= 27700 | total=1.7509 | leaf=1.7509 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.9777e-01 | t=29.4s
|
| 318 |
+
[step 27800] step= 27800 | total=1.8047 | leaf=1.8047 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.2520e-01 | t=29.4s
|
| 319 |
+
[step 27900] step= 27900 | total=1.6861 | leaf=1.6861 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.3282e-01 | t=29.4s
|
| 320 |
+
[step 28000] [GRAD STATS] total_norm=1.7006e-01 max_single=1.0530e-01 (cond_bias)
|
| 321 |
+
[step 28000] step= 28000 | total=1.7588 | leaf=1.7588 | ancestor=0.0000 | lr=4.99e-04 | gnorm=1.7006e-01 | t=29.4s
|
| 322 |
+
[step 28100] step= 28100 | total=1.7983 | leaf=1.7983 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.7704e-01 | t=29.4s
|
| 323 |
+
[step 28200] step= 28200 | total=1.8230 | leaf=1.8230 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.6060e-01 | t=29.4s
|
| 324 |
+
[step 28300] step= 28300 | total=1.7152 | leaf=1.7152 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.3323e-01 | t=29.4s
|
| 325 |
+
[step 28400] step= 28400 | total=1.7802 | leaf=1.7802 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.3823e-01 | t=29.4s
|
| 326 |
+
[step 28500] step= 28500 | total=1.6682 | leaf=1.6682 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.1058e-01 | t=29.4s
|
| 327 |
+
[step 28600] step= 28600 | total=1.8420 | leaf=1.8420 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.3106e-01 | t=29.4s
|
| 328 |
+
[step 28700] step= 28700 | total=1.8225 | leaf=1.8225 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.2027e-01 | t=29.4s
|
| 329 |
+
[step 28800] step= 28800 | total=1.8672 | leaf=1.8672 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.3634e-01 | t=29.4s
|
| 330 |
+
[step 28900] step= 28900 | total=1.7515 | leaf=1.7515 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.3122e-01 | t=29.4s
|
| 331 |
+
[step 29000] [GRAD STATS] total_norm=2.3960e-01 max_single=9.9754e-02 (cond_bias)
|
| 332 |
+
[step 29000] step= 29000 | total=1.6943 | leaf=1.6943 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.3960e-01 | t=29.4s
|
| 333 |
+
[step 29100] step= 29100 | total=1.5570 | leaf=1.5570 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.3853e-01 | t=29.4s
|
| 334 |
+
[step 29200] step= 29200 | total=1.6013 | leaf=1.6013 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.3520e-01 | t=29.4s
|
| 335 |
+
[step 29300] step= 29300 | total=1.7603 | leaf=1.7603 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.2788e-01 | t=29.4s
|
| 336 |
+
[step 29400] step= 29400 | total=1.8308 | leaf=1.8308 | ancestor=0.0000 | lr=4.98e-04 | gnorm=3.2146e-01 | t=29.4s
|
| 337 |
+
[step 29500] step= 29500 | total=1.8534 | leaf=1.8534 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.6808e-01 | t=29.4s
|
| 338 |
+
[step 29600] step= 29600 | total=1.7410 | leaf=1.7410 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.9966e-01 | t=29.4s
|
| 339 |
+
[step 29700] step= 29700 | total=1.6687 | leaf=1.6687 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.9493e-01 | t=29.4s
|
| 340 |
+
[step 29800] step= 29800 | total=1.8379 | leaf=1.8379 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.3448e-01 | t=29.4s
|
| 341 |
+
[step 29900] step= 29900 | total=1.6964 | leaf=1.6964 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.2132e-01 | t=29.4s
|
| 342 |
+
[step 30000] [GRAD STATS] total_norm=1.4353e-01 max_single=5.3977e-02 (vocab_embed.embedding)
|
| 343 |
+
[step 30000] step= 30000 | total=1.7719 | leaf=1.7719 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.4353e-01 | t=29.4s
|
| 344 |
+
VAL | loss_leaf=1.7455 | loss_ancestor=0.0000 | loss_total=1.7455
|
| 345 |
+
Saved checkpoint: outputs/sad_new/ckpt_30000.pt
|
| 346 |
+
[step 30100] step= 30100 | total=1.6532 | leaf=1.6532 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.4543e-01 | t=45.8s
|
| 347 |
+
[step 30200] step= 30200 | total=1.6497 | leaf=1.6497 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.7207e-01 | t=29.4s
|
| 348 |
+
[step 30300] step= 30300 | total=1.6794 | leaf=1.6794 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.9024e-01 | t=29.4s
|
| 349 |
+
[step 30400] step= 30400 | total=1.7692 | leaf=1.7692 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.0267e-01 | t=29.4s
|
| 350 |
+
[step 30500] step= 30500 | total=1.6996 | leaf=1.6996 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.2209e-01 | t=29.4s
|
| 351 |
+
[step 30600] step= 30600 | total=1.7415 | leaf=1.7415 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.3575e-01 | t=29.4s
|
| 352 |
+
[step 30700] step= 30700 | total=1.7863 | leaf=1.7863 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.0077e-01 | t=29.4s
|
| 353 |
+
[step 30800] step= 30800 | total=1.8307 | leaf=1.8307 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.6796e-01 | t=29.4s
|
| 354 |
+
[step 30900] step= 30900 | total=1.6859 | leaf=1.6859 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.2014e-01 | t=29.4s
|
| 355 |
+
[step 31000] [GRAD STATS] total_norm=1.5857e-01 max_single=6.0599e-02 (blocks.0.adaLN_modulation.bias)
|
| 356 |
+
[step 31000] step= 31000 | total=1.6019 | leaf=1.6019 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.5857e-01 | t=29.6s
|
| 357 |
+
[step 31100] step= 31100 | total=1.7577 | leaf=1.7577 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.6331e-01 | t=29.4s
|
| 358 |
+
[step 31200] step= 31200 | total=1.7680 | leaf=1.7680 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.7157e-01 | t=29.4s
|
| 359 |
+
[step 31300] step= 31300 | total=1.6763 | leaf=1.6763 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.4515e-01 | t=29.4s
|
| 360 |
+
[step 31400] step= 31400 | total=1.7651 | leaf=1.7651 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.4498e-01 | t=29.4s
|
| 361 |
+
[step 31500] step= 31500 | total=1.7459 | leaf=1.7459 | ancestor=0.0000 | lr=4.98e-04 | gnorm=3.0375e-01 | t=29.4s
|
| 362 |
+
[step 31600] step= 31600 | total=1.7046 | leaf=1.7046 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.5445e-01 | t=29.4s
|
| 363 |
+
[step 31700] step= 31700 | total=1.6835 | leaf=1.6835 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.4439e-01 | t=29.4s
|
| 364 |
+
[step 31800] step= 31800 | total=1.7753 | leaf=1.7753 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.2827e-01 | t=29.4s
|
| 365 |
+
[step 31900] step= 31900 | total=1.6322 | leaf=1.6322 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.3179e-01 | t=29.4s
|
| 366 |
+
[step 32000] [GRAD STATS] total_norm=2.2195e-01 max_single=1.5349e-01 (cond_bias)
|
| 367 |
+
[step 32000] step= 32000 | total=1.7601 | leaf=1.7601 | ancestor=0.0000 | lr=4.98e-04 | gnorm=2.2195e-01 | t=29.4s
|
| 368 |
+
[step 32100] step= 32100 | total=1.7491 | leaf=1.7491 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.7777e-01 | t=29.4s
|
| 369 |
+
[step 32200] step= 32200 | total=1.6696 | leaf=1.6696 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.1925e-01 | t=29.4s
|
| 370 |
+
[step 32300] step= 32300 | total=1.7295 | leaf=1.7295 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.9644e-01 | t=29.5s
|
| 371 |
+
[step 32400] step= 32400 | total=1.7698 | leaf=1.7698 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.6392e-01 | t=29.4s
|
| 372 |
+
[step 32500] step= 32500 | total=1.7791 | leaf=1.7791 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.2715e-01 | t=29.4s
|
| 373 |
+
[step 32600] step= 32600 | total=1.6561 | leaf=1.6561 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.5283e-01 | t=29.4s
|
| 374 |
+
[step 32700] step= 32700 | total=1.6476 | leaf=1.6476 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.5287e-01 | t=29.4s
|
| 375 |
+
[step 32800] step= 32800 | total=1.7665 | leaf=1.7665 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.5072e-01 | t=29.4s
|
| 376 |
+
[step 32900] step= 32900 | total=1.7809 | leaf=1.7809 | ancestor=0.0000 | lr=4.98e-04 | gnorm=3.2810e-01 | t=29.4s
|
| 377 |
+
[step 33000] [GRAD STATS] total_norm=1.7771e-01 max_single=1.1652e-01 (cond_bias)
|
| 378 |
+
[step 33000] step= 33000 | total=1.6729 | leaf=1.6729 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.7771e-01 | t=29.4s
|
| 379 |
+
[step 33100] step= 33100 | total=1.6401 | leaf=1.6401 | ancestor=0.0000 | lr=4.98e-04 | gnorm=1.5888e-01 | t=29.4s
|
| 380 |
+
[step 33200] step= 33200 | total=1.7991 | leaf=1.7991 | ancestor=0.0000 | lr=4.98e-04 | gnorm=3.4491e-01 | t=29.4s
|
| 381 |
+
[step 33300] step= 33300 | total=1.8100 | leaf=1.8100 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.6395e-01 | t=29.4s
|
| 382 |
+
[step 33400] step= 33400 | total=1.7759 | leaf=1.7759 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.8759e-01 | t=29.4s
|
| 383 |
+
[step 33500] step= 33500 | total=1.6095 | leaf=1.6095 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.9043e-01 | t=29.4s
|
| 384 |
+
[step 33600] step= 33600 | total=1.8240 | leaf=1.8240 | ancestor=0.0000 | lr=4.97e-04 | gnorm=3.4383e-01 | t=29.4s
|
| 385 |
+
[step 33700] step= 33700 | total=1.7394 | leaf=1.7394 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.7620e-01 | t=29.4s
|
| 386 |
+
[step 33800] step= 33800 | total=1.7843 | leaf=1.7843 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.2773e-01 | t=29.4s
|
| 387 |
+
[step 33900] step= 33900 | total=1.6842 | leaf=1.6842 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.5116e-01 | t=29.4s
|
| 388 |
+
[step 34000] [GRAD STATS] total_norm=1.6858e-01 max_single=7.7298e-02 (blocks.0.adaLN_modulation.bias)
|
| 389 |
+
[step 34000] step= 34000 | total=1.7918 | leaf=1.7918 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.6858e-01 | t=29.4s
|
| 390 |
+
[step 34100] step= 34100 | total=1.8616 | leaf=1.8616 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.9853e-01 | t=29.4s
|
| 391 |
+
[step 34200] step= 34200 | total=1.7831 | leaf=1.7831 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.5796e-01 | t=29.4s
|
| 392 |
+
[step 34300] step= 34300 | total=2.2362 | leaf=2.2362 | ancestor=0.0000 | lr=4.97e-04 | gnorm=4.5933e+00 | t=29.4s
|
| 393 |
+
[step 34400] step= 34400 | total=1.6765 | leaf=1.6765 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.7681e-01 | t=29.4s
|
| 394 |
+
[step 34500] step= 34500 | total=1.6324 | leaf=1.6324 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.4854e-01 | t=29.4s
|
| 395 |
+
[step 34600] step= 34600 | total=1.7645 | leaf=1.7645 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.4647e-01 | t=29.4s
|
| 396 |
+
[step 34700] step= 34700 | total=1.7385 | leaf=1.7385 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.4540e-01 | t=29.4s
|
| 397 |
+
[step 34800] step= 34800 | total=1.6823 | leaf=1.6823 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.5930e-01 | t=29.4s
|
| 398 |
+
[step 34900] step= 34900 | total=1.7517 | leaf=1.7517 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.7804e-01 | t=29.4s
|
| 399 |
+
[step 35000] [GRAD STATS] total_norm=1.5000e-01 max_single=5.8912e-02 (cond_bias)
|
| 400 |
+
[step 35000] step= 35000 | total=1.7545 | leaf=1.7545 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.5000e-01 | t=29.4s
|
| 401 |
+
VAL | loss_leaf=1.7046 | loss_ancestor=0.0000 | loss_total=1.7046
|
| 402 |
+
[step 35100] step= 35100 | total=1.8018 | leaf=1.8018 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.1020e-01 | t=35.1s
|
| 403 |
+
[step 35200] step= 35200 | total=1.8045 | leaf=1.8045 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.0198e-01 | t=29.4s
|
| 404 |
+
[step 35300] step= 35300 | total=1.6831 | leaf=1.6831 | ancestor=0.0000 | lr=4.97e-04 | gnorm=4.2396e-01 | t=29.4s
|
| 405 |
+
[step 35400] step= 35400 | total=1.6896 | leaf=1.6896 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.8627e-01 | t=29.4s
|
| 406 |
+
[step 35500] step= 35500 | total=1.7982 | leaf=1.7982 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.9962e-01 | t=29.4s
|
| 407 |
+
[step 35600] step= 35600 | total=1.9271 | leaf=1.9271 | ancestor=0.0000 | lr=4.97e-04 | gnorm=4.0882e-01 | t=29.4s
|
| 408 |
+
[step 35700] step= 35700 | total=1.6803 | leaf=1.6803 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.4898e-01 | t=29.4s
|
| 409 |
+
[step 35800] step= 35800 | total=1.6754 | leaf=1.6754 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.4050e-01 | t=29.4s
|
| 410 |
+
[step 35900] step= 35900 | total=1.6432 | leaf=1.6432 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.9168e-01 | t=29.4s
|
| 411 |
+
[step 36000] [GRAD STATS] total_norm=1.5338e-01 max_single=5.6143e-02 (vocab_embed.embedding)
|
| 412 |
+
[step 36000] step= 36000 | total=1.7665 | leaf=1.7665 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.5338e-01 | t=29.4s
|
| 413 |
+
[step 36100] step= 36100 | total=1.7873 | leaf=1.7873 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.0176e-01 | t=29.4s
|
| 414 |
+
[step 36200] step= 36200 | total=1.5751 | leaf=1.5751 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.8792e-01 | t=29.4s
|
| 415 |
+
[step 36300] step= 36300 | total=1.6410 | leaf=1.6410 | ancestor=0.0000 | lr=4.97e-04 | gnorm=3.0570e-01 | t=29.4s
|
| 416 |
+
[step 36400] step= 36400 | total=1.7803 | leaf=1.7803 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.7217e-01 | t=29.4s
|
| 417 |
+
[step 36500] step= 36500 | total=1.6553 | leaf=1.6553 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.4971e-01 | t=29.4s
|
| 418 |
+
[step 36600] step= 36600 | total=1.7430 | leaf=1.7430 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.1271e-01 | t=29.4s
|
| 419 |
+
[step 36700] step= 36700 | total=1.8408 | leaf=1.8408 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.4074e-01 | t=29.4s
|
| 420 |
+
[step 36800] step= 36800 | total=1.8226 | leaf=1.8226 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.6710e-01 | t=29.4s
|
| 421 |
+
[step 36900] step= 36900 | total=1.6256 | leaf=1.6256 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.7410e-01 | t=29.4s
|
| 422 |
+
[step 37000] [GRAD STATS] total_norm=3.6073e-01 max_single=2.0038e-01 (blocks.0.adaLN_modulation.bias)
|
| 423 |
+
[step 37000] step= 37000 | total=1.7968 | leaf=1.7968 | ancestor=0.0000 | lr=4.97e-04 | gnorm=3.6073e-01 | t=29.4s
|
| 424 |
+
[step 37100] step= 37100 | total=1.6708 | leaf=1.6708 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.9020e-01 | t=29.4s
|
| 425 |
+
[step 37200] step= 37200 | total=1.5711 | leaf=1.5711 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.8042e-01 | t=29.4s
|
| 426 |
+
[step 37300] step= 37300 | total=1.8151 | leaf=1.8151 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.1220e-01 | t=29.4s
|
| 427 |
+
[step 37400] step= 37400 | total=1.6855 | leaf=1.6855 | ancestor=0.0000 | lr=4.97e-04 | gnorm=2.0382e-01 | t=29.4s
|
| 428 |
+
[step 37500] step= 37500 | total=1.8442 | leaf=1.8442 | ancestor=0.0000 | lr=4.97e-04 | gnorm=1.6047e-01 | t=29.4s
|
| 429 |
+
[step 37600] step= 37600 | total=1.6276 | leaf=1.6276 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.1306e-01 | t=29.4s
|
| 430 |
+
[step 37700] step= 37700 | total=1.6026 | leaf=1.6026 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.7100e-01 | t=29.4s
|
| 431 |
+
[step 37800] step= 37800 | total=1.7725 | leaf=1.7725 | ancestor=0.0000 | lr=4.96e-04 | gnorm=3.5153e-01 | t=29.4s
|
| 432 |
+
[step 37900] step= 37900 | total=1.6208 | leaf=1.6208 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.6113e-01 | t=29.4s
|
| 433 |
+
[step 38000] [GRAD STATS] total_norm=1.3546e-01 max_single=4.9673e-02 (vocab_embed.embedding)
|
| 434 |
+
[step 38000] step= 38000 | total=1.6613 | leaf=1.6613 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.3546e-01 | t=29.4s
|
| 435 |
+
[step 38100] step= 38100 | total=1.8199 | leaf=1.8199 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.8547e-01 | t=29.4s
|
| 436 |
+
[step 38200] step= 38200 | total=1.8111 | leaf=1.8111 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.4278e-01 | t=29.4s
|
| 437 |
+
[step 38300] step= 38300 | total=1.6863 | leaf=1.6863 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.5828e-01 | t=29.4s
|
| 438 |
+
[step 38400] step= 38400 | total=1.6799 | leaf=1.6799 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.8385e-01 | t=29.4s
|
| 439 |
+
[step 38500] step= 38500 | total=1.7155 | leaf=1.7155 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.3163e-01 | t=29.4s
|
| 440 |
+
[step 38600] step= 38600 | total=1.6822 | leaf=1.6822 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.1225e-01 | t=29.4s
|
| 441 |
+
[step 38700] step= 38700 | total=1.6925 | leaf=1.6925 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.6574e-01 | t=29.4s
|
| 442 |
+
[step 38800] step= 38800 | total=1.6616 | leaf=1.6616 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.5778e-01 | t=29.4s
|
| 443 |
+
[step 38900] step= 38900 | total=1.7293 | leaf=1.7293 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.1698e-01 | t=29.4s
|
| 444 |
+
[step 39000] [GRAD STATS] total_norm=1.9727e-01 max_single=1.2420e-01 (cond_bias)
|
| 445 |
+
[step 39000] step= 39000 | total=1.7498 | leaf=1.7498 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.9727e-01 | t=29.4s
|
| 446 |
+
[step 39100] step= 39100 | total=1.7289 | leaf=1.7289 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.8079e-01 | t=29.4s
|
| 447 |
+
[step 39200] step= 39200 | total=1.7529 | leaf=1.7529 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.5390e-01 | t=29.4s
|
| 448 |
+
[step 39300] step= 39300 | total=1.7814 | leaf=1.7814 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.5010e-01 | t=29.4s
|
| 449 |
+
[step 39400] step= 39400 | total=1.6924 | leaf=1.6924 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.7328e-01 | t=29.4s
|
| 450 |
+
[step 39500] step= 39500 | total=1.8035 | leaf=1.8035 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.0153e-01 | t=29.4s
|
| 451 |
+
[step 39600] step= 39600 | total=1.7431 | leaf=1.7431 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.8831e-01 | t=29.4s
|
| 452 |
+
[step 39700] step= 39700 | total=1.7788 | leaf=1.7788 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.2430e-01 | t=29.4s
|
| 453 |
+
[step 39800] step= 39800 | total=1.6611 | leaf=1.6611 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.8210e-01 | t=29.4s
|
| 454 |
+
[step 39900] step= 39900 | total=1.7018 | leaf=1.7018 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.7840e-01 | t=29.4s
|
| 455 |
+
[step 40000] [GRAD STATS] total_norm=4.5227e-01 max_single=2.4705e-01 (segment_embed.weight)
|
| 456 |
+
[step 40000] step= 40000 | total=1.8196 | leaf=1.8196 | ancestor=0.0000 | lr=4.96e-04 | gnorm=4.5227e-01 | t=29.4s
|
| 457 |
+
VAL | loss_leaf=1.7304 | loss_ancestor=0.0000 | loss_total=1.7304
|
| 458 |
+
Saved checkpoint: outputs/sad_new/ckpt_40000.pt
|
| 459 |
+
[step 40100] step= 40100 | total=1.8208 | leaf=1.8208 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.3948e-01 | t=45.7s
|
| 460 |
+
[step 40200] step= 40200 | total=1.7107 | leaf=1.7107 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.4943e-01 | t=29.4s
|
| 461 |
+
[step 40300] step= 40300 | total=1.7511 | leaf=1.7511 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.8183e-01 | t=29.4s
|
| 462 |
+
[step 40400] step= 40400 | total=1.6942 | leaf=1.6942 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.6130e-01 | t=29.4s
|
| 463 |
+
[step 40500] step= 40500 | total=1.7786 | leaf=1.7786 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.4236e-01 | t=29.4s
|
| 464 |
+
[step 40600] step= 40600 | total=1.6020 | leaf=1.6020 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.5861e-01 | t=29.4s
|
| 465 |
+
[step 40700] step= 40700 | total=1.7223 | leaf=1.7223 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.5458e-01 | t=29.4s
|
| 466 |
+
[step 40800] step= 40800 | total=1.6824 | leaf=1.6824 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.6628e-01 | t=29.4s
|
| 467 |
+
[step 40900] step= 40900 | total=1.7330 | leaf=1.7330 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.7690e-01 | t=29.4s
|
| 468 |
+
[step 41000] [GRAD STATS] total_norm=2.0968e-01 max_single=1.0475e-01 (cond_bias)
|
| 469 |
+
[step 41000] step= 41000 | total=1.6463 | leaf=1.6463 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.0968e-01 | t=29.4s
|
| 470 |
+
[step 41100] step= 41100 | total=1.5640 | leaf=1.5640 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.5948e-01 | t=29.4s
|
| 471 |
+
[step 41200] step= 41200 | total=1.6296 | leaf=1.6296 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.9248e-01 | t=29.4s
|
| 472 |
+
[step 41300] step= 41300 | total=1.7148 | leaf=1.7148 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.8573e-01 | t=29.4s
|
| 473 |
+
[step 41400] step= 41400 | total=1.7542 | leaf=1.7542 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.4527e-01 | t=29.4s
|
| 474 |
+
[step 41500] step= 41500 | total=1.7241 | leaf=1.7241 | ancestor=0.0000 | lr=4.95e-04 | gnorm=3.9497e-01 | t=29.4s
|
| 475 |
+
[step 41600] step= 41600 | total=1.7461 | leaf=1.7461 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.9268e-01 | t=29.4s
|
| 476 |
+
[step 41700] step= 41700 | total=1.6738 | leaf=1.6738 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.5610e-01 | t=29.4s
|
| 477 |
+
[step 41800] step= 41800 | total=1.7547 | leaf=1.7547 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.5770e-01 | t=29.4s
|
| 478 |
+
[step 41900] step= 41900 | total=1.7754 | leaf=1.7754 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.4531e-01 | t=29.4s
|
| 479 |
+
[step 42000] [GRAD STATS] total_norm=1.6283e-01 max_single=6.0241e-02 (blocks.2.adaLN_modulation.bias)
|
| 480 |
+
[step 42000] step= 42000 | total=1.7217 | leaf=1.7217 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.6283e-01 | t=29.4s
|
| 481 |
+
File "/home/sunhc/diffusion/sad/scripts/train.py", line 716, in <module>
|
| 482 |
+
main()
|
| 483 |
+
File "/home/sunhc/diffusion/sad/scripts/train.py", line 614, in main
|
| 484 |
+
loss.backward()
|
| 485 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/_tensor.py", line 626, in backward
|
| 486 |
+
torch.autograd.backward(
|
| 487 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 488 |
+
_engine_run_backward(
|
| 489 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/graph.py", line 823, in _engine_run_backward
|
| 490 |
+
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 491 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 492 |
+
KeyboardInterrupt
|
| 493 |
+
[rank0]: Traceback (most recent call last):
|
| 494 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train.py", line 716, in <module>
|
| 495 |
+
[rank0]: main()
|
| 496 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train.py", line 614, in main
|
| 497 |
+
[rank0]: loss.backward()
|
| 498 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/_tensor.py", line 626, in backward
|
| 499 |
+
[rank0]: torch.autograd.backward(
|
| 500 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/__init__.py", line 347, in backward
|
| 501 |
+
[rank0]: _engine_run_backward(
|
| 502 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/autograd/graph.py", line 823, in _engine_run_backward
|
| 503 |
+
[rank0]: return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
|
| 504 |
+
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 505 |
+
[rank0]: KeyboardInterrupt
|
wandb/run-20260423_060125-7vxzycy6/logs/debug-core.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T06:01:25.378222109Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpssb7b3y8/port-1026041.txt","pid":1026041,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T06:01:25.378605431Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1026041}
|
| 3 |
+
{"time":"2026-04-23T06:01:25.37860065Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1026041-1032032-3008527617/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T06:01:25.561387485Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T06:01:25.56883891Z","level":"INFO","msg":"handleInformInit: received","streamId":"7vxzycy6","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T06:01:26.412833877Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"7vxzycy6","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T06:01:32.334471613Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"eub1ww2geh7p"}
|
| 8 |
+
{"time":"2026-04-23T09:29:51.153031335Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-23T09:29:51.153088839Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-04-23T09:29:51.153145543Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-23T09:29:51.153102233Z","level":"INFO","msg":"server is shutting down"}
|
| 12 |
+
{"time":"2026-04-23T09:29:51.153204847Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1026041-1032032-3008527617/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-23T09:29:52.525704687Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
wandb/run-20260423_060125-7vxzycy6/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/run-20260423_060125-7vxzycy6/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 06:01:25,344 INFO MainThread:1026041 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-23 06:01:25,344 INFO MainThread:1026041 [wandb_setup.py:_flush():81] Configure stats pid to 1026041
|
| 3 |
+
2026-04-23 06:01:25,344 INFO MainThread:1026041 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 06:01:25,344 INFO MainThread:1026041 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260423_060125-7vxzycy6/logs/debug.log
|
| 5 |
+
2026-04-23 06:01:25,344 INFO MainThread:1026041 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260423_060125-7vxzycy6/logs/debug-internal.log
|
| 6 |
+
2026-04-23 06:01:25,344 INFO MainThread:1026041 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-23 06:01:25,344 INFO MainThread:1026041 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 8, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top3_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0005, 'lr_min': 5e-05, 'warmup_steps': 10000, 'weight_decay': 0.02, 'grad_clip': 1.0, 'adam_betas': [0.9, 0.99], 'adam_eps': 1e-09, 'dtype': 'bf16', 'compile': 'default', 'curriculum_t_max_init': 0.3, 'curriculum_steps': 20000, 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_new_owt', 'save_dir': 'outputs/sad_new'}, 'debug': {'debug_mode': True, 'raise_on_nan': False, 'check_inputs': True, 'check_batch_every': 100, 'check_outputs': True, 'check_outputs_every': 100, 'check_loss': True, 'check_grads': True, 'check_params': True, 'check_params_every': 1000, 'use_hooks': False, 'save_on_nan': True, 'print_stats_every': 1000, 'grad_norm_warn_threshold': 100000.0}, '_wandb': {}}
|
| 9 |
+
2026-04-23 06:01:25,344 INFO MainThread:1026041 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-23 06:01:25,561 INFO MainThread:1026041 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-23 06:01:25,566 INFO MainThread:1026041 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-23 06:01:25,567 INFO MainThread:1026041 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-23 06:01:25,571 INFO MainThread:1026041 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 06:01:27,200 INFO MainThread:1026041 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-23 06:01:27,321 INFO MainThread:1026041 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-23 06:01:27,321 INFO MainThread:1026041 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-23 06:01:27,322 INFO MainThread:1026041 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-23 06:01:27,322 INFO MainThread:1026041 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-23 06:01:27,326 INFO MainThread:1026041 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-23 09:29:51,153 INFO wandb-AsyncioManager-main:1026041 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-23 09:29:51,153 INFO wandb-AsyncioManager-main:1026041 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260423_093030-7hhwxxjb/files/output.log
ADDED
|
@@ -0,0 +1,517 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
full-denoise training: 8%|▊ | 40001/500000 [00:00<?, ?it/s][rank0]:W0423 09:30:33.637000 1259107 site-packages/torch/_logging/_internal.py:1089] [0/0] Profiler function <class 'torch.autograd.profiler.record_function'> will be ignored
|
| 2 |
+
full-denoise training: 9%|▊ | 43000/500000 [15:04<37:18:44, 3.40it/s, leaf=1.7878, lr=4.9e-04]
|
| 3 |
+
[step 40100] step= 40100 | total=1.6928 | leaf=1.6928 | ancestor=0.0000 | lr=4.96e-04 | gnorm=3.0718e-01 | t=51.3s
|
| 4 |
+
[step 40200] step= 40200 | total=1.6292 | leaf=1.6292 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.5443e-01 | t=29.4s
|
| 5 |
+
[step 40300] step= 40300 | total=1.6007 | leaf=1.6007 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.7929e-01 | t=29.4s
|
| 6 |
+
[step 40400] step= 40400 | total=1.7107 | leaf=1.7107 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.6868e-01 | t=29.4s
|
| 7 |
+
[step 40500] step= 40500 | total=1.7258 | leaf=1.7258 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.7699e-01 | t=29.4s
|
| 8 |
+
[step 40600] step= 40600 | total=1.8098 | leaf=1.8098 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.5847e-01 | t=29.4s
|
| 9 |
+
[step 40700] step= 40700 | total=1.7410 | leaf=1.7410 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.0510e-01 | t=29.4s
|
| 10 |
+
[step 40800] step= 40800 | total=1.8032 | leaf=1.8032 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.8621e-01 | t=29.4s
|
| 11 |
+
[step 40900] step= 40900 | total=1.6939 | leaf=1.6939 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.7576e-01 | t=29.6s
|
| 12 |
+
[step 41000] [GRAD STATS] total_norm=2.0791e-01 max_single=1.2756e-01 (cond_bias)
|
| 13 |
+
[step 41000] step= 41000 | total=1.6895 | leaf=1.6895 | ancestor=0.0000 | lr=4.96e-04 | gnorm=2.0791e-01 | t=29.4s
|
| 14 |
+
[step 41100] step= 41100 | total=1.7408 | leaf=1.7408 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.8943e-01 | t=29.4s
|
| 15 |
+
[step 41200] step= 41200 | total=1.7493 | leaf=1.7493 | ancestor=0.0000 | lr=4.96e-04 | gnorm=1.9070e-01 | t=29.4s
|
| 16 |
+
[step 41300] step= 41300 | total=1.7765 | leaf=1.7765 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.3651e-01 | t=29.4s
|
| 17 |
+
[step 41400] step= 41400 | total=1.7377 | leaf=1.7377 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.8142e-01 | t=29.4s
|
| 18 |
+
[step 41500] step= 41500 | total=1.5782 | leaf=1.5782 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.9769e-01 | t=29.4s
|
| 19 |
+
[step 41600] step= 41600 | total=1.8093 | leaf=1.8093 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.7489e-01 | t=29.4s
|
| 20 |
+
[step 41700] step= 41700 | total=1.7964 | leaf=1.7964 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.8091e-01 | t=29.4s
|
| 21 |
+
[step 41800] step= 41800 | total=1.6742 | leaf=1.6742 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.5359e-01 | t=29.4s
|
| 22 |
+
[step 41900] step= 41900 | total=1.6001 | leaf=1.6001 | ancestor=0.0000 | lr=4.95e-04 | gnorm=3.1378e-01 | t=29.4s
|
| 23 |
+
[step 42000] [GRAD STATS] total_norm=1.6624e-01 max_single=8.3476e-02 (cond_bias)
|
| 24 |
+
[step 42000] step= 42000 | total=1.6769 | leaf=1.6769 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.6624e-01 | t=29.4s
|
| 25 |
+
[step 42100] step= 42100 | total=1.6673 | leaf=1.6673 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.9881e-01 | t=29.9s
|
| 26 |
+
[step 42200] step= 42200 | total=1.6910 | leaf=1.6910 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.9124e-01 | t=29.4s
|
| 27 |
+
[step 42300] step= 42300 | total=1.7357 | leaf=1.7357 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.0962e-01 | t=29.4s
|
| 28 |
+
[step 42400] step= 42400 | total=1.7835 | leaf=1.7835 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.9266e-01 | t=29.3s
|
| 29 |
+
[step 42500] step= 42500 | total=1.7918 | leaf=1.7918 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.8622e-01 | t=29.6s
|
| 30 |
+
[step 42600] step= 42600 | total=1.6254 | leaf=1.6254 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.5668e-01 | t=29.4s
|
| 31 |
+
[step 42700] step= 42700 | total=1.5691 | leaf=1.5691 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.0511e-01 | t=29.4s
|
| 32 |
+
[step 42800] step= 42800 | total=1.7210 | leaf=1.7210 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.5046e-01 | t=29.4s
|
| 33 |
+
[step 42900] step= 42900 | total=1.8635 | leaf=1.8635 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.8982e-01 | t=29.4s
|
| 34 |
+
[step 43000] [GRAD STATS] total_norm=1.8345e-01 max_single=9.1504e-02 (cond_bias)
|
| 35 |
+
[step 43000] step= 43000 | total=1.6308 | leaf=1.6308 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.8345e-01 | t=29.4s
|
| 36 |
+
[step 43100] step= 43100 | total=1.7119 | leaf=1.7119 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.6315e-01 | t=29.4s
|
| 37 |
+
[step 43200] step= 43200 | total=1.7441 | leaf=1.7441 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.7150e-01 | t=29.4s
|
| 38 |
+
[step 43300] step= 43300 | total=1.7557 | leaf=1.7557 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.4895e-01 | t=29.4s
|
| 39 |
+
[step 43400] step= 43400 | total=1.7510 | leaf=1.7510 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.6456e-01 | t=29.4s
|
| 40 |
+
[step 43500] step= 43500 | total=1.7341 | leaf=1.7341 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.8939e-01 | t=29.4s
|
| 41 |
+
[step 43600] step= 43600 | total=1.7543 | leaf=1.7543 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.0581e-01 | t=29.4s
|
| 42 |
+
[step 43700] step= 43700 | total=1.6518 | leaf=1.6518 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.1042e-01 | t=29.4s
|
| 43 |
+
[step 43800] step= 43800 | total=1.6562 | leaf=1.6562 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.2061e-01 | t=29.4s
|
| 44 |
+
[step 43900] step= 43900 | total=1.6244 | leaf=1.6244 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.6444e-01 | t=29.4s
|
| 45 |
+
[step 44000] [GRAD STATS] total_norm=1.6349e-01 max_single=5.9706e-02 (blocks.2.adaLN_modulation.bias)
|
| 46 |
+
[step 44000] step= 44000 | total=1.6969 | leaf=1.6969 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.6349e-01 | t=29.4s
|
| 47 |
+
[step 44100] step= 44100 | total=1.7241 | leaf=1.7241 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.6536e-01 | t=29.4s
|
| 48 |
+
[step 44200] step= 44200 | total=1.7482 | leaf=1.7482 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.8820e-01 | t=29.4s
|
| 49 |
+
[step 44300] step= 44300 | total=1.5458 | leaf=1.5458 | ancestor=0.0000 | lr=4.95e-04 | gnorm=2.0706e-01 | t=29.4s
|
| 50 |
+
[step 44400] step= 44400 | total=1.6939 | leaf=1.6939 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.8324e-01 | t=29.4s
|
| 51 |
+
[step 44500] step= 44500 | total=1.7272 | leaf=1.7272 | ancestor=0.0000 | lr=4.95e-04 | gnorm=1.5921e-01 | t=29.4s
|
| 52 |
+
[step 44600] step= 44600 | total=1.7809 | leaf=1.7809 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.7503e-01 | t=29.4s
|
| 53 |
+
[step 44700] step= 44700 | total=1.6666 | leaf=1.6666 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.2389e-01 | t=29.4s
|
| 54 |
+
[step 44800] step= 44800 | total=1.8081 | leaf=1.8081 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.7190e-01 | t=29.4s
|
| 55 |
+
[step 44900] step= 44900 | total=1.7100 | leaf=1.7100 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.5761e-01 | t=29.4s
|
| 56 |
+
[step 45000] [GRAD STATS] total_norm=1.6970e-01 max_single=6.2731e-02 (blocks.2.adaLN_modulation.bias)
|
| 57 |
+
[step 45000] step= 45000 | total=1.5619 | leaf=1.5619 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.6970e-01 | t=29.4s
|
| 58 |
+
torch._dynamo.utils.warn_once(msg)
|
| 59 |
+
VAL | loss_leaf=1.6978 | loss_ancestor=0.0000 | loss_total=1.6978
|
| 60 |
+
full-denoise training: 10%|▉ | 48000/500000 [39:49<36:49:57, 3.41it/s, leaf=1.6704, lr=4.9e-04]
|
| 61 |
+
[step 45100] step= 45100 | total=1.5542 | leaf=1.5542 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.8730e-01 | t=45.7s
|
| 62 |
+
[step 45200] step= 45200 | total=1.8165 | leaf=1.8165 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.1371e-01 | t=29.4s
|
| 63 |
+
[step 45300] step= 45300 | total=1.6492 | leaf=1.6492 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.9716e-01 | t=29.4s
|
| 64 |
+
[step 45400] step= 45400 | total=1.7136 | leaf=1.7136 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.0714e-01 | t=29.4s
|
| 65 |
+
[step 45500] step= 45500 | total=1.6793 | leaf=1.6793 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.8014e-01 | t=29.4s
|
| 66 |
+
[step 45600] step= 45600 | total=1.6946 | leaf=1.6946 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.3154e-01 | t=29.4s
|
| 67 |
+
[step 45700] step= 45700 | total=1.6918 | leaf=1.6918 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.1727e-01 | t=29.4s
|
| 68 |
+
[step 45800] step= 45800 | total=1.7335 | leaf=1.7335 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.7945e-01 | t=29.4s
|
| 69 |
+
[step 45900] step= 45900 | total=1.7229 | leaf=1.7229 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.5223e-01 | t=29.4s
|
| 70 |
+
[step 46000] [GRAD STATS] total_norm=1.6097e-01 max_single=5.5388e-02 (blocks.2.adaLN_modulation.bias)
|
| 71 |
+
[step 46000] step= 46000 | total=1.7267 | leaf=1.7267 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.6097e-01 | t=29.4s
|
| 72 |
+
[step 46100] step= 46100 | total=1.7639 | leaf=1.7639 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.7670e-01 | t=29.4s
|
| 73 |
+
[step 46200] step= 46200 | total=1.6617 | leaf=1.6617 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.4434e-01 | t=29.4s
|
| 74 |
+
[step 46300] step= 46300 | total=1.6840 | leaf=1.6840 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.7592e-01 | t=29.4s
|
| 75 |
+
[step 46400] step= 46400 | total=1.8280 | leaf=1.8280 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.6902e-01 | t=29.4s
|
| 76 |
+
[step 46500] step= 46500 | total=1.8349 | leaf=1.8349 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.6907e-01 | t=29.4s
|
| 77 |
+
[step 46600] step= 46600 | total=1.7713 | leaf=1.7713 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.0147e-01 | t=29.4s
|
| 78 |
+
[step 46700] step= 46700 | total=1.6143 | leaf=1.6143 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.8344e-01 | t=29.4s
|
| 79 |
+
[step 46800] step= 46800 | total=1.8207 | leaf=1.8207 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.6117e-01 | t=29.4s
|
| 80 |
+
[step 46900] step= 46900 | total=1.5709 | leaf=1.5709 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.1632e-01 | t=29.4s
|
| 81 |
+
[step 47000] [GRAD STATS] total_norm=2.0441e-01 max_single=7.2341e-02 (segment_embed.weight)
|
| 82 |
+
[step 47000] step= 47000 | total=1.8717 | leaf=1.8717 | ancestor=0.0000 | lr=4.94e-04 | gnorm=2.0441e-01 | t=29.4s
|
| 83 |
+
[step 47100] step= 47100 | total=1.6323 | leaf=1.6323 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.8542e-01 | t=29.4s
|
| 84 |
+
[step 47200] step= 47200 | total=1.5316 | leaf=1.5316 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.7542e-01 | t=29.4s
|
| 85 |
+
[step 47300] step= 47300 | total=1.7194 | leaf=1.7194 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.9247e-01 | t=29.4s
|
| 86 |
+
[step 47400] step= 47400 | total=1.5736 | leaf=1.5736 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.7751e-01 | t=29.4s
|
| 87 |
+
[step 47500] step= 47500 | total=1.7393 | leaf=1.7393 | ancestor=0.0000 | lr=4.94e-04 | gnorm=1.6041e-01 | t=29.4s
|
| 88 |
+
[step 47600] step= 47600 | total=1.7786 | leaf=1.7786 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.9520e-01 | t=29.4s
|
| 89 |
+
[step 47700] step= 47700 | total=1.6867 | leaf=1.6867 | ancestor=0.0000 | lr=4.93e-04 | gnorm=2.4536e-01 | t=29.4s
|
| 90 |
+
[step 47800] step= 47800 | total=1.6359 | leaf=1.6359 | ancestor=0.0000 | lr=4.93e-04 | gnorm=2.1003e-01 | t=29.4s
|
| 91 |
+
[step 47900] step= 47900 | total=1.5973 | leaf=1.5973 | ancestor=0.0000 | lr=4.93e-04 | gnorm=2.1028e-01 | t=29.4s
|
| 92 |
+
[step 48000] [GRAD STATS] total_norm=1.9236e-01 max_single=1.0991e-01 (cond_bias)
|
| 93 |
+
[step 48000] step= 48000 | total=1.7740 | leaf=1.7740 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.9236e-01 | t=29.4s
|
| 94 |
+
[step 48100] step= 48100 | total=1.7112 | leaf=1.7112 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.6037e-01 | t=29.4s
|
| 95 |
+
[step 48200] step= 48200 | total=1.8563 | leaf=1.8563 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.8536e-01 | t=29.4s
|
| 96 |
+
[step 48300] step= 48300 | total=1.7056 | leaf=1.7056 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.6913e-01 | t=29.4s
|
| 97 |
+
[step 48400] step= 48400 | total=1.6308 | leaf=1.6308 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.9111e-01 | t=29.4s
|
| 98 |
+
[step 48500] step= 48500 | total=1.7130 | leaf=1.7130 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.7033e-01 | t=29.4s
|
| 99 |
+
[step 48600] step= 48600 | total=1.6187 | leaf=1.6187 | ancestor=0.0000 | lr=4.93e-04 | gnorm=2.5319e-01 | t=29.3s
|
| 100 |
+
[step 48700] step= 48700 | total=1.6033 | leaf=1.6033 | ancestor=0.0000 | lr=4.93e-04 | gnorm=2.1662e-01 | t=29.4s
|
| 101 |
+
[step 48800] step= 48800 | total=1.7382 | leaf=1.7382 | ancestor=0.0000 | lr=4.93e-04 | gnorm=2.5475e-01 | t=29.4s
|
| 102 |
+
[step 48900] step= 48900 | total=1.7088 | leaf=1.7088 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.8435e-01 | t=29.4s
|
| 103 |
+
[step 49000] [GRAD STATS] total_norm=1.7120e-01 max_single=5.7557e-02 (blocks.2.adaLN_modulation.bias)
|
| 104 |
+
[step 49000] step= 49000 | total=1.6981 | leaf=1.6981 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.7120e-01 | t=29.4s
|
| 105 |
+
[step 49100] step= 49100 | total=1.7496 | leaf=1.7496 | ancestor=0.0000 | lr=4.93e-04 | gnorm=2.0434e-01 | t=29.4s
|
| 106 |
+
[step 49200] step= 49200 | total=1.7451 | leaf=1.7451 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.7269e-01 | t=29.4s
|
| 107 |
+
[step 49300] step= 49300 | total=1.6889 | leaf=1.6889 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.5636e-01 | t=29.4s
|
| 108 |
+
[step 49400] step= 49400 | total=1.7245 | leaf=1.7245 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.7189e-01 | t=29.4s
|
| 109 |
+
[step 49500] step= 49500 | total=1.7914 | leaf=1.7914 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.7489e-01 | t=29.4s
|
| 110 |
+
[step 49600] step= 49600 | total=1.5726 | leaf=1.5726 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.6649e-01 | t=29.4s
|
| 111 |
+
[step 49700] step= 49700 | total=1.5909 | leaf=1.5909 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.7456e-01 | t=29.4s
|
| 112 |
+
[step 49800] step= 49800 | total=1.7570 | leaf=1.7570 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.4906e-01 | t=29.4s
|
| 113 |
+
[step 49900] step= 49900 | total=1.7039 | leaf=1.7039 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.5593e-01 | t=29.4s
|
| 114 |
+
[step 50000] [GRAD STATS] total_norm=2.2859e-01 max_single=1.5799e-01 (cond_bias)
|
| 115 |
+
[step 50000] step= 50000 | total=1.6935 | leaf=1.6935 | ancestor=0.0000 | lr=4.93e-04 | gnorm=2.2859e-01 | t=29.4s
|
| 116 |
+
VAL | loss_leaf=1.6889 | loss_ancestor=0.0000 | loss_total=1.6889
|
| 117 |
+
Saved checkpoint: outputs/sad_new/ckpt_50000.pt
|
| 118 |
+
[step 50100] step= 50100 | total=1.7298 | leaf=1.7298 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.6608e-01 | t=45.3s
|
| 119 |
+
[step 50200] step= 50200 | total=1.7242 | leaf=1.7242 | ancestor=0.0000 | lr=4.93e-04 | gnorm=1.9634e-01 | t=29.4s
|
| 120 |
+
[step 50300] step= 50300 | total=1.7896 | leaf=1.7896 | ancestor=0.0000 | lr=4.93e-04 | gnorm=2.3528e-01 | t=29.4s
|
| 121 |
+
[step 50400] step= 50400 | total=1.7385 | leaf=1.7385 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.9873e-01 | t=29.4s
|
| 122 |
+
[step 50500] step= 50500 | total=1.7126 | leaf=1.7126 | ancestor=0.0000 | lr=4.92e-04 | gnorm=2.3082e-01 | t=29.4s
|
| 123 |
+
[step 50600] step= 50600 | total=1.7010 | leaf=1.7010 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.6088e-01 | t=29.4s
|
| 124 |
+
[step 50700] step= 50700 | total=1.7358 | leaf=1.7358 | ancestor=0.0000 | lr=4.92e-04 | gnorm=2.4073e-01 | t=29.4s
|
| 125 |
+
[step 50800] step= 50800 | total=1.5881 | leaf=1.5881 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.8175e-01 | t=29.4s
|
| 126 |
+
[step 50900] step= 50900 | total=1.6514 | leaf=1.6514 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.6515e-01 | t=29.4s
|
| 127 |
+
[step 51000] [GRAD STATS] total_norm=2.2401e-01 max_single=1.4830e-01 (cond_bias)
|
| 128 |
+
[step 51000] step= 51000 | total=1.6878 | leaf=1.6878 | ancestor=0.0000 | lr=4.92e-04 | gnorm=2.2401e-01 | t=29.4s
|
| 129 |
+
[step 51100] step= 51100 | total=1.6283 | leaf=1.6283 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.5191e-01 | t=29.4s
|
| 130 |
+
[step 51200] step= 51200 | total=1.7079 | leaf=1.7079 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.6447e-01 | t=29.4s
|
| 131 |
+
[step 51300] step= 51300 | total=1.6054 | leaf=1.6054 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.6221e-01 | t=29.4s
|
| 132 |
+
[step 51400] step= 51400 | total=1.6089 | leaf=1.6089 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.7069e-01 | t=29.4s
|
| 133 |
+
[step 51500] step= 51500 | total=1.7232 | leaf=1.7232 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.7665e-01 | t=29.4s
|
| 134 |
+
[step 51600] step= 51600 | total=1.6874 | leaf=1.6874 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.6278e-01 | t=29.4s
|
| 135 |
+
[step 51700] step= 51700 | total=1.7654 | leaf=1.7654 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.7613e-01 | t=29.4s
|
| 136 |
+
[step 51800] step= 51800 | total=1.7048 | leaf=1.7048 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.5769e-01 | t=29.4s
|
| 137 |
+
[step 51900] step= 51900 | total=1.7223 | leaf=1.7223 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.7379e-01 | t=29.4s
|
| 138 |
+
[step 52000] [GRAD STATS] total_norm=1.8562e-01 max_single=7.0660e-02 (cond_bias)
|
| 139 |
+
[step 52000] step= 52000 | total=1.7169 | leaf=1.7169 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.8562e-01 | t=29.4s
|
| 140 |
+
[step 52100] step= 52100 | total=1.7878 | leaf=1.7878 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.8376e-01 | t=29.4s
|
| 141 |
+
[step 52200] step= 52200 | total=1.7890 | leaf=1.7890 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.7763e-01 | t=29.4s
|
| 142 |
+
[step 52300] step= 52300 | total=1.7515 | leaf=1.7515 | ancestor=0.0000 | lr=4.92e-04 | gnorm=2.1571e-01 | t=29.4s
|
| 143 |
+
[step 52400] step= 52400 | total=1.6542 | leaf=1.6542 | ancestor=0.0000 | lr=4.92e-04 | gnorm=3.3246e-01 | t=29.4s
|
| 144 |
+
[step 52500] step= 52500 | total=1.6498 | leaf=1.6498 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.9306e-01 | t=29.4s
|
| 145 |
+
[step 52600] step= 52600 | total=1.7170 | leaf=1.7170 | ancestor=0.0000 | lr=4.92e-04 | gnorm=3.4824e-01 | t=29.4s
|
| 146 |
+
[step 52700] step= 52700 | total=1.7633 | leaf=1.7633 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.8215e-01 | t=29.4s
|
| 147 |
+
[step 52800] step= 52800 | total=1.7489 | leaf=1.7489 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.8889e-01 | t=29.4s
|
| 148 |
+
[step 52900] step= 52900 | total=1.6352 | leaf=1.6352 | ancestor=0.0000 | lr=4.92e-04 | gnorm=2.1285e-01 | t=29.4s
|
| 149 |
+
[step 53000] [GRAD STATS] total_norm=1.7937e-01 max_single=8.7930e-02 (cond_bias)
|
| 150 |
+
[step 53000] step= 53000 | total=1.7077 | leaf=1.7077 | ancestor=0.0000 | lr=4.92e-04 | gnorm=1.7937e-01 | t=29.4s
|
| 151 |
+
[step 53100] step= 53100 | total=1.6929 | leaf=1.6929 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.0328e-01 | t=29.4s
|
| 152 |
+
[step 53200] step= 53200 | total=1.7441 | leaf=1.7441 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.6949e-01 | t=29.4s
|
| 153 |
+
[step 53300] step= 53300 | total=1.7446 | leaf=1.7446 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.1652e-01 | t=29.4s
|
| 154 |
+
[step 53400] step= 53400 | total=1.6831 | leaf=1.6831 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.0921e-01 | t=29.4s
|
| 155 |
+
[step 53500] step= 53500 | total=1.6330 | leaf=1.6330 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.9953e-01 | t=29.4s
|
| 156 |
+
[step 53600] step= 53600 | total=1.6544 | leaf=1.6544 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.3755e-01 | t=29.4s
|
| 157 |
+
[step 53700] step= 53700 | total=1.6818 | leaf=1.6818 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.8323e-01 | t=29.4s
|
| 158 |
+
[step 53800] step= 53800 | total=1.6937 | leaf=1.6937 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.4717e-01 | t=29.4s
|
| 159 |
+
[step 53900] step= 53900 | total=1.6144 | leaf=1.6144 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.7751e-01 | t=29.4s
|
| 160 |
+
[step 54000] [GRAD STATS] total_norm=1.5498e-01 max_single=5.6497e-02 (blocks.2.adaLN_modulation.bias)
|
| 161 |
+
[step 54000] step= 54000 | total=1.8185 | leaf=1.8185 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.5498e-01 | t=29.4s
|
| 162 |
+
[step 54100] step= 54100 | total=1.6436 | leaf=1.6436 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.0058e-01 | t=29.4s
|
| 163 |
+
[step 54200] step= 54200 | total=1.6183 | leaf=1.6183 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.7998e-01 | t=29.4s
|
| 164 |
+
[step 54300] step= 54300 | total=1.6238 | leaf=1.6238 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.2020e-01 | t=29.4s
|
| 165 |
+
[step 54400] step= 54400 | total=1.6534 | leaf=1.6534 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.2622e-01 | t=29.4s
|
| 166 |
+
[step 54500] step= 54500 | total=1.5530 | leaf=1.5530 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.8370e-01 | t=29.4s
|
| 167 |
+
[step 54600] step= 54600 | total=1.5713 | leaf=1.5713 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.6862e-01 | t=29.4s
|
| 168 |
+
[step 54700] step= 54700 | total=1.6150 | leaf=1.6150 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.0426e-01 | t=29.4s
|
| 169 |
+
[step 54800] step= 54800 | total=1.7081 | leaf=1.7081 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.7338e-01 | t=29.4s
|
| 170 |
+
[step 54900] step= 54900 | total=1.7185 | leaf=1.7185 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.6771e-01 | t=29.4s
|
| 171 |
+
[step 55000] [GRAD STATS] total_norm=1.6733e-01 max_single=5.5365e-02 (blocks.2.adaLN_modulation.bias)
|
| 172 |
+
[step 55000] step= 55000 | total=1.6666 | leaf=1.6666 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.6733e-01 | t=29.4s
|
| 173 |
+
VAL | loss_leaf=1.6896 | loss_ancestor=0.0000 | loss_total=1.6896
|
| 174 |
+
[step 55100] step= 55100 | total=1.6776 | leaf=1.6776 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.8437e-01 | t=35.1s
|
| 175 |
+
[step 55200] step= 55200 | total=1.6602 | leaf=1.6602 | ancestor=0.0000 | lr=4.91e-04 | gnorm=2.4247e-01 | t=29.4s
|
| 176 |
+
[step 55300] step= 55300 | total=1.6098 | leaf=1.6098 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.7082e-01 | t=29.4s
|
| 177 |
+
[step 55400] step= 55400 | total=1.6209 | leaf=1.6209 | ancestor=0.0000 | lr=4.91e-04 | gnorm=1.9629e-01 | t=29.4s
|
| 178 |
+
[step 55500] step= 55500 | total=1.6444 | leaf=1.6444 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.5306e-01 | t=54.8s
|
| 179 |
+
[step 55600] step= 55600 | total=1.6754 | leaf=1.6754 | ancestor=0.0000 | lr=4.90e-04 | gnorm=2.3453e-01 | t=29.4s
|
| 180 |
+
[step 55700] step= 55700 | total=1.7091 | leaf=1.7091 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.5063e-01 | t=29.4s
|
| 181 |
+
[step 55800] step= 55800 | total=1.6931 | leaf=1.6931 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.6703e-01 | t=29.4s
|
| 182 |
+
[step 55900] step= 55900 | total=1.7867 | leaf=1.7867 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.7719e-01 | t=29.4s
|
| 183 |
+
[step 56000] [GRAD STATS] total_norm=1.6462e-01 max_single=5.6253e-02 (cond_bias)
|
| 184 |
+
[step 56000] step= 56000 | total=1.7874 | leaf=1.7874 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.6462e-01 | t=29.4s
|
| 185 |
+
[step 56100] step= 56100 | total=1.7111 | leaf=1.7111 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.8300e-01 | t=29.4s
|
| 186 |
+
[step 56200] step= 56200 | total=1.7537 | leaf=1.7537 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.9190e-01 | t=29.4s
|
| 187 |
+
[step 56300] step= 56300 | total=1.7468 | leaf=1.7468 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.5592e-01 | t=29.4s
|
| 188 |
+
[step 56400] step= 56400 | total=1.6177 | leaf=1.6177 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.5475e-01 | t=29.4s
|
| 189 |
+
[step 56500] step= 56500 | total=1.6739 | leaf=1.6739 | ancestor=0.0000 | lr=4.90e-04 | gnorm=2.0999e-01 | t=29.4s
|
| 190 |
+
[step 56600] step= 56600 | total=1.7049 | leaf=1.7049 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.8240e-01 | t=29.4s
|
| 191 |
+
[step 56700] step= 56700 | total=1.5238 | leaf=1.5238 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.7581e-01 | t=29.4s
|
| 192 |
+
[step 56800] step= 56800 | total=1.7553 | leaf=1.7553 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.6240e-01 | t=29.4s
|
| 193 |
+
[step 56900] step= 56900 | total=1.6218 | leaf=1.6218 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.6394e-01 | t=29.4s
|
| 194 |
+
[step 57000] [GRAD STATS] total_norm=1.4954e-01 max_single=5.0184e-02 (vocab_embed.embedding)
|
| 195 |
+
[step 57000] step= 57000 | total=1.7468 | leaf=1.7468 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.4954e-01 | t=29.4s
|
| 196 |
+
[step 57100] step= 57100 | total=1.6925 | leaf=1.6925 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.8742e-01 | t=29.4s
|
| 197 |
+
[step 57200] step= 57200 | total=1.7842 | leaf=1.7842 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.6259e-01 | t=29.4s
|
| 198 |
+
[step 57300] step= 57300 | total=1.6493 | leaf=1.6493 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.8790e-01 | t=29.4s
|
| 199 |
+
[step 57400] step= 57400 | total=1.6885 | leaf=1.6885 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.7125e-01 | t=29.4s
|
| 200 |
+
[step 57500] step= 57500 | total=1.7639 | leaf=1.7639 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.6511e-01 | t=29.4s
|
| 201 |
+
[step 57600] step= 57600 | total=1.6796 | leaf=1.6796 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.8792e-01 | t=29.4s
|
| 202 |
+
[step 57700] step= 57700 | total=1.7124 | leaf=1.7124 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.5465e-01 | t=29.4s
|
| 203 |
+
[step 57800] step= 57800 | total=1.5479 | leaf=1.5479 | ancestor=0.0000 | lr=4.90e-04 | gnorm=1.7641e-01 | t=29.4s
|
| 204 |
+
[step 57900] step= 57900 | total=1.7488 | leaf=1.7488 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.6747e-01 | t=29.4s
|
| 205 |
+
[step 58000] [GRAD STATS] total_norm=2.2801e-01 max_single=1.6390e-01 (cond_bias)
|
| 206 |
+
[step 58000] step= 58000 | total=1.6000 | leaf=1.6000 | ancestor=0.0000 | lr=4.89e-04 | gnorm=2.2801e-01 | t=29.4s
|
| 207 |
+
[step 58100] step= 58100 | total=1.6304 | leaf=1.6304 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.9355e-01 | t=29.4s
|
| 208 |
+
[step 58200] step= 58200 | total=1.6446 | leaf=1.6446 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.9298e-01 | t=29.4s
|
| 209 |
+
[step 58300] step= 58300 | total=1.6995 | leaf=1.6995 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.6343e-01 | t=29.4s
|
| 210 |
+
[step 58400] step= 58400 | total=1.7757 | leaf=1.7757 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.9531e-01 | t=29.4s
|
| 211 |
+
[step 58500] step= 58500 | total=1.7437 | leaf=1.7437 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.6818e-01 | t=29.4s
|
| 212 |
+
[step 58600] step= 58600 | total=1.5967 | leaf=1.5967 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.4834e-01 | t=29.4s
|
| 213 |
+
[step 58700] step= 58700 | total=1.7226 | leaf=1.7226 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.7216e-01 | t=29.4s
|
| 214 |
+
[step 58800] step= 58800 | total=1.7318 | leaf=1.7318 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.4310e-01 | t=29.4s
|
| 215 |
+
[step 58900] step= 58900 | total=1.6292 | leaf=1.6292 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.5437e-01 | t=29.4s
|
| 216 |
+
[step 59000] [GRAD STATS] total_norm=1.6162e-01 max_single=5.5519e-02 (vocab_embed.embedding)
|
| 217 |
+
[step 59000] step= 59000 | total=1.6816 | leaf=1.6816 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.6162e-01 | t=29.4s
|
| 218 |
+
[step 59100] step= 59100 | total=1.6786 | leaf=1.6786 | ancestor=0.0000 | lr=4.89e-04 | gnorm=2.0923e-01 | t=29.4s
|
| 219 |
+
[step 59200] step= 59200 | total=1.7447 | leaf=1.7447 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.6547e-01 | t=29.4s
|
| 220 |
+
[step 59300] step= 59300 | total=1.6707 | leaf=1.6707 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.6131e-01 | t=29.4s
|
| 221 |
+
[step 59400] step= 59400 | total=1.6316 | leaf=1.6316 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.9180e-01 | t=29.4s
|
| 222 |
+
[step 59500] step= 59500 | total=1.6692 | leaf=1.6692 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.8314e-01 | t=29.4s
|
| 223 |
+
[step 59600] step= 59600 | total=1.6116 | leaf=1.6116 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.6048e-01 | t=29.4s
|
| 224 |
+
[step 59700] step= 59700 | total=1.6260 | leaf=1.6260 | ancestor=0.0000 | lr=4.89e-04 | gnorm=2.3340e-01 | t=29.4s
|
| 225 |
+
[step 59800] step= 59800 | total=1.7498 | leaf=1.7498 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.6099e-01 | t=29.4s
|
| 226 |
+
[step 59900] step= 59900 | total=1.6918 | leaf=1.6918 | ancestor=0.0000 | lr=4.89e-04 | gnorm=2.7432e-01 | t=29.4s
|
| 227 |
+
[step 60000] [GRAD STATS] total_norm=1.6627e-01 max_single=7.2694e-02 (cond_bias)
|
| 228 |
+
[step 60000] step= 60000 | total=1.6931 | leaf=1.6931 | ancestor=0.0000 | lr=4.89e-04 | gnorm=1.6627e-01 | t=29.4s
|
| 229 |
+
VAL | loss_leaf=1.6595 | loss_ancestor=0.0000 | loss_total=1.6595
|
| 230 |
+
Saved checkpoint: outputs/sad_new/ckpt_60000.pt
|
| 231 |
+
[step 60100] step= 60100 | total=1.6211 | leaf=1.6211 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.8110e-01 | t=45.5s
|
| 232 |
+
[step 60200] step= 60200 | total=1.6007 | leaf=1.6007 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.6551e-01 | t=29.4s
|
| 233 |
+
[step 60300] step= 60300 | total=1.6579 | leaf=1.6579 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.4653e-01 | t=29.4s
|
| 234 |
+
[step 60400] step= 60400 | total=1.6469 | leaf=1.6469 | ancestor=0.0000 | lr=4.88e-04 | gnorm=2.1423e-01 | t=29.4s
|
| 235 |
+
[step 60500] step= 60500 | total=1.5703 | leaf=1.5703 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.7669e-01 | t=29.4s
|
| 236 |
+
[step 60600] step= 60600 | total=1.6035 | leaf=1.6035 | ancestor=0.0000 | lr=4.88e-04 | gnorm=2.5151e-01 | t=29.4s
|
| 237 |
+
[step 60700] step= 60700 | total=1.7415 | leaf=1.7415 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.4181e-01 | t=29.4s
|
| 238 |
+
[step 60800] step= 60800 | total=1.6376 | leaf=1.6376 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.4131e-01 | t=29.4s
|
| 239 |
+
[step 60900] step= 60900 | total=1.7361 | leaf=1.7361 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.6379e-01 | t=29.4s
|
| 240 |
+
[step 61000] [GRAD STATS] total_norm=1.5746e-01 max_single=5.5580e-02 (vocab_embed.embedding)
|
| 241 |
+
[step 61000] step= 61000 | total=1.6016 | leaf=1.6016 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.5746e-01 | t=29.4s
|
| 242 |
+
[step 61100] step= 61100 | total=1.6894 | leaf=1.6894 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.6527e-01 | t=29.4s
|
| 243 |
+
[step 61200] step= 61200 | total=1.6886 | leaf=1.6886 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.6522e-01 | t=29.4s
|
| 244 |
+
[step 61300] step= 61300 | total=1.6387 | leaf=1.6387 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.5177e-01 | t=29.4s
|
| 245 |
+
[step 61400] step= 61400 | total=1.5261 | leaf=1.5261 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.6625e-01 | t=29.4s
|
| 246 |
+
[step 61500] step= 61500 | total=1.6631 | leaf=1.6631 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.3888e-01 | t=29.4s
|
| 247 |
+
[step 61600] step= 61600 | total=1.6199 | leaf=1.6199 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.5258e-01 | t=29.4s
|
| 248 |
+
[step 61700] step= 61700 | total=1.7124 | leaf=1.7124 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.5857e-01 | t=29.4s
|
| 249 |
+
[step 61800] step= 61800 | total=1.7385 | leaf=1.7385 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.5338e-01 | t=29.4s
|
| 250 |
+
[step 61900] step= 61900 | total=1.6819 | leaf=1.6819 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.5268e-01 | t=29.4s
|
| 251 |
+
[step 62000] [GRAD STATS] total_norm=3.2652e-01 max_single=2.9102e-01 (cond_bias)
|
| 252 |
+
[step 62000] step= 62000 | total=1.6216 | leaf=1.6216 | ancestor=0.0000 | lr=4.88e-04 | gnorm=3.2652e-01 | t=29.4s
|
| 253 |
+
[step 62100] step= 62100 | total=1.6256 | leaf=1.6256 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.6799e-01 | t=29.4s
|
| 254 |
+
[step 62200] step= 62200 | total=1.6823 | leaf=1.6823 | ancestor=0.0000 | lr=4.88e-04 | gnorm=1.7183e-01 | t=29.4s
|
| 255 |
+
[step 62300] step= 62300 | total=1.5884 | leaf=1.5884 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.7847e-01 | t=29.4s
|
| 256 |
+
[step 62400] step= 62400 | total=1.6302 | leaf=1.6302 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.4561e-01 | t=29.4s
|
| 257 |
+
[step 62500] step= 62500 | total=1.7142 | leaf=1.7142 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.8802e-01 | t=29.4s
|
| 258 |
+
[step 62600] step= 62600 | total=1.6270 | leaf=1.6270 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.5313e-01 | t=29.4s
|
| 259 |
+
[step 62700] step= 62700 | total=1.7216 | leaf=1.7216 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.9066e-01 | t=29.4s
|
| 260 |
+
[step 62800] step= 62800 | total=1.7355 | leaf=1.7355 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.6400e-01 | t=29.4s
|
| 261 |
+
[step 62900] step= 62900 | total=1.6545 | leaf=1.6545 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.5417e-01 | t=29.4s
|
| 262 |
+
[step 63000] [GRAD STATS] total_norm=2.1738e-01 max_single=1.6686e-01 (cond_bias)
|
| 263 |
+
[step 63000] step= 63000 | total=1.5992 | leaf=1.5992 | ancestor=0.0000 | lr=4.87e-04 | gnorm=2.1738e-01 | t=29.4s
|
| 264 |
+
[step 63100] step= 63100 | total=1.6780 | leaf=1.6780 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.9450e-01 | t=29.4s
|
| 265 |
+
[step 63200] step= 63200 | total=1.7396 | leaf=1.7396 | ancestor=0.0000 | lr=4.87e-04 | gnorm=2.3368e-01 | t=29.4s
|
| 266 |
+
[step 63300] step= 63300 | total=1.6656 | leaf=1.6656 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.7365e-01 | t=29.4s
|
| 267 |
+
[step 63400] step= 63400 | total=1.6371 | leaf=1.6371 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.8803e-01 | t=29.4s
|
| 268 |
+
[step 63500] step= 63500 | total=1.6772 | leaf=1.6772 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.6042e-01 | t=29.4s
|
| 269 |
+
[step 63600] step= 63600 | total=1.6361 | leaf=1.6361 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.7313e-01 | t=29.4s
|
| 270 |
+
[step 63700] step= 63700 | total=1.5778 | leaf=1.5778 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.9048e-01 | t=29.4s
|
| 271 |
+
[step 63800] step= 63800 | total=1.6205 | leaf=1.6205 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.6852e-01 | t=29.4s
|
| 272 |
+
[step 63900] step= 63900 | total=1.6537 | leaf=1.6537 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.4203e-01 | t=29.4s
|
| 273 |
+
[step 64000] [GRAD STATS] total_norm=1.6786e-01 max_single=6.9538e-02 (cond_bias)
|
| 274 |
+
[step 64000] step= 64000 | total=1.6523 | leaf=1.6523 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.6786e-01 | t=29.4s
|
| 275 |
+
[step 64100] step= 64100 | total=1.6871 | leaf=1.6871 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.7666e-01 | t=29.4s
|
| 276 |
+
[step 64200] step= 64200 | total=1.6121 | leaf=1.6121 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.4752e-01 | t=29.4s
|
| 277 |
+
[step 64300] step= 64300 | total=1.7231 | leaf=1.7231 | ancestor=0.0000 | lr=4.87e-04 | gnorm=1.6762e-01 | t=29.4s
|
| 278 |
+
[step 64400] step= 64400 | total=1.5856 | leaf=1.5856 | ancestor=0.0000 | lr=4.86e-04 | gnorm=2.2080e-01 | t=29.4s
|
| 279 |
+
[step 64500] step= 64500 | total=1.7242 | leaf=1.7242 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.5340e-01 | t=29.4s
|
| 280 |
+
[step 64600] step= 64600 | total=1.6489 | leaf=1.6489 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.9247e-01 | t=29.4s
|
| 281 |
+
[step 64700] step= 64700 | total=1.6625 | leaf=1.6625 | ancestor=0.0000 | lr=4.86e-04 | gnorm=2.0776e-01 | t=29.4s
|
| 282 |
+
[step 64800] step= 64800 | total=1.5959 | leaf=1.5959 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.4619e-01 | t=29.4s
|
| 283 |
+
[step 64900] step= 64900 | total=1.6929 | leaf=1.6929 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.5298e-01 | t=29.4s
|
| 284 |
+
[step 65000] [GRAD STATS] total_norm=1.5534e-01 max_single=5.1606e-02 (blocks.2.adaLN_modulation.bias)
|
| 285 |
+
[step 65000] step= 65000 | total=1.8359 | leaf=1.8359 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.5534e-01 | t=29.4s
|
| 286 |
+
VAL | loss_leaf=1.6870 | loss_ancestor=0.0000 | loss_total=1.6870
|
| 287 |
+
[step 65100] step= 65100 | total=1.6779 | leaf=1.6779 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.7465e-01 | t=35.2s
|
| 288 |
+
[step 65200] step= 65200 | total=1.5728 | leaf=1.5728 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.5006e-01 | t=29.4s
|
| 289 |
+
[step 65300] step= 65300 | total=1.6049 | leaf=1.6049 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.6854e-01 | t=29.4s
|
| 290 |
+
[step 65400] step= 65400 | total=1.7479 | leaf=1.7479 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.6611e-01 | t=29.4s
|
| 291 |
+
[step 65500] step= 65500 | total=1.6832 | leaf=1.6832 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.7662e-01 | t=29.4s
|
| 292 |
+
[step 65600] step= 65600 | total=1.7206 | leaf=1.7206 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.5659e-01 | t=29.4s
|
| 293 |
+
[step 65700] step= 65700 | total=1.6541 | leaf=1.6541 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.6248e-01 | t=29.4s
|
| 294 |
+
[step 65800] step= 65800 | total=1.7460 | leaf=1.7460 | ancestor=0.0000 | lr=4.86e-04 | gnorm=2.0040e-01 | t=29.4s
|
| 295 |
+
[step 65900] step= 65900 | total=1.7865 | leaf=1.7865 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.9724e-01 | t=29.4s
|
| 296 |
+
[step 66000] [GRAD STATS] total_norm=1.9808e-01 max_single=1.3258e-01 (cond_bias)
|
| 297 |
+
[step 66000] step= 66000 | total=1.6966 | leaf=1.6966 | ancestor=0.0000 | lr=4.86e-04 | gnorm=1.9808e-01 | t=29.4s
|
| 298 |
+
[step 66100] step= 66100 | total=1.8127 | leaf=1.8127 | ancestor=0.0000 | lr=4.86e-04 | gnorm=2.1122e-01 | t=29.4s
|
| 299 |
+
[step 66200] step= 66200 | total=1.6657 | leaf=1.6657 | ancestor=0.0000 | lr=4.86e-04 | gnorm=2.2460e-01 | t=29.4s
|
| 300 |
+
[step 66300] step= 66300 | total=1.5649 | leaf=1.5649 | ancestor=0.0000 | lr=4.86e-04 | gnorm=2.1049e-01 | t=29.4s
|
| 301 |
+
[step 66400] step= 66400 | total=1.6271 | leaf=1.6271 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.4267e-01 | t=29.4s
|
| 302 |
+
[step 66500] step= 66500 | total=1.6712 | leaf=1.6712 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.7580e-01 | t=29.4s
|
| 303 |
+
[step 66600] step= 66600 | total=1.6660 | leaf=1.6660 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.4811e-01 | t=29.4s
|
| 304 |
+
[step 66700] step= 66700 | total=1.7673 | leaf=1.7673 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.3254e-01 | t=29.4s
|
| 305 |
+
[step 66800] step= 66800 | total=1.5879 | leaf=1.5879 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.4041e-01 | t=29.4s
|
| 306 |
+
[step 66900] step= 66900 | total=1.6973 | leaf=1.6973 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.5577e-01 | t=29.4s
|
| 307 |
+
[step 67000] [GRAD STATS] total_norm=1.4476e-01 max_single=5.4222e-02 (cond_bias)
|
| 308 |
+
[step 67000] step= 67000 | total=1.6885 | leaf=1.6885 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.4476e-01 | t=29.4s
|
| 309 |
+
[step 67100] step= 67100 | total=1.5969 | leaf=1.5969 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.6148e-01 | t=29.4s
|
| 310 |
+
[step 67200] step= 67200 | total=1.6792 | leaf=1.6792 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.4123e-01 | t=29.4s
|
| 311 |
+
[step 67300] step= 67300 | total=1.7296 | leaf=1.7296 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.9610e-01 | t=29.4s
|
| 312 |
+
[step 67400] step= 67400 | total=1.6658 | leaf=1.6658 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.6470e-01 | t=29.4s
|
| 313 |
+
[step 67500] step= 67500 | total=1.4471 | leaf=1.4471 | ancestor=0.0000 | lr=4.85e-04 | gnorm=2.0719e-01 | t=29.4s
|
| 314 |
+
[step 67600] step= 67600 | total=1.6749 | leaf=1.6749 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.4170e-01 | t=29.4s
|
| 315 |
+
[step 67700] step= 67700 | total=1.7487 | leaf=1.7487 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.5683e-01 | t=29.4s
|
| 316 |
+
[step 67800] step= 67800 | total=1.7138 | leaf=1.7138 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.7454e-01 | t=29.4s
|
| 317 |
+
[step 67900] step= 67900 | total=1.5902 | leaf=1.5902 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.3706e-01 | t=29.4s
|
| 318 |
+
[step 68000] [GRAD STATS] total_norm=1.9275e-01 max_single=1.3105e-01 (cond_bias)
|
| 319 |
+
[step 68000] step= 68000 | total=1.7304 | leaf=1.7304 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.9275e-01 | t=29.4s
|
| 320 |
+
[step 68100] step= 68100 | total=1.5369 | leaf=1.5369 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.6084e-01 | t=29.4s
|
| 321 |
+
[step 68200] step= 68200 | total=1.7103 | leaf=1.7103 | ancestor=0.0000 | lr=4.85e-04 | gnorm=1.5547e-01 | t=29.4s
|
| 322 |
+
[step 68300] step= 68300 | total=1.5637 | leaf=1.5637 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.5955e-01 | t=29.4s
|
| 323 |
+
[step 68400] step= 68400 | total=1.6015 | leaf=1.6015 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.6391e-01 | t=29.4s
|
| 324 |
+
[step 68500] step= 68500 | total=1.7069 | leaf=1.7069 | ancestor=0.0000 | lr=4.84e-04 | gnorm=2.5777e-01 | t=29.4s
|
| 325 |
+
[step 68600] step= 68600 | total=1.6954 | leaf=1.6954 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.5043e-01 | t=29.4s
|
| 326 |
+
[step 68700] step= 68700 | total=1.7062 | leaf=1.7062 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.4991e-01 | t=29.4s
|
| 327 |
+
[step 68800] step= 68800 | total=1.6699 | leaf=1.6699 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.6656e-01 | t=29.4s
|
| 328 |
+
[step 68900] step= 68900 | total=1.6953 | leaf=1.6953 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.7979e-01 | t=29.4s
|
| 329 |
+
[step 69000] [GRAD STATS] total_norm=1.6508e-01 max_single=1.0040e-01 (cond_bias)
|
| 330 |
+
[step 69000] step= 69000 | total=1.7515 | leaf=1.7515 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.6508e-01 | t=29.4s
|
| 331 |
+
[step 69100] step= 69100 | total=1.7513 | leaf=1.7513 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.7659e-01 | t=29.4s
|
| 332 |
+
[step 69200] step= 69200 | total=1.8290 | leaf=1.8290 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.9133e-01 | t=29.4s
|
| 333 |
+
[step 69300] step= 69300 | total=1.7576 | leaf=1.7576 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.8627e-01 | t=29.4s
|
| 334 |
+
[step 69400] step= 69400 | total=1.7017 | leaf=1.7017 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.7324e-01 | t=29.4s
|
| 335 |
+
[step 69500] step= 69500 | total=1.7388 | leaf=1.7388 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.4469e-01 | t=29.4s
|
| 336 |
+
[step 69600] step= 69600 | total=1.6485 | leaf=1.6485 | ancestor=0.0000 | lr=4.84e-04 | gnorm=2.0231e-01 | t=29.4s
|
| 337 |
+
[step 69700] step= 69700 | total=1.6582 | leaf=1.6582 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.4261e-01 | t=29.4s
|
| 338 |
+
[step 69800] step= 69800 | total=1.7196 | leaf=1.7196 | ancestor=0.0000 | lr=4.84e-04 | gnorm=2.2882e-01 | t=29.4s
|
| 339 |
+
[step 69900] step= 69900 | total=1.6324 | leaf=1.6324 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.3978e-01 | t=29.4s
|
| 340 |
+
[step 70000] [GRAD STATS] total_norm=1.3133e-01 max_single=4.8917e-02 (vocab_embed.embedding)
|
| 341 |
+
[step 70000] step= 70000 | total=1.6516 | leaf=1.6516 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.3133e-01 | t=29.4s
|
| 342 |
+
VAL | loss_leaf=1.6988 | loss_ancestor=0.0000 | loss_total=1.6988
|
| 343 |
+
Saved checkpoint: outputs/sad_new/ckpt_70000.pt
|
| 344 |
+
[step 70100] step= 70100 | total=1.7090 | leaf=1.7090 | ancestor=0.0000 | lr=4.84e-04 | gnorm=1.9492e-01 | t=45.2s
|
| 345 |
+
[step 70200] step= 70200 | total=1.6397 | leaf=1.6397 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.6381e-01 | t=29.4s
|
| 346 |
+
[step 70300] step= 70300 | total=1.7359 | leaf=1.7359 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.5546e-01 | t=29.4s
|
| 347 |
+
[step 70400] step= 70400 | total=1.6837 | leaf=1.6837 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.3671e-01 | t=29.4s
|
| 348 |
+
[step 70500] step= 70500 | total=1.7988 | leaf=1.7988 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.6073e-01 | t=29.4s
|
| 349 |
+
[step 70600] step= 70600 | total=1.5856 | leaf=1.5856 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.3561e-01 | t=29.4s
|
| 350 |
+
[step 70700] step= 70700 | total=1.6540 | leaf=1.6540 | ancestor=0.0000 | lr=4.83e-04 | gnorm=2.2117e-01 | t=29.4s
|
| 351 |
+
[step 70800] step= 70800 | total=1.7218 | leaf=1.7218 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.3630e-01 | t=29.4s
|
| 352 |
+
[step 70900] step= 70900 | total=1.6739 | leaf=1.6739 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.5986e-01 | t=29.4s
|
| 353 |
+
[step 71000] [GRAD STATS] total_norm=1.4029e-01 max_single=4.8562e-02 (vocab_embed.embedding)
|
| 354 |
+
[step 71000] step= 71000 | total=1.6636 | leaf=1.6636 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.4029e-01 | t=29.5s
|
| 355 |
+
[step 71100] step= 71100 | total=1.6232 | leaf=1.6232 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.4279e-01 | t=29.4s
|
| 356 |
+
[step 71200] step= 71200 | total=1.7109 | leaf=1.7109 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.7071e-01 | t=29.4s
|
| 357 |
+
[step 71300] step= 71300 | total=1.4698 | leaf=1.4698 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.6542e-01 | t=29.4s
|
| 358 |
+
[step 71400] step= 71400 | total=1.6540 | leaf=1.6540 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.4418e-01 | t=29.4s
|
| 359 |
+
[step 71500] step= 71500 | total=1.6178 | leaf=1.6178 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.5081e-01 | t=29.4s
|
| 360 |
+
[step 71600] step= 71600 | total=1.6782 | leaf=1.6782 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.3567e-01 | t=29.4s
|
| 361 |
+
[step 71700] step= 71700 | total=1.5286 | leaf=1.5286 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.3491e-01 | t=29.4s
|
| 362 |
+
[step 71800] step= 71800 | total=1.6589 | leaf=1.6589 | ancestor=0.0000 | lr=4.83e-04 | gnorm=1.4681e-01 | t=29.4s
|
| 363 |
+
[step 71900] step= 71900 | total=1.5867 | leaf=1.5867 | ancestor=0.0000 | lr=4.83e-04 | gnorm=2.4990e-01 | t=29.4s
|
| 364 |
+
[step 72000] [GRAD STATS] total_norm=1.6408e-01 max_single=5.8985e-02 (cond_bias)
|
| 365 |
+
[step 72000] step= 72000 | total=1.7418 | leaf=1.7418 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.6408e-01 | t=29.4s
|
| 366 |
+
[step 72100] step= 72100 | total=1.7059 | leaf=1.7059 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.3470e-01 | t=29.4s
|
| 367 |
+
[step 72200] step= 72200 | total=1.6199 | leaf=1.6199 | ancestor=0.0000 | lr=4.82e-04 | gnorm=2.3167e-01 | t=29.4s
|
| 368 |
+
[step 72300] step= 72300 | total=1.6562 | leaf=1.6562 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.5067e-01 | t=29.4s
|
| 369 |
+
[step 72400] step= 72400 | total=1.6393 | leaf=1.6393 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.6301e-01 | t=29.4s
|
| 370 |
+
[step 72500] step= 72500 | total=1.6254 | leaf=1.6254 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.5098e-01 | t=29.4s
|
| 371 |
+
[step 72600] step= 72600 | total=1.6483 | leaf=1.6483 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.3672e-01 | t=29.4s
|
| 372 |
+
[step 72700] step= 72700 | total=1.6722 | leaf=1.6722 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.8102e-01 | t=29.4s
|
| 373 |
+
[step 72800] step= 72800 | total=1.6622 | leaf=1.6622 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.6728e-01 | t=29.4s
|
| 374 |
+
[step 72900] step= 72900 | total=1.6610 | leaf=1.6610 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.3910e-01 | t=29.4s
|
| 375 |
+
[step 73000] [GRAD STATS] total_norm=1.5449e-01 max_single=5.8862e-02 (cond_bias)
|
| 376 |
+
[step 73000] step= 73000 | total=1.6547 | leaf=1.6547 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.5449e-01 | t=29.4s
|
| 377 |
+
[step 73100] step= 73100 | total=1.6012 | leaf=1.6012 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.5961e-01 | t=29.4s
|
| 378 |
+
[step 73200] step= 73200 | total=1.8152 | leaf=1.8152 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.4777e-01 | t=29.4s
|
| 379 |
+
[step 73300] step= 73300 | total=1.7231 | leaf=1.7231 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.4236e-01 | t=29.4s
|
| 380 |
+
[step 73400] step= 73400 | total=1.6719 | leaf=1.6719 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.8646e-01 | t=29.4s
|
| 381 |
+
[step 73500] step= 73500 | total=1.6734 | leaf=1.6734 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.3912e-01 | t=29.4s
|
| 382 |
+
[step 73600] step= 73600 | total=1.6755 | leaf=1.6755 | ancestor=0.0000 | lr=4.82e-04 | gnorm=1.4946e-01 | t=29.4s
|
| 383 |
+
[step 73700] step= 73700 | total=1.6258 | leaf=1.6258 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.3630e-01 | t=29.4s
|
| 384 |
+
[step 73800] step= 73800 | total=1.6201 | leaf=1.6201 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.8114e-01 | t=29.4s
|
| 385 |
+
[step 73900] step= 73900 | total=1.6944 | leaf=1.6944 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.5660e-01 | t=29.4s
|
| 386 |
+
[step 74000] [GRAD STATS] total_norm=1.6261e-01 max_single=9.5840e-02 (cond_bias)
|
| 387 |
+
[step 74000] step= 74000 | total=1.6259 | leaf=1.6259 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.6261e-01 | t=29.4s
|
| 388 |
+
[step 74100] step= 74100 | total=1.6624 | leaf=1.6624 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.4686e-01 | t=29.4s
|
| 389 |
+
[step 74200] step= 74200 | total=1.6980 | leaf=1.6980 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.9481e-01 | t=29.4s
|
| 390 |
+
[step 74300] step= 74300 | total=1.6770 | leaf=1.6770 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.6858e-01 | t=29.4s
|
| 391 |
+
[step 74400] step= 74400 | total=1.6853 | leaf=1.6853 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.4249e-01 | t=29.4s
|
| 392 |
+
[step 74500] step= 74500 | total=1.6999 | leaf=1.6999 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.5354e-01 | t=29.4s
|
| 393 |
+
[step 74600] step= 74600 | total=1.5823 | leaf=1.5823 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.5364e-01 | t=29.4s
|
| 394 |
+
[step 74700] step= 74700 | total=1.6734 | leaf=1.6734 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.4177e-01 | t=29.4s
|
| 395 |
+
[step 74800] step= 74800 | total=1.6466 | leaf=1.6466 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.3932e-01 | t=29.4s
|
| 396 |
+
[step 74900] step= 74900 | total=1.6903 | leaf=1.6903 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.4816e-01 | t=29.4s
|
| 397 |
+
[step 75000] [GRAD STATS] total_norm=1.3522e-01 max_single=4.7961e-02 (vocab_embed.embedding)
|
| 398 |
+
[step 75000] step= 75000 | total=1.7434 | leaf=1.7434 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.3522e-01 | t=29.4s
|
| 399 |
+
VAL | loss_leaf=1.6804 | loss_ancestor=0.0000 | loss_total=1.6804
|
| 400 |
+
[step 75100] step= 75100 | total=1.7183 | leaf=1.7183 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.5708e-01 | t=35.2s
|
| 401 |
+
[step 75200] step= 75200 | total=1.6771 | leaf=1.6771 | ancestor=0.0000 | lr=4.81e-04 | gnorm=2.6821e-01 | t=29.4s
|
| 402 |
+
[step 75300] step= 75300 | total=1.7936 | leaf=1.7936 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.4377e-01 | t=29.4s
|
| 403 |
+
[step 75400] step= 75400 | total=1.7046 | leaf=1.7046 | ancestor=0.0000 | lr=4.81e-04 | gnorm=1.5642e-01 | t=29.4s
|
| 404 |
+
[step 75500] step= 75500 | total=1.6964 | leaf=1.6964 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.6397e-01 | t=29.4s
|
| 405 |
+
[step 75600] step= 75600 | total=1.6761 | leaf=1.6761 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.3972e-01 | t=29.4s
|
| 406 |
+
[step 75700] step= 75700 | total=1.7130 | leaf=1.7130 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.3996e-01 | t=29.4s
|
| 407 |
+
[step 75800] step= 75800 | total=1.7413 | leaf=1.7413 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.7830e-01 | t=29.4s
|
| 408 |
+
[step 75900] step= 75900 | total=1.6238 | leaf=1.6238 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.3791e-01 | t=29.4s
|
| 409 |
+
[step 76000] [GRAD STATS] total_norm=2.0344e-01 max_single=1.5330e-01 (cond_bias)
|
| 410 |
+
[step 76000] step= 76000 | total=1.5425 | leaf=1.5425 | ancestor=0.0000 | lr=4.80e-04 | gnorm=2.0344e-01 | t=29.4s
|
| 411 |
+
[step 76100] step= 76100 | total=1.6461 | leaf=1.6461 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.8285e-01 | t=29.4s
|
| 412 |
+
[step 76200] step= 76200 | total=1.6065 | leaf=1.6065 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.4446e-01 | t=29.4s
|
| 413 |
+
[step 76300] step= 76300 | total=1.5556 | leaf=1.5556 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.5392e-01 | t=29.4s
|
| 414 |
+
[step 76400] step= 76400 | total=1.8598 | leaf=1.8598 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.6925e-01 | t=29.4s
|
| 415 |
+
[step 76500] step= 76500 | total=1.6464 | leaf=1.6464 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.5851e-01 | t=29.4s
|
| 416 |
+
[step 76600] step= 76600 | total=1.7214 | leaf=1.7214 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.4042e-01 | t=29.4s
|
| 417 |
+
[step 76700] step= 76700 | total=1.8083 | leaf=1.8083 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.3794e-01 | t=29.4s
|
| 418 |
+
[step 76800] step= 76800 | total=1.7032 | leaf=1.7032 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.5037e-01 | t=29.3s
|
| 419 |
+
[step 76900] step= 76900 | total=1.6417 | leaf=1.6417 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.2436e-01 | t=29.4s
|
| 420 |
+
[step 77000] [GRAD STATS] total_norm=1.2736e+00 max_single=9.6650e-01 (cond_bias)
|
| 421 |
+
[step 77000] step= 77000 | total=1.7998 | leaf=1.7998 | ancestor=0.0000 | lr=4.80e-04 | gnorm=1.2736e+00 | t=29.4s
|
| 422 |
+
[step 77100] step= 77100 | total=1.6109 | leaf=1.6109 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.9753e-01 | t=29.4s
|
| 423 |
+
[step 77200] step= 77200 | total=1.6850 | leaf=1.6850 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.3362e-01 | t=29.4s
|
| 424 |
+
[step 77300] step= 77300 | total=1.6471 | leaf=1.6471 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.6398e-01 | t=29.4s
|
| 425 |
+
[step 77400] step= 77400 | total=1.7305 | leaf=1.7305 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.4951e-01 | t=29.4s
|
| 426 |
+
[step 77500] step= 77500 | total=1.7112 | leaf=1.7112 | ancestor=0.0000 | lr=4.79e-04 | gnorm=2.2057e-01 | t=29.4s
|
| 427 |
+
[step 77600] step= 77600 | total=1.5773 | leaf=1.5773 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.7400e-01 | t=29.4s
|
| 428 |
+
[step 77700] step= 77700 | total=1.5612 | leaf=1.5612 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.6646e-01 | t=29.4s
|
| 429 |
+
[step 77800] step= 77800 | total=1.6785 | leaf=1.6785 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.4282e-01 | t=29.4s
|
| 430 |
+
[step 77900] step= 77900 | total=1.6156 | leaf=1.6156 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.5648e-01 | t=29.4s
|
| 431 |
+
[step 78000] [GRAD STATS] total_norm=1.7338e-01 max_single=1.1301e-01 (cond_bias)
|
| 432 |
+
[step 78000] step= 78000 | total=1.6466 | leaf=1.6466 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.7338e-01 | t=29.4s
|
| 433 |
+
[step 78100] step= 78100 | total=1.6760 | leaf=1.6760 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.7093e-01 | t=29.4s
|
| 434 |
+
[step 78200] step= 78200 | total=1.5785 | leaf=1.5785 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.8512e-01 | t=29.4s
|
| 435 |
+
[step 78300] step= 78300 | total=1.7146 | leaf=1.7146 | ancestor=0.0000 | lr=4.79e-04 | gnorm=5.5491e-01 | t=29.4s
|
| 436 |
+
[step 78400] step= 78400 | total=1.5605 | leaf=1.5605 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.9457e-01 | t=29.4s
|
| 437 |
+
[step 78500] step= 78500 | total=1.6406 | leaf=1.6406 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.4481e-01 | t=29.4s
|
| 438 |
+
[step 78600] step= 78600 | total=1.5374 | leaf=1.5374 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.9257e-01 | t=29.4s
|
| 439 |
+
[step 78700] step= 78700 | total=1.7437 | leaf=1.7437 | ancestor=0.0000 | lr=4.79e-04 | gnorm=1.5490e-01 | t=29.4s
|
| 440 |
+
[step 78800] step= 78800 | total=1.7811 | leaf=1.7811 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.6204e-01 | t=29.4s
|
| 441 |
+
[step 78900] step= 78900 | total=1.6770 | leaf=1.6770 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.3678e-01 | t=29.4s
|
| 442 |
+
[step 79000] [GRAD STATS] total_norm=1.5521e-01 max_single=7.4286e-02 (cond_bias)
|
| 443 |
+
[step 79000] step= 79000 | total=1.6020 | leaf=1.6020 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.5521e-01 | t=29.4s
|
| 444 |
+
[step 79100] step= 79100 | total=1.6445 | leaf=1.6445 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.3656e-01 | t=29.4s
|
| 445 |
+
[step 79200] step= 79200 | total=1.6579 | leaf=1.6579 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.5916e-01 | t=29.4s
|
| 446 |
+
[step 79300] step= 79300 | total=1.6523 | leaf=1.6523 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.4782e-01 | t=29.4s
|
| 447 |
+
[step 79400] step= 79400 | total=1.5564 | leaf=1.5564 | ancestor=0.0000 | lr=4.78e-04 | gnorm=2.5338e-01 | t=29.4s
|
| 448 |
+
[step 79500] step= 79500 | total=1.6459 | leaf=1.6459 | ancestor=0.0000 | lr=4.78e-04 | gnorm=3.5189e-01 | t=29.4s
|
| 449 |
+
[step 79600] step= 79600 | total=1.5907 | leaf=1.5907 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.9920e-01 | t=29.4s
|
| 450 |
+
[step 79700] step= 79700 | total=1.5915 | leaf=1.5915 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.8415e-01 | t=29.4s
|
| 451 |
+
[step 79800] step= 79800 | total=1.5998 | leaf=1.5998 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.6393e-01 | t=29.4s
|
| 452 |
+
[step 79900] step= 79900 | total=1.6635 | leaf=1.6635 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.7371e-01 | t=29.4s
|
| 453 |
+
[step 80000] [GRAD STATS] total_norm=1.2990e-01 max_single=4.7465e-02 (vocab_embed.embedding)
|
| 454 |
+
[step 80000] step= 80000 | total=1.7610 | leaf=1.7610 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.2990e-01 | t=29.4s
|
| 455 |
+
VAL | loss_leaf=1.6713 | loss_ancestor=0.0000 | loss_total=1.6713
|
| 456 |
+
Saved checkpoint: outputs/sad_new/ckpt_80000.pt
|
| 457 |
+
[step 80100] step= 80100 | total=1.6292 | leaf=1.6292 | ancestor=0.0000 | lr=4.78e-04 | gnorm=2.3447e-01 | t=45.9s
|
| 458 |
+
[step 80200] step= 80200 | total=1.7046 | leaf=1.7046 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.8097e-01 | t=29.4s
|
| 459 |
+
[step 80300] step= 80300 | total=1.6426 | leaf=1.6426 | ancestor=0.0000 | lr=4.78e-04 | gnorm=1.4524e-01 | t=29.4s
|
| 460 |
+
[step 80400] step= 80400 | total=1.8383 | leaf=1.8383 | ancestor=0.0000 | lr=4.77e-04 | gnorm=2.7141e-01 | t=29.4s
|
| 461 |
+
[step 80500] step= 80500 | total=1.7846 | leaf=1.7846 | ancestor=0.0000 | lr=4.77e-04 | gnorm=1.0453e+00 | t=29.4s
|
| 462 |
+
[step 80600] step= 80600 | total=1.6109 | leaf=1.6109 | ancestor=0.0000 | lr=4.77e-04 | gnorm=1.2309e-01 | t=29.4s
|
| 463 |
+
[step 80700] step= 80700 | total=1.6549 | leaf=1.6549 | ancestor=0.0000 | lr=4.77e-04 | gnorm=3.9672e-01 | t=29.4s
|
| 464 |
+
[step 80800] step= 80800 | total=1.5566 | leaf=1.5566 | ancestor=0.0000 | lr=4.77e-04 | gnorm=1.4033e-01 | t=29.4s
|
| 465 |
+
[step 80900] step= 80900 | total=1.7379 | leaf=1.7379 | ancestor=0.0000 | lr=4.77e-04 | gnorm=1.6260e-01 | t=29.4s
|
| 466 |
+
[step 81000] [GRAD STATS] total_norm=1.3210e-01 max_single=5.1436e-02 (cond_bias)
|
| 467 |
+
[step 81000] step= 81000 | total=1.7643 | leaf=1.7643 | ancestor=0.0000 | lr=4.77e-04 | gnorm=1.3210e-01 | t=29.4s
|
| 468 |
+
[step 81100] step= 81100 | total=1.6296 | leaf=1.6296 | ancestor=0.0000 | lr=4.77e-04 | gnorm=1.4525e-01 | t=29.4s
|
| 469 |
+
[step 81200] step= 81200 | total=1.6965 | leaf=1.6965 | ancestor=0.0000 | lr=4.77e-04 | gnorm=1.4997e-01 | t=29.4s
|
| 470 |
+
[step 81300] step= 81300 | total=1.6597 | leaf=1.6597 | ancestor=0.0000 | lr=4.77e-04 | gnorm=2.7777e-01 | t=29.4s
|
| 471 |
+
[step 81400] step= 81400 | total=1.6872 | leaf=1.6872 | ancestor=0.0000 | lr=4.77e-04 | gnorm=3.3915e-01 | t=29.4s
|
| 472 |
+
[step 81500] step= 81500 | total=1.6170 | leaf=1.6170 | ancestor=0.0000 | lr=4.77e-04 | gnorm=2.3771e-01 | t=29.4s
|
| 473 |
+
[step 81600] step= 81600 | total=1.6623 | leaf=1.6623 | ancestor=0.0000 | lr=4.77e-04 | gnorm=2.6690e-01 | t=29.4s
|
| 474 |
+
[step 81700] step= 81700 | total=1.6704 | leaf=1.6704 | ancestor=0.0000 | lr=4.77e-04 | gnorm=2.5626e-01 | t=29.4s
|
| 475 |
+
[step 81800] step= 81800 | total=1.6573 | leaf=1.6573 | ancestor=0.0000 | lr=4.77e-04 | gnorm=2.3904e-01 | t=29.4s
|
| 476 |
+
[step 81900] step= 81900 | total=1.5394 | leaf=1.5394 | ancestor=0.0000 | lr=4.77e-04 | gnorm=1.5953e-01 | t=29.4s
|
| 477 |
+
[step 82000] [GRAD STATS] total_norm=3.9102e-01 max_single=3.5490e-01 (cond_bias)
|
| 478 |
+
[step 82000] step= 82000 | total=1.6804 | leaf=1.6804 | ancestor=0.0000 | lr=4.76e-04 | gnorm=3.9102e-01 | t=29.4s
|
| 479 |
+
[step 82100] step= 82100 | total=2.3186 | leaf=2.3186 | ancestor=0.0000 | lr=4.76e-04 | gnorm=1.3618e+01 | t=29.4s
|
| 480 |
+
[step 82120] [GRAD WARN] grad_norm=1.0183e+05 (very large). Largest single-param grad=9.6345e-01 in cond_bias
|
| 481 |
+
[step 82121] [GRAD WARN] grad_norm=1.5547e+05 (very large). Largest single-param grad=9.5347e-01 in cond_bias
|
| 482 |
+
[step 82122] [GRAD WARN] grad_norm=5.0868e+05 (very large). Largest single-param grad=5.7146e-01 in vocab_embed.embedding
|
| 483 |
+
[step 82123] [GRAD WARN] grad_norm=3.5232e+05 (very large). Largest single-param grad=9.4440e-01 in cond_bias
|
| 484 |
+
[step 82124] [GRAD WARN] grad_norm=9.4963e+05 (very large). Largest single-param grad=6.0842e-01 in cond_bias
|
| 485 |
+
[step 82125] [GRAD WARN] grad_norm=6.8272e+05 (very large). Largest single-param grad=9.4152e-01 in cond_bias
|
| 486 |
+
[step 82126] [GRAD WARN] grad_norm=1.2148e+06 (very large). Largest single-param grad=9.2043e-01 in cond_bias
|
| 487 |
+
[step 82127] [GRAD WARN] grad_norm=1.8992e+06 (very large). Largest single-param grad=9.2457e-01 in cond_bias
|
| 488 |
+
[step 82128] [GRAD WARN] grad_norm=3.4793e+06 (very large). Largest single-param grad=9.0946e-01 in cond_bias
|
| 489 |
+
[step 82129] [GRAD WARN] grad_norm=2.9167e+07 (very large). Largest single-param grad=5.7449e-01 in input_proj.weight
|
| 490 |
+
[step 82130] [GRAD WARN] grad_norm=1.3404e+07 (very large). Largest single-param grad=7.7525e-01 in cond_bias
|
| 491 |
+
[step 82131] [GRAD WARN] grad_norm=2.9055e+08 (very large). Largest single-param grad=5.4502e-01 in vocab_embed.embedding
|
| 492 |
+
[step 82132] [GRAD WARN] grad_norm=9.4327e+08 (very large). Largest single-param grad=7.1316e-01 in ancestor_table.embeddings.0
|
| 493 |
+
[step 82133] [GRAD WARN] grad_norm=3.8279e+08 (very large). Largest single-param grad=5.5909e-01 in input_proj.weight
|
| 494 |
+
[step 82134] [GRAD WARN] grad_norm=4.4741e+10 (very large). Largest single-param grad=5.7886e-01 in vocab_embed.embedding
|
| 495 |
+
[step 82135] [GRAD WARN] grad_norm=7.4656e+15 (very large). Largest single-param grad=5.9911e-01 in vocab_embed.embedding
|
| 496 |
+
[step 82136] [GRAD WARN] grad_norm=1.9887e+12 (very large). Largest single-param grad=6.1054e-01 in vocab_embed.embedding
|
| 497 |
+
[step 82137] [GRAD WARN] grad_norm=1.4541e+12 (very large). Largest single-param grad=6.4434e-01 in vocab_embed.embedding
|
| 498 |
+
[step 82138] [GRAD WARN] grad_norm=1.4108e+14 (very large). Largest single-param grad=6.5839e-01 in ancestor_table.embeddings.0
|
| 499 |
+
[step 82139] [GRAD WARN] grad_norm=2.0521e+14 (very large). Largest single-param grad=7.1371e-01 in ancestor_table.embeddings.0
|
| 500 |
+
[step 82140] [CLIP ERROR] total_norm non-finite after clip_grad_norm_: inf
|
| 501 |
+
File "/home/sunhc/diffusion/sad/scripts/train.py", line 716, in <module>
|
| 502 |
+
main()
|
| 503 |
+
File "/home/sunhc/diffusion/sad/scripts/train.py", line 630, in main
|
| 504 |
+
clipped_norm = debugger.clip_grads(step)
|
| 505 |
+
^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 506 |
+
File "/home/sunhc/diffusion/sad/src/utils/debug_utils.py", line 873, in clip_grads
|
| 507 |
+
raise RuntimeError(f"{prefix}Non-finite grad norm after clipping")
|
| 508 |
+
RuntimeError: [step 82140] Non-finite grad norm after clipping
|
| 509 |
+
[rank0]: Traceback (most recent call last):
|
| 510 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train.py", line 716, in <module>
|
| 511 |
+
[rank0]: main()
|
| 512 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train.py", line 630, in main
|
| 513 |
+
[rank0]: clipped_norm = debugger.clip_grads(step)
|
| 514 |
+
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^
|
| 515 |
+
[rank0]: File "/home/sunhc/diffusion/sad/src/utils/debug_utils.py", line 873, in clip_grads
|
| 516 |
+
[rank0]: raise RuntimeError(f"{prefix}Non-finite grad norm after clipping")
|
| 517 |
+
[rank0]: RuntimeError: [step 82140] Non-finite grad norm after clipping
|
wandb/run-20260423_093030-7hhwxxjb/logs/debug-core.log
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T09:30:30.818208074Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpml0577la/port-1259107.txt","pid":1259107,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T09:30:30.818638638Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1259107}
|
| 3 |
+
{"time":"2026-04-23T09:30:30.818621902Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1259107-1265135-2717039742/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T09:30:31.003445164Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T09:30:31.01152075Z","level":"INFO","msg":"handleInformInit: received","streamId":"7hhwxxjb","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T09:30:31.850681309Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"7hhwxxjb","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T09:30:38.399550382Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"t1kebqjefian"}
|
| 8 |
+
{"time":"2026-04-23T12:59:35.106799432Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-23T12:59:35.106849543Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-04-23T12:59:35.106868095Z","level":"INFO","msg":"server is shutting down"}
|
| 11 |
+
{"time":"2026-04-23T12:59:35.106908447Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-23T12:59:35.106953182Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1259107-1265135-2717039742/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-23T12:59:37.385871331Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-04-23T12:59:37.385912749Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 15 |
+
{"time":"2026-04-23T12:59:37.385928567Z","level":"INFO","msg":"server is closed"}
|
wandb/run-20260423_093030-7hhwxxjb/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/run-20260423_093030-7hhwxxjb/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 09:30:30,786 INFO MainThread:1259107 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-23 09:30:30,786 INFO MainThread:1259107 [wandb_setup.py:_flush():81] Configure stats pid to 1259107
|
| 3 |
+
2026-04-23 09:30:30,786 INFO MainThread:1259107 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 09:30:30,786 INFO MainThread:1259107 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260423_093030-7hhwxxjb/logs/debug.log
|
| 5 |
+
2026-04-23 09:30:30,786 INFO MainThread:1259107 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260423_093030-7hhwxxjb/logs/debug-internal.log
|
| 6 |
+
2026-04-23 09:30:30,786 INFO MainThread:1259107 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-23 09:30:30,786 INFO MainThread:1259107 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 8, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top3_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0005, 'lr_min': 5e-05, 'warmup_steps': 10000, 'weight_decay': 0.02, 'grad_clip': 1.0, 'adam_betas': [0.9, 0.99], 'adam_eps': 1e-09, 'dtype': 'bf16', 'compile': 'default', 'curriculum_t_max_init': 0.3, 'curriculum_steps': 20000, 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_new_owt', 'save_dir': 'outputs/sad_new'}, 'debug': {'debug_mode': True, 'raise_on_nan': True, 'check_inputs': True, 'check_batch_every': 100, 'check_outputs': True, 'check_outputs_every': 100, 'check_loss': True, 'check_grads': True, 'check_params': True, 'check_params_every': 1000, 'use_hooks': False, 'save_on_nan': True, 'print_stats_every': 1000, 'grad_norm_warn_threshold': 100000.0}, '_wandb': {}}
|
| 9 |
+
2026-04-23 09:30:30,786 INFO MainThread:1259107 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-23 09:30:31,004 INFO MainThread:1259107 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-23 09:30:31,007 INFO MainThread:1259107 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-23 09:30:31,009 INFO MainThread:1259107 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-23 09:30:31,015 INFO MainThread:1259107 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 09:30:33,268 INFO MainThread:1259107 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-23 09:30:33,389 INFO MainThread:1259107 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-23 09:30:33,389 INFO MainThread:1259107 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-23 09:30:33,389 INFO MainThread:1259107 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-23 09:30:33,389 INFO MainThread:1259107 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-23 09:30:33,392 INFO MainThread:1259107 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-23 12:59:35,106 INFO wandb-AsyncioManager-main:1259107 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-23 12:59:35,107 INFO wandb-AsyncioManager-main:1259107 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260423_151526-rkikhrj0/files/output.log
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
full-denoise training: 100%|██████████| 200/200 [01:27<00:00, 2.27it/s, leaf=185.3901, lr=1.0e-05]
|
| 2 |
+
[step 0] [GRAD STATS] total_norm=2.6399e+02 max_single=6.6615e-01 (vocab_embed.embedding)
|
| 3 |
+
[step 0] step= 0 | total=212.8117 | leaf=212.8117 | ancestor=0.0000 | lr=0.00e+00 | gnorm=2.6399e+02 | t=26.8s
|
| 4 |
+
[step 100] step= 100 | total=131.1876 | leaf=131.1876 | ancestor=0.0000 | lr=5.00e-06 | gnorm=5.9224e+01 | t=27.4s
|
| 5 |
+
Saved checkpoint: outputs/sad_new/ckpt_200.pt
|
| 6 |
+
Training complete.
|
wandb/run-20260423_151526-rkikhrj0/logs/debug-core.log
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T15:15:26.311492577Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpumew7l2n/port-1632880.txt","pid":1632880,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T15:15:26.317381602Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1632880}
|
| 3 |
+
{"time":"2026-04-23T15:15:26.317363712Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1632880-1634160-1171806119/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T15:15:26.50676938Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T15:15:26.517275579Z","level":"INFO","msg":"handleInformInit: received","streamId":"rkikhrj0","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T15:15:27.723516447Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"rkikhrj0","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T15:15:33.760610853Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"v4ef2ultmpj6"}
|
| 8 |
+
{"time":"2026-04-23T15:16:56.771247387Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-23T15:16:56.771271921Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-04-23T15:16:56.771324776Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-23T15:16:56.771340334Z","level":"INFO","msg":"server is shutting down"}
|
| 12 |
+
{"time":"2026-04-23T15:16:56.771544106Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1632880-1634160-1171806119/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-23T15:16:59.532971915Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-04-23T15:16:59.533006556Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 15 |
+
{"time":"2026-04-23T15:16:59.533022521Z","level":"INFO","msg":"server is closed"}
|
wandb/run-20260423_151526-rkikhrj0/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T15:15:26.518013017Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-23T15:15:26.520925598Z","level":"INFO","msg":"stream: starting","core version":"0.26.0"}
|
| 3 |
+
{"time":"2026-04-23T15:15:27.723237822Z","level":"INFO","msg":"stream: created new stream","id":"rkikhrj0"}
|
| 4 |
+
{"time":"2026-04-23T15:15:27.723324771Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-23T15:15:27.723497607Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-23T15:15:27.724015618Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-04-23T15:15:27.724022404Z","level":"INFO","msg":"writer: started","stream_id":"rkikhrj0"}
|
| 8 |
+
{"time":"2026-04-23T15:15:28.756080437Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-04-23T15:15:29.912772065Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-23T15:15:43.756582674Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":1,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-04-23T15:15:44.16830069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-23T15:15:58.756317542Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":0,"console_lines":3}
|
| 13 |
+
{"time":"2026-04-23T15:15:59.262190949Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-04-23T15:16:13.756789966Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":0,"console_lines":1}
|
| 15 |
+
{"time":"2026-04-23T15:16:14.163409255Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-04-23T15:16:28.757231889Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":6,"events_lines":2,"console_offset":0,"console_lines":1}
|
| 17 |
+
{"time":"2026-04-23T15:16:29.194789141Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-04-23T15:16:43.756816666Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":0,"console_lines":1}
|
| 19 |
+
{"time":"2026-04-23T15:16:44.216776766Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 20 |
+
{"time":"2026-04-23T15:16:56.771293153Z","level":"INFO","msg":"stream: closing"}
|
| 21 |
+
{"time":"2026-04-23T15:16:57.967692249Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 22 |
+
{"time":"2026-04-23T15:16:57.967865932Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":0,"console_lines":1,"uploaded_len":3}
|
| 23 |
+
{"time":"2026-04-23T15:16:59.044382264Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 24 |
+
{"time":"2026-04-23T15:16:59.044622164Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":3,"console_lines":3,"complete":true,"exit_code":0}
|
| 25 |
+
{"time":"2026-04-23T15:16:59.53195078Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 26 |
+
{"time":"2026-04-23T15:16:59.532109965Z","level":"INFO","msg":"handler: closed"}
|
| 27 |
+
{"time":"2026-04-23T15:16:59.532244558Z","level":"INFO","msg":"sender: closed"}
|
| 28 |
+
{"time":"2026-04-23T15:16:59.532260458Z","level":"INFO","msg":"stream: closed"}
|
wandb/run-20260423_151526-rkikhrj0/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 15:15:26,077 INFO MainThread:1632880 [wandb_setup.py:_flush():81] Current SDK version is 0.26.0
|
| 2 |
+
2026-04-23 15:15:26,077 INFO MainThread:1632880 [wandb_setup.py:_flush():81] Configure stats pid to 1632880
|
| 3 |
+
2026-04-23 15:15:26,077 INFO MainThread:1632880 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 15:15:26,077 INFO MainThread:1632880 [wandb_init.py:setup_run_log_directory():721] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260423_151526-rkikhrj0/logs/debug.log
|
| 5 |
+
2026-04-23 15:15:26,077 INFO MainThread:1632880 [wandb_init.py:setup_run_log_directory():722] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260423_151526-rkikhrj0/logs/debug-internal.log
|
| 6 |
+
2026-04-23 15:15:26,077 INFO MainThread:1632880 [wandb_init.py:init():848] calling init triggers
|
| 7 |
+
2026-04-23 15:15:26,077 INFO MainThread:1632880 [wandb_init.py:init():853] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 8, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top3_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True, 'use_mdlm': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 200, 'lr': 0.0005, 'lr_min': 5e-05, 'warmup_steps': 10000, 'weight_decay': 0.02, 'grad_clip': 1.0, 'adam_betas': [0.9, 0.99], 'adam_eps': 1e-09, 'dtype': 'bf16', 'compile': 'default', 't_eps': 0.001, 'curriculum_t_max_init': 0.3, 'curriculum_steps': 20000, 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_new_owt', 'save_dir': 'outputs/sad_new'}, 'debug': {'debug_mode': True, 'raise_on_nan': True, 'check_inputs': True, 'check_batch_every': 100, 'check_outputs': True, 'check_outputs_every': 100, 'check_loss': True, 'check_grads': True, 'check_params': True, 'check_params_every': 1000, 'use_hooks': False, 'save_on_nan': True, 'print_stats_every': 1000, 'grad_norm_warn_threshold': 100000.0}, '_wandb': {}}
|
| 9 |
+
2026-04-23 15:15:26,077 INFO MainThread:1632880 [wandb_init.py:init():896] starting backend
|
| 10 |
+
2026-04-23 15:15:26,507 INFO MainThread:1632880 [wandb_init.py:init():911] sending inform_init request
|
| 11 |
+
2026-04-23 15:15:26,512 INFO MainThread:1632880 [wandb_init.py:init():919] backend started and connected
|
| 12 |
+
2026-04-23 15:15:26,514 INFO MainThread:1632880 [wandb_init.py:init():989] updated telemetry
|
| 13 |
+
2026-04-23 15:15:26,533 INFO MainThread:1632880 [wandb_init.py:init():1013] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 15:15:28,632 INFO MainThread:1632880 [wandb_init.py:init():1058] starting run threads in backend
|
| 15 |
+
2026-04-23 15:15:28,749 INFO MainThread:1632880 [wandb_run.py:_console_start():2542] atexit reg
|
| 16 |
+
2026-04-23 15:15:28,749 INFO MainThread:1632880 [wandb_run.py:_redirect():2391] redirect: wrap_raw
|
| 17 |
+
2026-04-23 15:15:28,749 INFO MainThread:1632880 [wandb_run.py:_redirect():2460] Wrapping output streams.
|
| 18 |
+
2026-04-23 15:15:28,749 INFO MainThread:1632880 [wandb_run.py:_redirect():2483] Redirects installed.
|
| 19 |
+
2026-04-23 15:15:28,753 INFO MainThread:1632880 [wandb_init.py:init():1098] run started, returning control to user process
|
| 20 |
+
2026-04-23 15:16:56,771 INFO wandb-AsyncioManager-main:1632880 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-23 15:16:56,771 INFO wandb-AsyncioManager-main:1632880 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260423_152554-78ryrosu/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/run-20260423_152554-78ryrosu/logs/debug-core.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-23T15:25:54.50657622Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpkm8n75sf/port-1645301.txt","pid":1645301,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-23T15:25:54.506901443Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1645301}
|
| 3 |
+
{"time":"2026-04-23T15:25:54.506893942Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1645301-1651455-3776851959/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-23T15:25:54.690096586Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-23T15:25:54.698565666Z","level":"INFO","msg":"handleInformInit: received","streamId":"78ryrosu","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-23T15:25:56.069545774Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"78ryrosu","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-23T15:26:01.873853726Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"kxsu5yritu99"}
|
| 8 |
+
{"time":"2026-04-23T17:12:40.760304475Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-23T17:12:40.760381324Z","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2026-04-23T17:12:40.76037156Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-23T17:12:40.760468218Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-23T17:12:40.760469913Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1645301-1651455-3776851959/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-23T17:12:53.885863709Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
wandb/run-20260423_152554-78ryrosu/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/run-20260423_152554-78ryrosu/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-23 15:25:54,473 INFO MainThread:1645301 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-23 15:25:54,473 INFO MainThread:1645301 [wandb_setup.py:_flush():81] Configure stats pid to 1645301
|
| 3 |
+
2026-04-23 15:25:54,473 INFO MainThread:1645301 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-23 15:25:54,473 INFO MainThread:1645301 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260423_152554-78ryrosu/logs/debug.log
|
| 5 |
+
2026-04-23 15:25:54,473 INFO MainThread:1645301 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260423_152554-78ryrosu/logs/debug-internal.log
|
| 6 |
+
2026-04-23 15:25:54,473 INFO MainThread:1645301 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-23 15:25:54,473 INFO MainThread:1645301 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 8, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top3_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True, 'use_mdlm': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0005, 'lr_min': 5e-05, 'warmup_steps': 10000, 'weight_decay': 0.02, 'grad_clip': 1.0, 'adam_betas': [0.9, 0.99], 'adam_eps': 1e-09, 'dtype': 'bf16', 'compile': 'default', 't_eps': 0.001, 'curriculum_t_max_init': 0.3, 'curriculum_steps': 20000, 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_new_owt', 'save_dir': 'outputs/sad_new'}, 'debug': {'debug_mode': True, 'raise_on_nan': True, 'check_inputs': True, 'check_batch_every': 100, 'check_outputs': True, 'check_outputs_every': 100, 'check_loss': True, 'check_grads': True, 'check_params': True, 'check_params_every': 1000, 'use_hooks': False, 'save_on_nan': True, 'print_stats_every': 1000, 'grad_norm_warn_threshold': 100000.0}, '_wandb': {}}
|
| 9 |
+
2026-04-23 15:25:54,473 INFO MainThread:1645301 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-23 15:25:54,690 INFO MainThread:1645301 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-23 15:25:54,695 INFO MainThread:1645301 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-23 15:25:54,696 INFO MainThread:1645301 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-23 15:25:54,703 INFO MainThread:1645301 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-23 15:25:56,715 INFO MainThread:1645301 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-23 15:25:56,833 INFO MainThread:1645301 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-23 15:25:56,833 INFO MainThread:1645301 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-23 15:25:56,833 INFO MainThread:1645301 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-23 15:25:56,833 INFO MainThread:1645301 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-23 15:25:56,837 INFO MainThread:1645301 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-23 17:12:40,760 INFO wandb-AsyncioManager-main:1645301 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-23 17:12:40,760 INFO wandb-AsyncioManager-main:1645301 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260424_060920-vdilsesj/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/run-20260424_060920-vdilsesj/logs/debug-core.log
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-24T06:09:20.638772967Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphpz_ww5w/port-2542249.txt","pid":2542249,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-24T06:09:20.639257965Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2542249}
|
| 3 |
+
{"time":"2026-04-24T06:09:20.639230227Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2542249-2548421-1807225277/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-24T06:09:20.821509374Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-24T06:09:20.830057193Z","level":"INFO","msg":"handleInformInit: received","streamId":"vdilsesj","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-24T06:09:21.669182227Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vdilsesj","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-24T06:09:27.461763166Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"25jp0bc694ap"}
|
| 8 |
+
{"time":"2026-04-26T14:40:47.937906732Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-26T14:40:47.938001158Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 10 |
+
{"time":"2026-04-26T14:40:47.938026974Z","level":"INFO","msg":"server is shutting down"}
|
| 11 |
+
{"time":"2026-04-26T14:40:47.938082023Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-26T14:40:47.93817213Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2542249-2548421-1807225277/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-26T14:43:02.872790092Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-04-26T14:43:02.872851366Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 15 |
+
{"time":"2026-04-26T14:43:02.872884159Z","level":"INFO","msg":"server is closed"}
|
wandb/run-20260424_060920-vdilsesj/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/run-20260424_060920-vdilsesj/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-24 06:09:20,604 INFO MainThread:2542249 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-24 06:09:20,604 INFO MainThread:2542249 [wandb_setup.py:_flush():81] Configure stats pid to 2542249
|
| 3 |
+
2026-04-24 06:09:20,604 INFO MainThread:2542249 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-24 06:09:20,604 INFO MainThread:2542249 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260424_060920-vdilsesj/logs/debug.log
|
| 5 |
+
2026-04-24 06:09:20,605 INFO MainThread:2542249 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260424_060920-vdilsesj/logs/debug-internal.log
|
| 6 |
+
2026-04-24 06:09:20,605 INFO MainThread:2542249 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-24 06:09:20,605 INFO MainThread:2542249 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 32, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top3_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0003, 'lr_min': 3e-05, 'warmup_steps': 2000, 'weight_decay': 0.01, 'grad_clip': 1.0, 'dtype': 'bf16', 'compile': 'default', 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_block_ar', 'save_dir': 'outputs/sad_block_ar'}, '_wandb': {}}
|
| 9 |
+
2026-04-24 06:09:20,605 INFO MainThread:2542249 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-24 06:09:20,821 INFO MainThread:2542249 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-24 06:09:20,827 INFO MainThread:2542249 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-24 06:09:20,829 INFO MainThread:2542249 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-24 06:09:20,835 INFO MainThread:2542249 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-24 06:09:22,329 INFO MainThread:2542249 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-24 06:09:22,452 INFO MainThread:2542249 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-24 06:09:22,452 INFO MainThread:2542249 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-24 06:09:22,452 INFO MainThread:2542249 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-24 06:09:22,452 INFO MainThread:2542249 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-24 06:09:22,456 INFO MainThread:2542249 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-26 14:40:47,938 INFO wandb-AsyncioManager-main:2542249 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-26 14:40:47,939 INFO wandb-AsyncioManager-main:2542249 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260426_173211-4zyjccoe/files/output.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
block-ar training: 0%| | 0/1000000 [00:00<?, ?it/s][rank0]:W0426 17:32:13.210000 1302955 site-packages/torch/_logging/_internal.py:1089] [0/0] Profiler function <class 'torch.autograd.profiler.record_function'> will be ignored
|
| 2 |
+
block-ar training: 0%| | 133/1000000 [01:34<111:01:55, 2.50it/s, leaf=4.2410, lr=2.0e-05]Traceback (most recent call last):
|
| 3 |
+
step= 0 | total=5.6553 | leaf=5.6553 | ancestor=0.0000 | lr=0.00e+00 | t=41.0s
|
| 4 |
+
step= 100 | total=4.3164 | leaf=4.3164 | ancestor=0.0000 | lr=1.50e-05 | t=40.3s
|
| 5 |
+
File "/home/sunhc/diffusion/sad/scripts/train_block_ar.py", line 629, in <module>
|
| 6 |
+
main()
|
| 7 |
+
File "/home/sunhc/diffusion/sad/scripts/train_block_ar.py", line 531, in main
|
| 8 |
+
dist.all_reduce(finite_flag, op=dist.ReduceOp.MIN)
|
| 9 |
+
File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/distributed/c10d_logger.py", line 78, in wrapper
|
| 10 |
+
@functools.wraps(func)
|
| 11 |
+
|
| 12 |
+
KeyboardInterrupt
|
| 13 |
+
[rank0]: Traceback (most recent call last):
|
| 14 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train_block_ar.py", line 629, in <module>
|
| 15 |
+
[rank0]: main()
|
| 16 |
+
[rank0]: File "/home/sunhc/diffusion/sad/scripts/train_block_ar.py", line 531, in main
|
| 17 |
+
[rank0]: dist.all_reduce(finite_flag, op=dist.ReduceOp.MIN)
|
| 18 |
+
[rank0]: File "/home/sunhc/miniconda3/envs/sad/lib/python3.11/site-packages/torch/distributed/c10d_logger.py", line 78, in wrapper
|
| 19 |
+
[rank0]: @functools.wraps(func)
|
| 20 |
+
|
| 21 |
+
[rank0]: KeyboardInterrupt
|
wandb/run-20260426_173211-4zyjccoe/logs/debug-core.log
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-26T17:32:11.346081838Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphy5qwpj_/port-1302955.txt","pid":1302955,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-26T17:32:11.346466446Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1302955}
|
| 3 |
+
{"time":"2026-04-26T17:32:11.34646179Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1302955-1309168-3480502953/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-26T17:32:11.527659035Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-26T17:32:11.532371256Z","level":"INFO","msg":"handleInformInit: received","streamId":"4zyjccoe","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-26T17:32:12.398661732Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"4zyjccoe","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-26T17:32:17.997542984Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"5kign43xyh74"}
|
| 8 |
+
{"time":"2026-04-26T17:33:47.218281054Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-26T17:33:47.218330019Z","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2026-04-26T17:33:47.218325442Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-26T17:33:47.218410917Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 12 |
+
{"time":"2026-04-26T17:33:47.218432794Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1302955-1309168-3480502953/socket","Net":"unix"}}
|
| 13 |
+
{"time":"2026-04-26T17:34:04.467173036Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
wandb/run-20260426_173211-4zyjccoe/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-26T17:32:11.532442324Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-26T17:32:11.533481702Z","level":"INFO","msg":"stream: starting","core version":"0.25.1"}
|
| 3 |
+
{"time":"2026-04-26T17:32:12.398219777Z","level":"INFO","msg":"stream: created new stream","id":"4zyjccoe"}
|
| 4 |
+
{"time":"2026-04-26T17:32:12.398544422Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-26T17:32:12.398649203Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-26T17:32:12.39868661Z","level":"INFO","msg":"writer: started","stream_id":"4zyjccoe"}
|
| 7 |
+
{"time":"2026-04-26T17:32:12.398694065Z","level":"INFO","msg":"sender: started"}
|
| 8 |
+
{"time":"2026-04-26T17:32:12.981495274Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-04-26T17:32:13.709739728Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-26T17:32:27.982392729Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":1}
|
| 11 |
+
{"time":"2026-04-26T17:32:28.356102059Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-26T17:32:42.982587405Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
|
| 13 |
+
{"time":"2026-04-26T17:32:43.357601288Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-04-26T17:32:57.982642232Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":4,"events_lines":2,"console_offset":1,"console_lines":2}
|
| 15 |
+
{"time":"2026-04-26T17:32:58.405937921Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-04-26T17:33:12.982309713Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 17 |
+
{"time":"2026-04-26T17:33:13.385182916Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-04-26T17:33:27.981766459Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 19 |
+
{"time":"2026-04-26T17:33:28.34926984Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 20 |
+
{"time":"2026-04-26T17:33:42.982277446Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 21 |
+
{"time":"2026-04-26T17:33:43.514740481Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 22 |
+
{"time":"2026-04-26T17:33:47.218337017Z","level":"INFO","msg":"stream: closing"}
|
| 23 |
+
{"time":"2026-04-26T17:33:57.982250746Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":1,"console_lines":1}
|
| 24 |
+
{"time":"2026-04-26T17:33:58.382469207Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
wandb/run-20260426_173211-4zyjccoe/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-26 17:32:11,310 INFO MainThread:1302955 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-26 17:32:11,310 INFO MainThread:1302955 [wandb_setup.py:_flush():81] Configure stats pid to 1302955
|
| 3 |
+
2026-04-26 17:32:11,310 INFO MainThread:1302955 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-26 17:32:11,310 INFO MainThread:1302955 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260426_173211-4zyjccoe/logs/debug.log
|
| 5 |
+
2026-04-26 17:32:11,310 INFO MainThread:1302955 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260426_173211-4zyjccoe/logs/debug-internal.log
|
| 6 |
+
2026-04-26 17:32:11,310 INFO MainThread:1302955 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-26 17:32:11,310 INFO MainThread:1302955 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 32, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top2_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 1000000, 'lr': 0.0003, 'lr_min': 3e-05, 'warmup_steps': 2000, 'weight_decay': 0.01, 'grad_clip': 1.0, 'dtype': 'bf16', 'compile': 'default', 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_block_ar_b32_top2', 'save_dir': 'outputs/sad_block_ar_b32_top2'}, '_wandb': {}}
|
| 9 |
+
2026-04-26 17:32:11,310 INFO MainThread:1302955 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-26 17:32:11,527 INFO MainThread:1302955 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-26 17:32:11,531 INFO MainThread:1302955 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-26 17:32:11,532 INFO MainThread:1302955 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-26 17:32:11,536 INFO MainThread:1302955 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-26 17:32:12,874 INFO MainThread:1302955 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-26 17:32:12,976 INFO MainThread:1302955 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-26 17:32:12,976 INFO MainThread:1302955 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-26 17:32:12,977 INFO MainThread:1302955 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-26 17:32:12,977 INFO MainThread:1302955 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-26 17:32:12,979 INFO MainThread:1302955 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-26 17:33:47,218 INFO wandb-AsyncioManager-main:1302955 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-26 17:33:47,218 INFO wandb-AsyncioManager-main:1302955 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260426_173430-jy4xvva6/files/output.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/run-20260426_173430-jy4xvva6/logs/debug-core.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-26T17:34:30.830524558Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmph9j43wdx/port-1311904.txt","pid":1311904,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-26T17:34:30.830904175Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1311904}
|
| 3 |
+
{"time":"2026-04-26T17:34:30.830883546Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1311904-1317892-2603046640/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-26T17:34:31.013549231Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-26T17:34:31.021720952Z","level":"INFO","msg":"handleInformInit: received","streamId":"jy4xvva6","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-26T17:34:31.853680135Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"jy4xvva6","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-26T17:34:37.507422345Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"k2igugrwg03w"}
|
| 8 |
+
{"time":"2026-04-28T17:21:58.456921609Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
wandb/run-20260426_173430-jy4xvva6/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/run-20260426_173430-jy4xvva6/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-26 17:34:30,797 INFO MainThread:1311904 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-26 17:34:30,797 INFO MainThread:1311904 [wandb_setup.py:_flush():81] Configure stats pid to 1311904
|
| 3 |
+
2026-04-26 17:34:30,797 INFO MainThread:1311904 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-26 17:34:30,797 INFO MainThread:1311904 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260426_173430-jy4xvva6/logs/debug.log
|
| 5 |
+
2026-04-26 17:34:30,797 INFO MainThread:1311904 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260426_173430-jy4xvva6/logs/debug-internal.log
|
| 6 |
+
2026-04-26 17:34:30,797 INFO MainThread:1311904 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-26 17:34:30,797 INFO MainThread:1311904 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 32, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top2_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0003, 'lr_min': 3e-05, 'warmup_steps': 2000, 'weight_decay': 0.01, 'grad_clip': 1.0, 'dtype': 'bf16', 'compile': 'default', 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_block_ar_b32_top2', 'save_dir': 'outputs/sad_block_ar_b32_top2'}, '_wandb': {}}
|
| 9 |
+
2026-04-26 17:34:30,797 INFO MainThread:1311904 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-26 17:34:31,013 INFO MainThread:1311904 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-26 17:34:31,017 INFO MainThread:1311904 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-26 17:34:31,019 INFO MainThread:1311904 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-26 17:34:31,024 INFO MainThread:1311904 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-26 17:34:32,354 INFO MainThread:1311904 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-26 17:34:32,481 INFO MainThread:1311904 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-26 17:34:32,481 INFO MainThread:1311904 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-26 17:34:32,481 INFO MainThread:1311904 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-26 17:34:32,481 INFO MainThread:1311904 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-26 17:34:32,484 INFO MainThread:1311904 [wandb_init.py:init():1082] run started, returning control to user process
|
wandb/run-20260428_172322-yizklreh/files/output.log
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
block-ar training: 0%| | 0/500000 [00:00<?, ?it/s][rank0]:W0428 17:23:24.499000 3904379 site-packages/torch/_logging/_internal.py:1089] [0/0] Profiler function <class 'torch.autograd.profiler.record_function'> will be ignored
|
| 2 |
+
block-ar training: 0%| | 1479/500000 [10:38<56:22:21, 2.46it/s, leaf=2.6111, lr=2.2e-04]
|
| 3 |
+
step= 0 | total=5.6694 | leaf=5.6694 | ancestor=0.0000 | lr=0.00e+00 | t=39.0s
|
| 4 |
+
step= 100 | total=5.1955 | leaf=5.1955 | ancestor=0.0000 | lr=1.50e-05 | t=40.7s
|
| 5 |
+
step= 200 | total=3.9406 | leaf=3.9406 | ancestor=0.0000 | lr=3.00e-05 | t=40.4s
|
| 6 |
+
step= 300 | total=3.4730 | leaf=3.4730 | ancestor=0.0000 | lr=4.50e-05 | t=40.4s
|
| 7 |
+
step= 400 | total=3.3458 | leaf=3.3458 | ancestor=0.0000 | lr=6.00e-05 | t=40.5s
|
| 8 |
+
step= 500 | total=3.4864 | leaf=3.4864 | ancestor=0.0000 | lr=7.50e-05 | t=40.5s
|
| 9 |
+
step= 600 | total=3.1515 | leaf=3.1515 | ancestor=0.0000 | lr=9.00e-05 | t=40.9s
|
| 10 |
+
step= 700 | total=3.2125 | leaf=3.2125 | ancestor=0.0000 | lr=1.05e-04 | t=40.6s
|
| 11 |
+
step= 800 | total=3.4113 | leaf=3.4113 | ancestor=0.0000 | lr=1.20e-04 | t=40.6s
|
| 12 |
+
step= 900 | total=3.0316 | leaf=3.0316 | ancestor=0.0000 | lr=1.35e-04 | t=40.6s
|
| 13 |
+
step= 1000 | total=2.8569 | leaf=2.8569 | ancestor=0.0000 | lr=1.50e-04 | t=40.6s
|
| 14 |
+
step= 1100 | total=2.3887 | leaf=2.3887 | ancestor=0.0000 | lr=1.65e-04 | t=40.6s
|
| 15 |
+
step= 1200 | total=2.1182 | leaf=2.1182 | ancestor=0.0000 | lr=1.80e-04 | t=40.6s
|
| 16 |
+
step= 1300 | total=2.6883 | leaf=2.6883 | ancestor=0.0000 | lr=1.95e-04 | t=40.6s
|
| 17 |
+
step= 1400 | total=2.5370 | leaf=2.5370 | ancestor=0.0000 | lr=2.10e-04 | t=40.6s
|
wandb/run-20260428_172322-yizklreh/logs/debug-core.log
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-28T17:23:22.359551076Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpv7yj5csu/port-3904379.txt","pid":3904379,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-28T17:23:22.360003461Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3904379}
|
| 3 |
+
{"time":"2026-04-28T17:23:22.35998224Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3904379-3911032-373446568/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-28T17:23:22.544617382Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-28T17:23:22.55066584Z","level":"INFO","msg":"handleInformInit: received","streamId":"yizklreh","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-28T17:23:23.396958854Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"yizklreh","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-28T17:23:29.308441334Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"6pij4y0885ks"}
|
| 8 |
+
{"time":"2026-04-28T17:34:03.419064265Z","level":"INFO","msg":"server: parent process exited, terminating service process"}
|
wandb/run-20260428_172322-yizklreh/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-28T17:23:22.550745062Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-28T17:23:22.551862359Z","level":"INFO","msg":"stream: starting","core version":"0.25.1"}
|
| 3 |
+
{"time":"2026-04-28T17:23:23.396671918Z","level":"INFO","msg":"stream: created new stream","id":"yizklreh"}
|
| 4 |
+
{"time":"2026-04-28T17:23:23.39678834Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-28T17:23:23.396938974Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-28T17:23:23.396987841Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-04-28T17:23:23.396980711Z","level":"INFO","msg":"writer: started","stream_id":"yizklreh"}
|
| 8 |
+
{"time":"2026-04-28T17:23:24.284090891Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-04-28T17:23:25.13561181Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-28T17:23:39.284318523Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":1,"uploaded_len":2}
|
| 11 |
+
{"time":"2026-04-28T17:23:39.780062003Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 12 |
+
{"time":"2026-04-28T17:23:54.284730989Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
|
| 13 |
+
{"time":"2026-04-28T17:23:54.773681562Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-04-28T17:24:09.285189458Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":4,"events_lines":2,"console_offset":1,"console_lines":2}
|
| 15 |
+
{"time":"2026-04-28T17:24:09.857720429Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 16 |
+
{"time":"2026-04-28T17:24:24.284374602Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":6,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 17 |
+
{"time":"2026-04-28T17:24:24.838725755Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 18 |
+
{"time":"2026-04-28T17:24:39.28461773Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":8,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 19 |
+
{"time":"2026-04-28T17:24:39.83214335Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 20 |
+
{"time":"2026-04-28T17:24:54.28438644Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 21 |
+
{"time":"2026-04-28T17:24:54.857101792Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 22 |
+
{"time":"2026-04-28T17:25:09.284434368Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":12,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 23 |
+
{"time":"2026-04-28T17:25:09.725317325Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 24 |
+
{"time":"2026-04-28T17:25:24.284533565Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 25 |
+
{"time":"2026-04-28T17:25:25.3026319Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 26 |
+
{"time":"2026-04-28T17:25:39.285117147Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":16,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 27 |
+
{"time":"2026-04-28T17:25:39.827077055Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 28 |
+
{"time":"2026-04-28T17:25:54.284403738Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":18,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 29 |
+
{"time":"2026-04-28T17:25:54.747572012Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 30 |
+
{"time":"2026-04-28T17:26:09.284875555Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 31 |
+
{"time":"2026-04-28T17:26:09.800064449Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 32 |
+
{"time":"2026-04-28T17:26:24.284286479Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":22,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 33 |
+
{"time":"2026-04-28T17:26:24.769286835Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 34 |
+
{"time":"2026-04-28T17:26:39.284669979Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":24,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 35 |
+
{"time":"2026-04-28T17:26:39.792752069Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 36 |
+
{"time":"2026-04-28T17:26:54.284854701Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":4,"history_lines":1,"events_offset":26,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 37 |
+
{"time":"2026-04-28T17:26:54.928463612Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 38 |
+
{"time":"2026-04-28T17:27:09.284967409Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":28,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 39 |
+
{"time":"2026-04-28T17:27:09.71103086Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 40 |
+
{"time":"2026-04-28T17:27:24.284815292Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":30,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 41 |
+
{"time":"2026-04-28T17:27:24.775747748Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 42 |
+
{"time":"2026-04-28T17:27:39.284294408Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":32,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 43 |
+
{"time":"2026-04-28T17:27:39.709281084Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 44 |
+
{"time":"2026-04-28T17:27:54.284789177Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 45 |
+
{"time":"2026-04-28T17:27:54.798990731Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 46 |
+
{"time":"2026-04-28T17:28:09.284429021Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":36,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 47 |
+
{"time":"2026-04-28T17:28:09.806607089Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 48 |
+
{"time":"2026-04-28T17:28:24.28514054Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":38,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 49 |
+
{"time":"2026-04-28T17:28:24.708701791Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 50 |
+
{"time":"2026-04-28T17:28:39.285243315Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":40,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 51 |
+
{"time":"2026-04-28T17:28:39.970711072Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 52 |
+
{"time":"2026-04-28T17:28:54.284462797Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 53 |
+
{"time":"2026-04-28T17:28:54.787408118Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 54 |
+
{"time":"2026-04-28T17:29:09.285011067Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":44,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 55 |
+
{"time":"2026-04-28T17:29:09.721525133Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 56 |
+
{"time":"2026-04-28T17:29:24.284693039Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":46,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 57 |
+
{"time":"2026-04-28T17:29:24.787092266Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 58 |
+
{"time":"2026-04-28T17:29:39.284927454Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":8,"history_lines":1,"events_offset":48,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 59 |
+
{"time":"2026-04-28T17:29:39.811127164Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 60 |
+
{"time":"2026-04-28T17:29:54.284689942Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":50,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 61 |
+
{"time":"2026-04-28T17:29:54.785248425Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 62 |
+
{"time":"2026-04-28T17:30:09.284432085Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":1,"events_offset":52,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 63 |
+
{"time":"2026-04-28T17:30:09.813709058Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 64 |
+
{"time":"2026-04-28T17:30:24.285111327Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":54,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 65 |
+
{"time":"2026-04-28T17:30:24.797624597Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 66 |
+
{"time":"2026-04-28T17:30:39.284656314Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":56,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 67 |
+
{"time":"2026-04-28T17:30:40.474529615Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 68 |
+
{"time":"2026-04-28T17:30:54.284770911Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":10,"history_lines":1,"events_offset":58,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 69 |
+
{"time":"2026-04-28T17:30:55.247412614Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 70 |
+
{"time":"2026-04-28T17:31:09.284418002Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":60,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 71 |
+
{"time":"2026-04-28T17:31:09.778318746Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 72 |
+
{"time":"2026-04-28T17:31:24.285124033Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":62,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 73 |
+
{"time":"2026-04-28T17:31:24.727765575Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 74 |
+
{"time":"2026-04-28T17:31:39.284608407Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 75 |
+
{"time":"2026-04-28T17:31:39.76063836Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 76 |
+
{"time":"2026-04-28T17:31:54.284282945Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":66,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 77 |
+
{"time":"2026-04-28T17:31:54.706924357Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 78 |
+
{"time":"2026-04-28T17:32:09.285044426Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":68,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 79 |
+
{"time":"2026-04-28T17:32:09.814569181Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 80 |
+
{"time":"2026-04-28T17:32:24.285168099Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":12,"history_lines":1,"events_offset":70,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 81 |
+
{"time":"2026-04-28T17:32:24.755132373Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 82 |
+
{"time":"2026-04-28T17:32:39.285125212Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":72,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 83 |
+
{"time":"2026-04-28T17:32:39.811388225Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 84 |
+
{"time":"2026-04-28T17:32:54.285195111Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":74,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 85 |
+
{"time":"2026-04-28T17:32:54.834227138Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 86 |
+
{"time":"2026-04-28T17:33:09.284185364Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":76,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 87 |
+
{"time":"2026-04-28T17:33:09.735427957Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 88 |
+
{"time":"2026-04-28T17:33:24.284666869Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":78,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 89 |
+
{"time":"2026-04-28T17:33:24.730682398Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 90 |
+
{"time":"2026-04-28T17:33:39.284615575Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":80,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 91 |
+
{"time":"2026-04-28T17:33:39.690751122Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 92 |
+
{"time":"2026-04-28T17:33:54.284830137Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":82,"events_lines":2,"console_offset":1,"console_lines":1}
|
| 93 |
+
{"time":"2026-04-28T17:33:54.753650135Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
wandb/run-20260428_172322-yizklreh/logs/debug.log
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-28 17:23:22,328 INFO MainThread:3904379 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-28 17:23:22,328 INFO MainThread:3904379 [wandb_setup.py:_flush():81] Configure stats pid to 3904379
|
| 3 |
+
2026-04-28 17:23:22,328 INFO MainThread:3904379 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-28 17:23:22,328 INFO MainThread:3904379 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260428_172322-yizklreh/logs/debug.log
|
| 5 |
+
2026-04-28 17:23:22,328 INFO MainThread:3904379 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260428_172322-yizklreh/logs/debug-internal.log
|
| 6 |
+
2026-04-28 17:23:22,328 INFO MainThread:3904379 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-28 17:23:22,328 INFO MainThread:3904379 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 32, 'dropout': 0.0, 'num_levels': 4, 'level_sizes': [50258, 128, 32, 8]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128-32-8_leafdirect_top3-2-1_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128-32-8.pt'}, 'hierarchy': {'prototype_cache': 'data/hierarchy_prototypes_50257-128-32-8.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0003, 'lr_min': 3e-05, 'warmup_steps': 2000, 'weight_decay': 0.01, 'grad_clip': 1.0, 'dtype': 'bf16', 'compile': 'default', 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_block_ar_b32_h3_mixed', 'save_dir': 'outputs/sad_block_ar_b32_h3_mixed'}, '_wandb': {}}
|
| 9 |
+
2026-04-28 17:23:22,328 INFO MainThread:3904379 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-28 17:23:22,545 INFO MainThread:3904379 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-28 17:23:22,549 INFO MainThread:3904379 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-28 17:23:22,551 INFO MainThread:3904379 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-28 17:23:22,557 INFO MainThread:3904379 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-28 17:23:24,159 INFO MainThread:3904379 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-28 17:23:24,279 INFO MainThread:3904379 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-28 17:23:24,279 INFO MainThread:3904379 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-28 17:23:24,279 INFO MainThread:3904379 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-28 17:23:24,279 INFO MainThread:3904379 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-28 17:23:24,281 INFO MainThread:3904379 [wandb_init.py:init():1082] run started, returning control to user process
|
wandb/run-20260428_173540-bbc05m6k/files/output.log
ADDED
|
@@ -0,0 +1,930 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
block-ar training: 82%|███████████████████████████████████████████████████████▊ | 410001/500000 [00:00<?, ?it/s][rank0]:W0428 17:35:42.453000 4021893 site-packages/torch/_logging/_internal.py:1089] [0/0] Profiler function <class 'torch.autograd.profiler.record_function'> will be ignored
|
| 2 |
+
block-ar training: 83%|████████████████████▋ | 413201/500000 [22:07<9:43:27, 2.48it/s, leaf=1.2141, lr=5.0e-05]
|
| 3 |
+
step=410100 | total=1.1464 | leaf=1.1464 | ancestor=0.0000 | lr=5.11e-05 | t=79.2s
|
| 4 |
+
step=410200 | total=1.1329 | leaf=1.1329 | ancestor=0.0000 | lr=5.11e-05 | t=40.2s
|
| 5 |
+
step=410300 | total=1.4373 | leaf=1.4373 | ancestor=0.0000 | lr=5.10e-05 | t=40.2s
|
| 6 |
+
step=410400 | total=1.1019 | leaf=1.1019 | ancestor=0.0000 | lr=5.10e-05 | t=40.3s
|
| 7 |
+
step=410500 | total=1.1541 | leaf=1.1541 | ancestor=0.0000 | lr=5.10e-05 | t=40.3s
|
| 8 |
+
step=410600 | total=1.5538 | leaf=1.5538 | ancestor=0.0000 | lr=5.09e-05 | t=40.3s
|
| 9 |
+
step=410700 | total=1.2646 | leaf=1.2646 | ancestor=0.0000 | lr=5.09e-05 | t=40.3s
|
| 10 |
+
step=410800 | total=1.3426 | leaf=1.3426 | ancestor=0.0000 | lr=5.08e-05 | t=40.2s
|
| 11 |
+
step=410900 | total=1.1510 | leaf=1.1510 | ancestor=0.0000 | lr=5.08e-05 | t=40.3s
|
| 12 |
+
step=411000 | total=0.8522 | leaf=0.8522 | ancestor=0.0000 | lr=5.07e-05 | t=40.3s
|
| 13 |
+
step=411100 | total=0.8612 | leaf=0.8612 | ancestor=0.0000 | lr=5.07e-05 | t=40.3s
|
| 14 |
+
step=411200 | total=1.1683 | leaf=1.1683 | ancestor=0.0000 | lr=5.06e-05 | t=40.3s
|
| 15 |
+
step=411300 | total=1.3415 | leaf=1.3415 | ancestor=0.0000 | lr=5.06e-05 | t=40.3s
|
| 16 |
+
step=411400 | total=1.1345 | leaf=1.1345 | ancestor=0.0000 | lr=5.05e-05 | t=40.3s
|
| 17 |
+
step=411500 | total=1.3578 | leaf=1.3578 | ancestor=0.0000 | lr=5.05e-05 | t=40.3s
|
| 18 |
+
step=411600 | total=1.2499 | leaf=1.2499 | ancestor=0.0000 | lr=5.05e-05 | t=40.3s
|
| 19 |
+
step=411700 | total=1.3465 | leaf=1.3465 | ancestor=0.0000 | lr=5.04e-05 | t=40.3s
|
| 20 |
+
step=411800 | total=1.1922 | leaf=1.1922 | ancestor=0.0000 | lr=5.04e-05 | t=40.3s
|
| 21 |
+
step=411900 | total=1.4609 | leaf=1.4609 | ancestor=0.0000 | lr=5.03e-05 | t=40.3s
|
| 22 |
+
step=412000 | total=1.3178 | leaf=1.3178 | ancestor=0.0000 | lr=5.03e-05 | t=40.2s
|
| 23 |
+
step=412100 | total=0.8633 | leaf=0.8633 | ancestor=0.0000 | lr=5.02e-05 | t=40.3s
|
| 24 |
+
step=412200 | total=1.4572 | leaf=1.4572 | ancestor=0.0000 | lr=5.02e-05 | t=40.3s
|
| 25 |
+
step=412300 | total=1.3776 | leaf=1.3776 | ancestor=0.0000 | lr=5.01e-05 | t=40.3s
|
| 26 |
+
step=412400 | total=1.0817 | leaf=1.0817 | ancestor=0.0000 | lr=5.01e-05 | t=40.3s
|
| 27 |
+
step=412500 | total=0.9286 | leaf=0.9286 | ancestor=0.0000 | lr=5.00e-05 | t=40.3s
|
| 28 |
+
step=412600 | total=1.3610 | leaf=1.3610 | ancestor=0.0000 | lr=5.00e-05 | t=40.3s
|
| 29 |
+
step=412700 | total=0.9004 | leaf=0.9004 | ancestor=0.0000 | lr=5.00e-05 | t=40.3s
|
| 30 |
+
step=412800 | total=1.2368 | leaf=1.2368 | ancestor=0.0000 | lr=4.99e-05 | t=40.3s
|
| 31 |
+
step=412900 | total=1.3347 | leaf=1.3347 | ancestor=0.0000 | lr=4.99e-05 | t=40.3s
|
| 32 |
+
step=413000 | total=1.4344 | leaf=1.4344 | ancestor=0.0000 | lr=4.98e-05 | t=40.3s
|
| 33 |
+
step=413100 | total=1.0604 | leaf=1.0604 | ancestor=0.0000 | lr=4.98e-05 | t=40.2s
|
| 34 |
+
step=413200 | total=1.2141 | leaf=1.2141 | ancestor=0.0000 | lr=4.97e-05 | t=40.3s
|
| 35 |
+
step=413300 | total=1.0198 | leaf=1.0198 | ancestor=0.0000 | lr=4.97e-05 | t=40.3s
|
| 36 |
+
step=413400 | total=1.3908 | leaf=1.3908 | ancestor=0.0000 | lr=4.96e-05 | t=40.3s
|
| 37 |
+
step=413500 | total=1.1531 | leaf=1.1531 | ancestor=0.0000 | lr=4.96e-05 | t=40.2s
|
| 38 |
+
step=413600 | total=1.2622 | leaf=1.2622 | ancestor=0.0000 | lr=4.96e-05 | t=40.3s
|
| 39 |
+
step=413700 | total=1.2549 | leaf=1.2549 | ancestor=0.0000 | lr=4.95e-05 | t=40.3s
|
| 40 |
+
step=413800 | total=0.9237 | leaf=0.9237 | ancestor=0.0000 | lr=4.95e-05 | t=40.3s
|
| 41 |
+
step=413900 | total=1.0985 | leaf=1.0985 | ancestor=0.0000 | lr=4.94e-05 | t=40.2s
|
| 42 |
+
step=414000 | total=1.2676 | leaf=1.2676 | ancestor=0.0000 | lr=4.94e-05 | t=40.3s
|
| 43 |
+
step=414100 | total=1.1844 | leaf=1.1844 | ancestor=0.0000 | lr=4.93e-05 | t=40.3s
|
| 44 |
+
step=414200 | total=1.1527 | leaf=1.1527 | ancestor=0.0000 | lr=4.93e-05 | t=40.3s
|
| 45 |
+
step=414300 | total=1.1840 | leaf=1.1840 | ancestor=0.0000 | lr=4.93e-05 | t=40.3s
|
| 46 |
+
step=414400 | total=0.9929 | leaf=0.9929 | ancestor=0.0000 | lr=4.92e-05 | t=40.3s
|
| 47 |
+
step=414500 | total=1.2328 | leaf=1.2328 | ancestor=0.0000 | lr=4.92e-05 | t=40.3s
|
| 48 |
+
step=414600 | total=1.2141 | leaf=1.2141 | ancestor=0.0000 | lr=4.91e-05 | t=40.3s
|
| 49 |
+
step=414700 | total=1.3216 | leaf=1.3216 | ancestor=0.0000 | lr=4.91e-05 | t=40.3s
|
| 50 |
+
step=414800 | total=1.0138 | leaf=1.0138 | ancestor=0.0000 | lr=4.90e-05 | t=40.2s
|
| 51 |
+
step=414900 | total=1.3938 | leaf=1.3938 | ancestor=0.0000 | lr=4.90e-05 | t=40.2s
|
| 52 |
+
step=415000 | total=1.3352 | leaf=1.3352 | ancestor=0.0000 | lr=4.89e-05 | t=40.3s
|
| 53 |
+
torch._dynamo.utils.warn_once(msg)
|
| 54 |
+
VAL | loss_leaf=1.2499 | loss_ancestor=0.0000 | loss_total=1.2499
|
| 55 |
+
block-ar training: 84%|█████████████████████████████▎ | 418201/500000 [56:02<9:08:48, 2.48it/s, leaf=1.0307, lr=4.8e-05]
|
| 56 |
+
step=415100 | total=1.1143 | leaf=1.1143 | ancestor=0.0000 | lr=4.89e-05 | t=61.3s
|
| 57 |
+
step=415200 | total=1.5278 | leaf=1.5278 | ancestor=0.0000 | lr=4.89e-05 | t=40.3s
|
| 58 |
+
step=415300 | total=1.4997 | leaf=1.4997 | ancestor=0.0000 | lr=4.88e-05 | t=40.3s
|
| 59 |
+
step=415400 | total=0.9872 | leaf=0.9872 | ancestor=0.0000 | lr=4.88e-05 | t=40.3s
|
| 60 |
+
step=415500 | total=1.4456 | leaf=1.4456 | ancestor=0.0000 | lr=4.87e-05 | t=40.3s
|
| 61 |
+
step=415600 | total=1.5232 | leaf=1.5232 | ancestor=0.0000 | lr=4.87e-05 | t=40.3s
|
| 62 |
+
step=415700 | total=1.3019 | leaf=1.3019 | ancestor=0.0000 | lr=4.86e-05 | t=40.3s
|
| 63 |
+
step=415800 | total=1.1687 | leaf=1.1687 | ancestor=0.0000 | lr=4.86e-05 | t=40.3s
|
| 64 |
+
step=415900 | total=1.2894 | leaf=1.2894 | ancestor=0.0000 | lr=4.86e-05 | t=40.3s
|
| 65 |
+
step=416000 | total=1.4277 | leaf=1.4277 | ancestor=0.0000 | lr=4.85e-05 | t=40.3s
|
| 66 |
+
step=416100 | total=1.4393 | leaf=1.4393 | ancestor=0.0000 | lr=4.85e-05 | t=40.3s
|
| 67 |
+
step=416200 | total=1.3952 | leaf=1.3952 | ancestor=0.0000 | lr=4.84e-05 | t=40.3s
|
| 68 |
+
step=416300 | total=1.0397 | leaf=1.0397 | ancestor=0.0000 | lr=4.84e-05 | t=40.3s
|
| 69 |
+
step=416400 | total=1.1752 | leaf=1.1752 | ancestor=0.0000 | lr=4.83e-05 | t=40.3s
|
| 70 |
+
step=416500 | total=1.2791 | leaf=1.2791 | ancestor=0.0000 | lr=4.83e-05 | t=40.3s
|
| 71 |
+
step=416600 | total=1.3010 | leaf=1.3010 | ancestor=0.0000 | lr=4.83e-05 | t=40.3s
|
| 72 |
+
step=416700 | total=0.9498 | leaf=0.9498 | ancestor=0.0000 | lr=4.82e-05 | t=40.3s
|
| 73 |
+
step=416800 | total=1.3360 | leaf=1.3360 | ancestor=0.0000 | lr=4.82e-05 | t=40.3s
|
| 74 |
+
step=416900 | total=1.1022 | leaf=1.1022 | ancestor=0.0000 | lr=4.81e-05 | t=40.3s
|
| 75 |
+
step=417000 | total=1.4702 | leaf=1.4702 | ancestor=0.0000 | lr=4.81e-05 | t=40.3s
|
| 76 |
+
step=417100 | total=1.1916 | leaf=1.1916 | ancestor=0.0000 | lr=4.80e-05 | t=40.3s
|
| 77 |
+
step=417200 | total=1.4279 | leaf=1.4279 | ancestor=0.0000 | lr=4.80e-05 | t=40.3s
|
| 78 |
+
step=417300 | total=0.9164 | leaf=0.9164 | ancestor=0.0000 | lr=4.80e-05 | t=40.3s
|
| 79 |
+
step=417400 | total=1.0335 | leaf=1.0335 | ancestor=0.0000 | lr=4.79e-05 | t=40.3s
|
| 80 |
+
step=417500 | total=0.9983 | leaf=0.9983 | ancestor=0.0000 | lr=4.79e-05 | t=40.3s
|
| 81 |
+
step=417600 | total=1.2763 | leaf=1.2763 | ancestor=0.0000 | lr=4.78e-05 | t=40.3s
|
| 82 |
+
step=417700 | total=1.0458 | leaf=1.0458 | ancestor=0.0000 | lr=4.78e-05 | t=40.3s
|
| 83 |
+
step=417800 | total=1.4711 | leaf=1.4711 | ancestor=0.0000 | lr=4.77e-05 | t=40.3s
|
| 84 |
+
step=417900 | total=1.1983 | leaf=1.1983 | ancestor=0.0000 | lr=4.77e-05 | t=40.3s
|
| 85 |
+
step=418000 | total=1.3380 | leaf=1.3380 | ancestor=0.0000 | lr=4.77e-05 | t=40.3s
|
| 86 |
+
step=418100 | total=0.9604 | leaf=0.9604 | ancestor=0.0000 | lr=4.76e-05 | t=40.3s
|
| 87 |
+
step=418200 | total=1.0307 | leaf=1.0307 | ancestor=0.0000 | lr=4.76e-05 | t=40.3s
|
| 88 |
+
step=418300 | total=1.3033 | leaf=1.3033 | ancestor=0.0000 | lr=4.75e-05 | t=40.3s
|
| 89 |
+
step=418400 | total=1.3923 | leaf=1.3923 | ancestor=0.0000 | lr=4.75e-05 | t=40.3s
|
| 90 |
+
step=418500 | total=1.4541 | leaf=1.4541 | ancestor=0.0000 | lr=4.75e-05 | t=40.3s
|
| 91 |
+
step=418600 | total=1.4099 | leaf=1.4099 | ancestor=0.0000 | lr=4.74e-05 | t=40.3s
|
| 92 |
+
step=418700 | total=1.2497 | leaf=1.2497 | ancestor=0.0000 | lr=4.74e-05 | t=40.3s
|
| 93 |
+
step=418800 | total=1.1552 | leaf=1.1552 | ancestor=0.0000 | lr=4.73e-05 | t=40.3s
|
| 94 |
+
step=418900 | total=1.2614 | leaf=1.2614 | ancestor=0.0000 | lr=4.73e-05 | t=40.3s
|
| 95 |
+
step=419000 | total=1.3148 | leaf=1.3148 | ancestor=0.0000 | lr=4.72e-05 | t=40.3s
|
| 96 |
+
step=419100 | total=1.2678 | leaf=1.2678 | ancestor=0.0000 | lr=4.72e-05 | t=41.1s
|
| 97 |
+
step=419200 | total=1.4704 | leaf=1.4704 | ancestor=0.0000 | lr=4.72e-05 | t=40.3s
|
| 98 |
+
step=419300 | total=1.2114 | leaf=1.2114 | ancestor=0.0000 | lr=4.71e-05 | t=40.3s
|
| 99 |
+
step=419400 | total=1.3642 | leaf=1.3642 | ancestor=0.0000 | lr=4.71e-05 | t=40.3s
|
| 100 |
+
step=419500 | total=0.9905 | leaf=0.9905 | ancestor=0.0000 | lr=4.70e-05 | t=40.2s
|
| 101 |
+
step=419600 | total=1.2778 | leaf=1.2778 | ancestor=0.0000 | lr=4.70e-05 | t=40.3s
|
| 102 |
+
step=419700 | total=0.9737 | leaf=0.9737 | ancestor=0.0000 | lr=4.70e-05 | t=40.3s
|
| 103 |
+
step=419800 | total=1.0720 | leaf=1.0720 | ancestor=0.0000 | lr=4.69e-05 | t=40.3s
|
| 104 |
+
step=419900 | total=1.3633 | leaf=1.3633 | ancestor=0.0000 | lr=4.69e-05 | t=40.3s
|
| 105 |
+
step=420000 | total=1.4393 | leaf=1.4393 | ancestor=0.0000 | lr=4.68e-05 | t=40.3s
|
| 106 |
+
VAL | loss_leaf=1.2370 | loss_ancestor=0.0000 | loss_total=1.2370
|
| 107 |
+
Saved checkpoint: outputs/sad_block_ar_b32_top2/ckpt_420000.pt
|
| 108 |
+
step=420100 | total=1.2220 | leaf=1.2220 | ancestor=0.0000 | lr=4.68e-05 | t=56.5s
|
| 109 |
+
step=420200 | total=0.9696 | leaf=0.9696 | ancestor=0.0000 | lr=4.67e-05 | t=40.3s
|
| 110 |
+
step=420300 | total=1.2839 | leaf=1.2839 | ancestor=0.0000 | lr=4.67e-05 | t=40.3s
|
| 111 |
+
step=420400 | total=1.3221 | leaf=1.3221 | ancestor=0.0000 | lr=4.67e-05 | t=40.3s
|
| 112 |
+
step=420500 | total=1.3053 | leaf=1.3053 | ancestor=0.0000 | lr=4.66e-05 | t=40.3s
|
| 113 |
+
step=420600 | total=0.9545 | leaf=0.9545 | ancestor=0.0000 | lr=4.66e-05 | t=40.3s
|
| 114 |
+
step=420700 | total=1.2236 | leaf=1.2236 | ancestor=0.0000 | lr=4.65e-05 | t=40.3s
|
| 115 |
+
step=420800 | total=1.2155 | leaf=1.2155 | ancestor=0.0000 | lr=4.65e-05 | t=40.3s
|
| 116 |
+
step=420900 | total=1.1734 | leaf=1.1734 | ancestor=0.0000 | lr=4.65e-05 | t=40.3s
|
| 117 |
+
step=421000 | total=1.1965 | leaf=1.1965 | ancestor=0.0000 | lr=4.64e-05 | t=40.3s
|
| 118 |
+
step=421100 | total=1.1997 | leaf=1.1997 | ancestor=0.0000 | lr=4.64e-05 | t=40.3s
|
| 119 |
+
step=421200 | total=1.1197 | leaf=1.1197 | ancestor=0.0000 | lr=4.63e-05 | t=40.3s
|
| 120 |
+
step=421300 | total=1.0951 | leaf=1.0951 | ancestor=0.0000 | lr=4.63e-05 | t=40.3s
|
| 121 |
+
step=421400 | total=1.1336 | leaf=1.1336 | ancestor=0.0000 | lr=4.63e-05 | t=40.3s
|
| 122 |
+
step=421500 | total=1.0805 | leaf=1.0805 | ancestor=0.0000 | lr=4.62e-05 | t=40.2s
|
| 123 |
+
step=421600 | total=1.5962 | leaf=1.5962 | ancestor=0.0000 | lr=4.62e-05 | t=40.3s
|
| 124 |
+
step=421700 | total=0.9486 | leaf=0.9486 | ancestor=0.0000 | lr=4.61e-05 | t=40.3s
|
| 125 |
+
step=421800 | total=1.1652 | leaf=1.1652 | ancestor=0.0000 | lr=4.61e-05 | t=40.3s
|
| 126 |
+
step=421900 | total=1.5601 | leaf=1.5601 | ancestor=0.0000 | lr=4.61e-05 | t=40.3s
|
| 127 |
+
step=422000 | total=1.4331 | leaf=1.4331 | ancestor=0.0000 | lr=4.60e-05 | t=40.3s
|
| 128 |
+
step=422100 | total=1.3582 | leaf=1.3582 | ancestor=0.0000 | lr=4.60e-05 | t=40.3s
|
| 129 |
+
step=422200 | total=1.0290 | leaf=1.0290 | ancestor=0.0000 | lr=4.59e-05 | t=40.3s
|
| 130 |
+
step=422300 | total=1.1279 | leaf=1.1279 | ancestor=0.0000 | lr=4.59e-05 | t=40.3s
|
| 131 |
+
step=422400 | total=1.5949 | leaf=1.5949 | ancestor=0.0000 | lr=4.59e-05 | t=40.3s
|
| 132 |
+
step=422500 | total=1.5604 | leaf=1.5604 | ancestor=0.0000 | lr=4.58e-05 | t=40.3s
|
| 133 |
+
step=422600 | total=1.4545 | leaf=1.4545 | ancestor=0.0000 | lr=4.58e-05 | t=40.3s
|
| 134 |
+
step=422700 | total=1.1796 | leaf=1.1796 | ancestor=0.0000 | lr=4.57e-05 | t=40.3s
|
| 135 |
+
step=422800 | total=1.4778 | leaf=1.4778 | ancestor=0.0000 | lr=4.57e-05 | t=40.3s
|
| 136 |
+
step=422900 | total=1.0388 | leaf=1.0388 | ancestor=0.0000 | lr=4.57e-05 | t=40.3s
|
| 137 |
+
step=423000 | total=1.1423 | leaf=1.1423 | ancestor=0.0000 | lr=4.56e-05 | t=40.3s
|
| 138 |
+
step=423100 | total=1.2991 | leaf=1.2991 | ancestor=0.0000 | lr=4.56e-05 | t=40.3s
|
| 139 |
+
step=423200 | total=1.0551 | leaf=1.0551 | ancestor=0.0000 | lr=4.55e-05 | t=40.3s
|
| 140 |
+
step=423300 | total=1.5523 | leaf=1.5523 | ancestor=0.0000 | lr=4.55e-05 | t=40.3s
|
| 141 |
+
step=423400 | total=1.2208 | leaf=1.2208 | ancestor=0.0000 | lr=4.55e-05 | t=40.3s
|
| 142 |
+
step=423500 | total=1.0743 | leaf=1.0743 | ancestor=0.0000 | lr=4.54e-05 | t=40.3s
|
| 143 |
+
step=423600 | total=1.4084 | leaf=1.4084 | ancestor=0.0000 | lr=4.54e-05 | t=40.3s
|
| 144 |
+
step=423700 | total=1.1402 | leaf=1.1402 | ancestor=0.0000 | lr=4.53e-05 | t=40.3s
|
| 145 |
+
step=423800 | total=1.0788 | leaf=1.0788 | ancestor=0.0000 | lr=4.53e-05 | t=40.3s
|
| 146 |
+
step=423900 | total=1.0742 | leaf=1.0742 | ancestor=0.0000 | lr=4.53e-05 | t=40.3s
|
| 147 |
+
step=424000 | total=0.8999 | leaf=0.8999 | ancestor=0.0000 | lr=4.52e-05 | t=40.3s
|
| 148 |
+
step=424100 | total=1.2260 | leaf=1.2260 | ancestor=0.0000 | lr=4.52e-05 | t=40.3s
|
| 149 |
+
step=424200 | total=1.4743 | leaf=1.4743 | ancestor=0.0000 | lr=4.51e-05 | t=40.3s
|
| 150 |
+
step=424300 | total=0.8561 | leaf=0.8561 | ancestor=0.0000 | lr=4.51e-05 | t=40.3s
|
| 151 |
+
step=424400 | total=1.3305 | leaf=1.3305 | ancestor=0.0000 | lr=4.51e-05 | t=40.3s
|
| 152 |
+
step=424500 | total=1.7270 | leaf=1.7270 | ancestor=0.0000 | lr=4.50e-05 | t=40.3s
|
| 153 |
+
step=424600 | total=1.3142 | leaf=1.3142 | ancestor=0.0000 | lr=4.50e-05 | t=40.3s
|
| 154 |
+
step=424700 | total=1.5884 | leaf=1.5884 | ancestor=0.0000 | lr=4.49e-05 | t=40.3s
|
| 155 |
+
step=424800 | total=0.8186 | leaf=0.8186 | ancestor=0.0000 | lr=4.49e-05 | t=40.3s
|
| 156 |
+
step=424900 | total=0.9008 | leaf=0.9008 | ancestor=0.0000 | lr=4.49e-05 | t=40.3s
|
| 157 |
+
step=425000 | total=1.4898 | leaf=1.4898 | ancestor=0.0000 | lr=4.48e-05 | t=40.3s
|
| 158 |
+
VAL | loss_leaf=1.2443 | loss_ancestor=0.0000 | loss_total=1.2443
|
| 159 |
+
step=425100 | total=1.6797 | leaf=1.6797 | ancestor=0.0000 | lr=4.48e-05 | t=47.6s
|
| 160 |
+
step=425200 | total=1.6269 | leaf=1.6269 | ancestor=0.0000 | lr=4.48e-05 | t=40.3s
|
| 161 |
+
step=425300 | total=1.1943 | leaf=1.1943 | ancestor=0.0000 | lr=4.47e-05 | t=40.3s
|
| 162 |
+
step=425400 | total=1.1574 | leaf=1.1574 | ancestor=0.0000 | lr=4.47e-05 | t=40.3s
|
| 163 |
+
step=425500 | total=0.9751 | leaf=0.9751 | ancestor=0.0000 | lr=4.46e-05 | t=78.9s
|
| 164 |
+
step=425600 | total=1.0587 | leaf=1.0587 | ancestor=0.0000 | lr=4.46e-05 | t=40.3s
|
| 165 |
+
step=425700 | total=1.0705 | leaf=1.0705 | ancestor=0.0000 | lr=4.46e-05 | t=40.4s
|
| 166 |
+
step=425800 | total=1.1670 | leaf=1.1670 | ancestor=0.0000 | lr=4.45e-05 | t=40.3s
|
| 167 |
+
step=425900 | total=1.2762 | leaf=1.2762 | ancestor=0.0000 | lr=4.45e-05 | t=40.3s
|
| 168 |
+
step=426000 | total=1.3389 | leaf=1.3389 | ancestor=0.0000 | lr=4.44e-05 | t=40.3s
|
| 169 |
+
step=426100 | total=1.2804 | leaf=1.2804 | ancestor=0.0000 | lr=4.44e-05 | t=40.3s
|
| 170 |
+
step=426200 | total=0.8569 | leaf=0.8569 | ancestor=0.0000 | lr=4.44e-05 | t=40.3s
|
| 171 |
+
step=426300 | total=0.9764 | leaf=0.9764 | ancestor=0.0000 | lr=4.43e-05 | t=40.3s
|
| 172 |
+
step=426400 | total=1.4490 | leaf=1.4490 | ancestor=0.0000 | lr=4.43e-05 | t=40.3s
|
| 173 |
+
step=426500 | total=0.9180 | leaf=0.9180 | ancestor=0.0000 | lr=4.43e-05 | t=40.3s
|
| 174 |
+
step=426600 | total=1.1111 | leaf=1.1111 | ancestor=0.0000 | lr=4.42e-05 | t=40.3s
|
| 175 |
+
step=426700 | total=0.8156 | leaf=0.8156 | ancestor=0.0000 | lr=4.42e-05 | t=40.3s
|
| 176 |
+
step=426800 | total=1.2668 | leaf=1.2668 | ancestor=0.0000 | lr=4.41e-05 | t=40.3s
|
| 177 |
+
step=426900 | total=0.9718 | leaf=0.9718 | ancestor=0.0000 | lr=4.41e-05 | t=40.3s
|
| 178 |
+
step=427000 | total=1.3135 | leaf=1.3135 | ancestor=0.0000 | lr=4.41e-05 | t=40.3s
|
| 179 |
+
step=427100 | total=1.5240 | leaf=1.5240 | ancestor=0.0000 | lr=4.40e-05 | t=40.3s
|
| 180 |
+
step=427200 | total=1.4908 | leaf=1.4908 | ancestor=0.0000 | lr=4.40e-05 | t=40.3s
|
| 181 |
+
step=427300 | total=1.1639 | leaf=1.1639 | ancestor=0.0000 | lr=4.40e-05 | t=40.3s
|
| 182 |
+
step=427400 | total=0.9743 | leaf=0.9743 | ancestor=0.0000 | lr=4.39e-05 | t=40.3s
|
| 183 |
+
step=427500 | total=1.0417 | leaf=1.0417 | ancestor=0.0000 | lr=4.39e-05 | t=40.3s
|
| 184 |
+
step=427600 | total=1.2549 | leaf=1.2549 | ancestor=0.0000 | lr=4.38e-05 | t=40.3s
|
| 185 |
+
step=427700 | total=1.2046 | leaf=1.2046 | ancestor=0.0000 | lr=4.38e-05 | t=40.3s
|
| 186 |
+
step=427800 | total=0.9435 | leaf=0.9435 | ancestor=0.0000 | lr=4.38e-05 | t=40.3s
|
| 187 |
+
step=427900 | total=1.1936 | leaf=1.1936 | ancestor=0.0000 | lr=4.37e-05 | t=40.3s
|
| 188 |
+
step=428000 | total=1.1012 | leaf=1.1012 | ancestor=0.0000 | lr=4.37e-05 | t=40.3s
|
| 189 |
+
step=428100 | total=1.4997 | leaf=1.4997 | ancestor=0.0000 | lr=4.37e-05 | t=40.3s
|
| 190 |
+
step=428200 | total=1.2035 | leaf=1.2035 | ancestor=0.0000 | lr=4.36e-05 | t=40.3s
|
| 191 |
+
step=428300 | total=0.8781 | leaf=0.8781 | ancestor=0.0000 | lr=4.36e-05 | t=40.3s
|
| 192 |
+
step=428400 | total=1.2399 | leaf=1.2399 | ancestor=0.0000 | lr=4.35e-05 | t=40.3s
|
| 193 |
+
step=428500 | total=1.5526 | leaf=1.5526 | ancestor=0.0000 | lr=4.35e-05 | t=40.3s
|
| 194 |
+
step=428600 | total=0.9087 | leaf=0.9087 | ancestor=0.0000 | lr=4.35e-05 | t=40.3s
|
| 195 |
+
step=428700 | total=1.4247 | leaf=1.4247 | ancestor=0.0000 | lr=4.34e-05 | t=40.3s
|
| 196 |
+
step=428800 | total=1.3466 | leaf=1.3466 | ancestor=0.0000 | lr=4.34e-05 | t=40.3s
|
| 197 |
+
step=428900 | total=1.1001 | leaf=1.1001 | ancestor=0.0000 | lr=4.34e-05 | t=40.3s
|
| 198 |
+
step=429000 | total=0.8915 | leaf=0.8915 | ancestor=0.0000 | lr=4.33e-05 | t=40.3s
|
| 199 |
+
step=429100 | total=1.0700 | leaf=1.0700 | ancestor=0.0000 | lr=4.33e-05 | t=40.3s
|
| 200 |
+
step=429200 | total=1.1175 | leaf=1.1175 | ancestor=0.0000 | lr=4.32e-05 | t=40.3s
|
| 201 |
+
step=429300 | total=1.4647 | leaf=1.4647 | ancestor=0.0000 | lr=4.32e-05 | t=40.3s
|
| 202 |
+
step=429400 | total=1.2476 | leaf=1.2476 | ancestor=0.0000 | lr=4.32e-05 | t=40.3s
|
| 203 |
+
step=429500 | total=1.3851 | leaf=1.3851 | ancestor=0.0000 | lr=4.31e-05 | t=40.3s
|
| 204 |
+
step=429600 | total=1.4779 | leaf=1.4779 | ancestor=0.0000 | lr=4.31e-05 | t=40.3s
|
| 205 |
+
step=429700 | total=1.6588 | leaf=1.6588 | ancestor=0.0000 | lr=4.31e-05 | t=40.3s
|
| 206 |
+
step=429800 | total=1.2108 | leaf=1.2108 | ancestor=0.0000 | lr=4.30e-05 | t=40.3s
|
| 207 |
+
step=429900 | total=1.1808 | leaf=1.1808 | ancestor=0.0000 | lr=4.30e-05 | t=40.3s
|
| 208 |
+
step=430000 | total=0.9405 | leaf=0.9405 | ancestor=0.0000 | lr=4.30e-05 | t=40.3s
|
| 209 |
+
VAL | loss_leaf=1.2234 | loss_ancestor=0.0000 | loss_total=1.2234
|
| 210 |
+
Saved checkpoint: outputs/sad_block_ar_b32_top2/ckpt_430000.pt
|
| 211 |
+
step=430100 | total=1.3857 | leaf=1.3857 | ancestor=0.0000 | lr=4.29e-05 | t=56.9s
|
| 212 |
+
step=430200 | total=1.3944 | leaf=1.3944 | ancestor=0.0000 | lr=4.29e-05 | t=40.3s
|
| 213 |
+
step=430300 | total=1.1386 | leaf=1.1386 | ancestor=0.0000 | lr=4.28e-05 | t=40.3s
|
| 214 |
+
step=430400 | total=1.3882 | leaf=1.3882 | ancestor=0.0000 | lr=4.28e-05 | t=40.3s
|
| 215 |
+
step=430500 | total=1.2067 | leaf=1.2067 | ancestor=0.0000 | lr=4.28e-05 | t=40.3s
|
| 216 |
+
step=430600 | total=1.1775 | leaf=1.1775 | ancestor=0.0000 | lr=4.27e-05 | t=40.3s
|
| 217 |
+
step=430700 | total=1.2822 | leaf=1.2822 | ancestor=0.0000 | lr=4.27e-05 | t=40.3s
|
| 218 |
+
step=430800 | total=1.3991 | leaf=1.3991 | ancestor=0.0000 | lr=4.27e-05 | t=40.3s
|
| 219 |
+
step=430900 | total=1.3899 | leaf=1.3899 | ancestor=0.0000 | lr=4.26e-05 | t=40.3s
|
| 220 |
+
step=431000 | total=1.0058 | leaf=1.0058 | ancestor=0.0000 | lr=4.26e-05 | t=40.3s
|
| 221 |
+
step=431100 | total=1.2606 | leaf=1.2606 | ancestor=0.0000 | lr=4.26e-05 | t=40.3s
|
| 222 |
+
step=431200 | total=1.2779 | leaf=1.2779 | ancestor=0.0000 | lr=4.25e-05 | t=40.3s
|
| 223 |
+
step=431300 | total=1.3241 | leaf=1.3241 | ancestor=0.0000 | lr=4.25e-05 | t=40.3s
|
| 224 |
+
step=431400 | total=1.4623 | leaf=1.4623 | ancestor=0.0000 | lr=4.24e-05 | t=40.3s
|
| 225 |
+
step=431500 | total=1.2695 | leaf=1.2695 | ancestor=0.0000 | lr=4.24e-05 | t=40.3s
|
| 226 |
+
step=431600 | total=1.3697 | leaf=1.3697 | ancestor=0.0000 | lr=4.24e-05 | t=40.3s
|
| 227 |
+
step=431700 | total=0.8857 | leaf=0.8857 | ancestor=0.0000 | lr=4.23e-05 | t=40.3s
|
| 228 |
+
step=431800 | total=1.3153 | leaf=1.3153 | ancestor=0.0000 | lr=4.23e-05 | t=40.3s
|
| 229 |
+
step=431900 | total=0.9657 | leaf=0.9657 | ancestor=0.0000 | lr=4.23e-05 | t=40.3s
|
| 230 |
+
step=432000 | total=0.8793 | leaf=0.8793 | ancestor=0.0000 | lr=4.22e-05 | t=40.3s
|
| 231 |
+
step=432100 | total=1.1115 | leaf=1.1115 | ancestor=0.0000 | lr=4.22e-05 | t=40.3s
|
| 232 |
+
step=432200 | total=0.9992 | leaf=0.9992 | ancestor=0.0000 | lr=4.22e-05 | t=40.3s
|
| 233 |
+
step=432300 | total=1.0293 | leaf=1.0293 | ancestor=0.0000 | lr=4.21e-05 | t=40.3s
|
| 234 |
+
step=432400 | total=1.1425 | leaf=1.1425 | ancestor=0.0000 | lr=4.21e-05 | t=40.3s
|
| 235 |
+
step=432500 | total=1.1295 | leaf=1.1295 | ancestor=0.0000 | lr=4.21e-05 | t=40.3s
|
| 236 |
+
step=432600 | total=1.1051 | leaf=1.1051 | ancestor=0.0000 | lr=4.20e-05 | t=40.3s
|
| 237 |
+
step=432700 | total=1.3211 | leaf=1.3211 | ancestor=0.0000 | lr=4.20e-05 | t=40.3s
|
| 238 |
+
step=432800 | total=0.9871 | leaf=0.9871 | ancestor=0.0000 | lr=4.20e-05 | t=40.3s
|
| 239 |
+
step=432900 | total=0.9651 | leaf=0.9651 | ancestor=0.0000 | lr=4.19e-05 | t=40.3s
|
| 240 |
+
step=433000 | total=1.3421 | leaf=1.3421 | ancestor=0.0000 | lr=4.19e-05 | t=40.3s
|
| 241 |
+
step=433100 | total=1.3807 | leaf=1.3807 | ancestor=0.0000 | lr=4.18e-05 | t=40.3s
|
| 242 |
+
step=433200 | total=1.2792 | leaf=1.2792 | ancestor=0.0000 | lr=4.18e-05 | t=40.3s
|
| 243 |
+
step=433300 | total=1.1567 | leaf=1.1567 | ancestor=0.0000 | lr=4.18e-05 | t=40.3s
|
| 244 |
+
step=433400 | total=1.2361 | leaf=1.2361 | ancestor=0.0000 | lr=4.17e-05 | t=40.3s
|
| 245 |
+
step=433500 | total=1.1388 | leaf=1.1388 | ancestor=0.0000 | lr=4.17e-05 | t=40.3s
|
| 246 |
+
step=433600 | total=1.0371 | leaf=1.0371 | ancestor=0.0000 | lr=4.17e-05 | t=40.3s
|
| 247 |
+
step=433700 | total=1.2588 | leaf=1.2588 | ancestor=0.0000 | lr=4.16e-05 | t=40.3s
|
| 248 |
+
step=433800 | total=1.4319 | leaf=1.4319 | ancestor=0.0000 | lr=4.16e-05 | t=40.3s
|
| 249 |
+
step=433900 | total=1.0033 | leaf=1.0033 | ancestor=0.0000 | lr=4.16e-05 | t=40.3s
|
| 250 |
+
step=434000 | total=1.2453 | leaf=1.2453 | ancestor=0.0000 | lr=4.15e-05 | t=40.3s
|
| 251 |
+
step=434100 | total=1.3205 | leaf=1.3205 | ancestor=0.0000 | lr=4.15e-05 | t=40.2s
|
| 252 |
+
step=434200 | total=1.1705 | leaf=1.1705 | ancestor=0.0000 | lr=4.15e-05 | t=40.3s
|
| 253 |
+
step=434300 | total=1.2056 | leaf=1.2056 | ancestor=0.0000 | lr=4.14e-05 | t=40.3s
|
| 254 |
+
step=434400 | total=0.8367 | leaf=0.8367 | ancestor=0.0000 | lr=4.14e-05 | t=40.3s
|
| 255 |
+
step=434500 | total=1.0914 | leaf=1.0914 | ancestor=0.0000 | lr=4.14e-05 | t=40.3s
|
| 256 |
+
step=434600 | total=1.0539 | leaf=1.0539 | ancestor=0.0000 | lr=4.13e-05 | t=40.3s
|
| 257 |
+
step=434700 | total=1.5508 | leaf=1.5508 | ancestor=0.0000 | lr=4.13e-05 | t=40.3s
|
| 258 |
+
step=434800 | total=0.9442 | leaf=0.9442 | ancestor=0.0000 | lr=4.13e-05 | t=40.3s
|
| 259 |
+
step=434900 | total=1.2059 | leaf=1.2059 | ancestor=0.0000 | lr=4.12e-05 | t=40.3s
|
| 260 |
+
step=435000 | total=1.1030 | leaf=1.1030 | ancestor=0.0000 | lr=4.12e-05 | t=40.3s
|
| 261 |
+
VAL | loss_leaf=1.2535 | loss_ancestor=0.0000 | loss_total=1.2535
|
| 262 |
+
step=435100 | total=1.5107 | leaf=1.5107 | ancestor=0.0000 | lr=4.12e-05 | t=47.6s
|
| 263 |
+
step=435200 | total=1.2198 | leaf=1.2198 | ancestor=0.0000 | lr=4.11e-05 | t=40.3s
|
| 264 |
+
step=435300 | total=1.3648 | leaf=1.3648 | ancestor=0.0000 | lr=4.11e-05 | t=40.3s
|
| 265 |
+
step=435400 | total=1.0139 | leaf=1.0139 | ancestor=0.0000 | lr=4.11e-05 | t=40.3s
|
| 266 |
+
step=435500 | total=1.2582 | leaf=1.2582 | ancestor=0.0000 | lr=4.10e-05 | t=40.3s
|
| 267 |
+
step=435600 | total=1.2253 | leaf=1.2253 | ancestor=0.0000 | lr=4.10e-05 | t=40.3s
|
| 268 |
+
step=435700 | total=1.2636 | leaf=1.2636 | ancestor=0.0000 | lr=4.10e-05 | t=40.3s
|
| 269 |
+
step=435800 | total=1.1206 | leaf=1.1206 | ancestor=0.0000 | lr=4.09e-05 | t=40.3s
|
| 270 |
+
step=435900 | total=1.3249 | leaf=1.3249 | ancestor=0.0000 | lr=4.09e-05 | t=40.3s
|
| 271 |
+
step=436000 | total=1.3852 | leaf=1.3852 | ancestor=0.0000 | lr=4.09e-05 | t=40.3s
|
| 272 |
+
step=436100 | total=0.9904 | leaf=0.9904 | ancestor=0.0000 | lr=4.08e-05 | t=40.3s
|
| 273 |
+
step=436200 | total=0.8992 | leaf=0.8992 | ancestor=0.0000 | lr=4.08e-05 | t=40.3s
|
| 274 |
+
step=436300 | total=1.1194 | leaf=1.1194 | ancestor=0.0000 | lr=4.08e-05 | t=40.3s
|
| 275 |
+
step=436400 | total=0.9287 | leaf=0.9287 | ancestor=0.0000 | lr=4.07e-05 | t=40.3s
|
| 276 |
+
step=436500 | total=1.4030 | leaf=1.4030 | ancestor=0.0000 | lr=4.07e-05 | t=40.3s
|
| 277 |
+
step=436600 | total=1.5683 | leaf=1.5683 | ancestor=0.0000 | lr=4.07e-05 | t=40.3s
|
| 278 |
+
step=436700 | total=1.0064 | leaf=1.0064 | ancestor=0.0000 | lr=4.06e-05 | t=40.3s
|
| 279 |
+
step=436800 | total=1.0872 | leaf=1.0872 | ancestor=0.0000 | lr=4.06e-05 | t=40.3s
|
| 280 |
+
step=436900 | total=1.4174 | leaf=1.4174 | ancestor=0.0000 | lr=4.06e-05 | t=40.3s
|
| 281 |
+
step=437000 | total=1.3512 | leaf=1.3512 | ancestor=0.0000 | lr=4.05e-05 | t=40.3s
|
| 282 |
+
step=437100 | total=1.1645 | leaf=1.1645 | ancestor=0.0000 | lr=4.05e-05 | t=40.3s
|
| 283 |
+
step=437200 | total=1.1136 | leaf=1.1136 | ancestor=0.0000 | lr=4.05e-05 | t=40.3s
|
| 284 |
+
step=437300 | total=1.5552 | leaf=1.5552 | ancestor=0.0000 | lr=4.04e-05 | t=40.3s
|
| 285 |
+
step=437400 | total=1.1414 | leaf=1.1414 | ancestor=0.0000 | lr=4.04e-05 | t=40.3s
|
| 286 |
+
step=437500 | total=1.1786 | leaf=1.1786 | ancestor=0.0000 | lr=4.04e-05 | t=40.3s
|
| 287 |
+
step=437600 | total=1.1055 | leaf=1.1055 | ancestor=0.0000 | lr=4.03e-05 | t=40.3s
|
| 288 |
+
step=437700 | total=1.2134 | leaf=1.2134 | ancestor=0.0000 | lr=4.03e-05 | t=40.3s
|
| 289 |
+
step=437800 | total=1.1445 | leaf=1.1445 | ancestor=0.0000 | lr=4.03e-05 | t=40.3s
|
| 290 |
+
step=437900 | total=1.2711 | leaf=1.2711 | ancestor=0.0000 | lr=4.02e-05 | t=40.3s
|
| 291 |
+
step=438000 | total=0.9726 | leaf=0.9726 | ancestor=0.0000 | lr=4.02e-05 | t=40.3s
|
| 292 |
+
step=438100 | total=1.0019 | leaf=1.0019 | ancestor=0.0000 | lr=4.02e-05 | t=40.3s
|
| 293 |
+
step=438200 | total=0.7892 | leaf=0.7892 | ancestor=0.0000 | lr=4.01e-05 | t=40.3s
|
| 294 |
+
step=438300 | total=1.0740 | leaf=1.0740 | ancestor=0.0000 | lr=4.01e-05 | t=40.3s
|
| 295 |
+
step=438400 | total=1.3317 | leaf=1.3317 | ancestor=0.0000 | lr=4.01e-05 | t=40.3s
|
| 296 |
+
step=438500 | total=0.9645 | leaf=0.9645 | ancestor=0.0000 | lr=4.00e-05 | t=40.3s
|
| 297 |
+
step=438600 | total=1.5879 | leaf=1.5879 | ancestor=0.0000 | lr=4.00e-05 | t=40.3s
|
| 298 |
+
step=438700 | total=1.2463 | leaf=1.2463 | ancestor=0.0000 | lr=4.00e-05 | t=40.3s
|
| 299 |
+
step=438800 | total=0.9429 | leaf=0.9429 | ancestor=0.0000 | lr=3.99e-05 | t=40.3s
|
| 300 |
+
step=438900 | total=1.3516 | leaf=1.3516 | ancestor=0.0000 | lr=3.99e-05 | t=40.3s
|
| 301 |
+
step=439000 | total=1.1351 | leaf=1.1351 | ancestor=0.0000 | lr=3.99e-05 | t=40.3s
|
| 302 |
+
step=439100 | total=1.3537 | leaf=1.3537 | ancestor=0.0000 | lr=3.98e-05 | t=40.3s
|
| 303 |
+
step=439200 | total=1.2809 | leaf=1.2809 | ancestor=0.0000 | lr=3.98e-05 | t=40.3s
|
| 304 |
+
step=439300 | total=1.2890 | leaf=1.2890 | ancestor=0.0000 | lr=3.98e-05 | t=40.3s
|
| 305 |
+
step=439400 | total=0.9967 | leaf=0.9967 | ancestor=0.0000 | lr=3.97e-05 | t=40.3s
|
| 306 |
+
step=439500 | total=1.4379 | leaf=1.4379 | ancestor=0.0000 | lr=3.97e-05 | t=40.3s
|
| 307 |
+
step=439600 | total=1.1130 | leaf=1.1130 | ancestor=0.0000 | lr=3.97e-05 | t=40.3s
|
| 308 |
+
step=439700 | total=1.1017 | leaf=1.1017 | ancestor=0.0000 | lr=3.97e-05 | t=40.3s
|
| 309 |
+
step=439800 | total=1.3877 | leaf=1.3877 | ancestor=0.0000 | lr=3.96e-05 | t=40.3s
|
| 310 |
+
step=439900 | total=1.2916 | leaf=1.2916 | ancestor=0.0000 | lr=3.96e-05 | t=40.3s
|
| 311 |
+
step=440000 | total=1.2500 | leaf=1.2500 | ancestor=0.0000 | lr=3.96e-05 | t=40.3s
|
| 312 |
+
VAL | loss_leaf=1.2474 | loss_ancestor=0.0000 | loss_total=1.2474
|
| 313 |
+
Saved checkpoint: outputs/sad_block_ar_b32_top2/ckpt_440000.pt
|
| 314 |
+
step=440100 | total=1.0527 | leaf=1.0527 | ancestor=0.0000 | lr=3.95e-05 | t=56.9s
|
| 315 |
+
step=440200 | total=0.9997 | leaf=0.9997 | ancestor=0.0000 | lr=3.95e-05 | t=40.3s
|
| 316 |
+
step=440300 | total=1.0806 | leaf=1.0806 | ancestor=0.0000 | lr=3.95e-05 | t=40.3s
|
| 317 |
+
step=440400 | total=1.0644 | leaf=1.0644 | ancestor=0.0000 | lr=3.94e-05 | t=40.3s
|
| 318 |
+
step=440500 | total=1.3870 | leaf=1.3870 | ancestor=0.0000 | lr=3.94e-05 | t=40.3s
|
| 319 |
+
step=440600 | total=1.2315 | leaf=1.2315 | ancestor=0.0000 | lr=3.94e-05 | t=40.3s
|
| 320 |
+
step=440700 | total=1.1067 | leaf=1.1067 | ancestor=0.0000 | lr=3.93e-05 | t=40.3s
|
| 321 |
+
step=440800 | total=0.9915 | leaf=0.9915 | ancestor=0.0000 | lr=3.93e-05 | t=40.3s
|
| 322 |
+
step=440900 | total=1.1980 | leaf=1.1980 | ancestor=0.0000 | lr=3.93e-05 | t=40.3s
|
| 323 |
+
step=441000 | total=1.1150 | leaf=1.1150 | ancestor=0.0000 | lr=3.92e-05 | t=40.5s
|
| 324 |
+
step=441100 | total=1.4989 | leaf=1.4989 | ancestor=0.0000 | lr=3.92e-05 | t=40.3s
|
| 325 |
+
step=441200 | total=1.1358 | leaf=1.1358 | ancestor=0.0000 | lr=3.92e-05 | t=40.3s
|
| 326 |
+
step=441300 | total=1.1199 | leaf=1.1199 | ancestor=0.0000 | lr=3.92e-05 | t=40.3s
|
| 327 |
+
step=441400 | total=0.9904 | leaf=0.9904 | ancestor=0.0000 | lr=3.91e-05 | t=40.3s
|
| 328 |
+
step=441500 | total=1.0977 | leaf=1.0977 | ancestor=0.0000 | lr=3.91e-05 | t=40.3s
|
| 329 |
+
step=441600 | total=1.0217 | leaf=1.0217 | ancestor=0.0000 | lr=3.91e-05 | t=40.3s
|
| 330 |
+
step=441700 | total=1.2824 | leaf=1.2824 | ancestor=0.0000 | lr=3.90e-05 | t=40.3s
|
| 331 |
+
step=441800 | total=1.1568 | leaf=1.1568 | ancestor=0.0000 | lr=3.90e-05 | t=40.3s
|
| 332 |
+
step=441900 | total=1.3246 | leaf=1.3246 | ancestor=0.0000 | lr=3.90e-05 | t=40.3s
|
| 333 |
+
step=442000 | total=1.1613 | leaf=1.1613 | ancestor=0.0000 | lr=3.89e-05 | t=40.3s
|
| 334 |
+
step=442100 | total=1.3308 | leaf=1.3308 | ancestor=0.0000 | lr=3.89e-05 | t=40.3s
|
| 335 |
+
step=442200 | total=1.2593 | leaf=1.2593 | ancestor=0.0000 | lr=3.89e-05 | t=40.3s
|
| 336 |
+
step=442300 | total=0.9362 | leaf=0.9362 | ancestor=0.0000 | lr=3.88e-05 | t=40.3s
|
| 337 |
+
step=442400 | total=1.3548 | leaf=1.3548 | ancestor=0.0000 | lr=3.88e-05 | t=40.3s
|
| 338 |
+
step=442500 | total=1.2053 | leaf=1.2053 | ancestor=0.0000 | lr=3.88e-05 | t=40.3s
|
| 339 |
+
step=442600 | total=1.0512 | leaf=1.0512 | ancestor=0.0000 | lr=3.88e-05 | t=40.3s
|
| 340 |
+
step=442700 | total=1.3028 | leaf=1.3028 | ancestor=0.0000 | lr=3.87e-05 | t=40.3s
|
| 341 |
+
step=442800 | total=1.0826 | leaf=1.0826 | ancestor=0.0000 | lr=3.87e-05 | t=40.3s
|
| 342 |
+
step=442900 | total=1.2354 | leaf=1.2354 | ancestor=0.0000 | lr=3.87e-05 | t=40.3s
|
| 343 |
+
step=443000 | total=1.1239 | leaf=1.1239 | ancestor=0.0000 | lr=3.86e-05 | t=40.3s
|
| 344 |
+
step=443100 | total=1.6173 | leaf=1.6173 | ancestor=0.0000 | lr=3.86e-05 | t=40.3s
|
| 345 |
+
step=443200 | total=1.4340 | leaf=1.4340 | ancestor=0.0000 | lr=3.86e-05 | t=40.3s
|
| 346 |
+
step=443300 | total=1.0478 | leaf=1.0478 | ancestor=0.0000 | lr=3.85e-05 | t=40.3s
|
| 347 |
+
step=443400 | total=1.3807 | leaf=1.3807 | ancestor=0.0000 | lr=3.85e-05 | t=40.3s
|
| 348 |
+
step=443500 | total=1.3269 | leaf=1.3269 | ancestor=0.0000 | lr=3.85e-05 | t=40.3s
|
| 349 |
+
step=443600 | total=0.8619 | leaf=0.8619 | ancestor=0.0000 | lr=3.85e-05 | t=40.3s
|
| 350 |
+
step=443700 | total=1.7034 | leaf=1.7034 | ancestor=0.0000 | lr=3.84e-05 | t=40.3s
|
| 351 |
+
step=443800 | total=1.0349 | leaf=1.0349 | ancestor=0.0000 | lr=3.84e-05 | t=40.3s
|
| 352 |
+
step=443900 | total=1.5285 | leaf=1.5285 | ancestor=0.0000 | lr=3.84e-05 | t=40.3s
|
| 353 |
+
step=444000 | total=1.3013 | leaf=1.3013 | ancestor=0.0000 | lr=3.83e-05 | t=40.3s
|
| 354 |
+
step=444100 | total=1.0942 | leaf=1.0942 | ancestor=0.0000 | lr=3.83e-05 | t=40.3s
|
| 355 |
+
step=444200 | total=1.2974 | leaf=1.2974 | ancestor=0.0000 | lr=3.83e-05 | t=40.3s
|
| 356 |
+
step=444300 | total=1.5143 | leaf=1.5143 | ancestor=0.0000 | lr=3.82e-05 | t=40.3s
|
| 357 |
+
step=444400 | total=1.3297 | leaf=1.3297 | ancestor=0.0000 | lr=3.82e-05 | t=40.3s
|
| 358 |
+
step=444500 | total=1.2214 | leaf=1.2214 | ancestor=0.0000 | lr=3.82e-05 | t=40.3s
|
| 359 |
+
step=444600 | total=1.5719 | leaf=1.5719 | ancestor=0.0000 | lr=3.82e-05 | t=40.3s
|
| 360 |
+
step=444700 | total=1.1732 | leaf=1.1732 | ancestor=0.0000 | lr=3.81e-05 | t=40.3s
|
| 361 |
+
step=444800 | total=1.0727 | leaf=1.0727 | ancestor=0.0000 | lr=3.81e-05 | t=40.3s
|
| 362 |
+
step=444900 | total=1.0372 | leaf=1.0372 | ancestor=0.0000 | lr=3.81e-05 | t=40.3s
|
| 363 |
+
step=445000 | total=1.2506 | leaf=1.2506 | ancestor=0.0000 | lr=3.80e-05 | t=40.3s
|
| 364 |
+
VAL | loss_leaf=1.2314 | loss_ancestor=0.0000 | loss_total=1.2314
|
| 365 |
+
step=445100 | total=1.2679 | leaf=1.2679 | ancestor=0.0000 | lr=3.80e-05 | t=47.7s
|
| 366 |
+
step=445200 | total=0.8179 | leaf=0.8179 | ancestor=0.0000 | lr=3.80e-05 | t=40.3s
|
| 367 |
+
step=445300 | total=1.1472 | leaf=1.1472 | ancestor=0.0000 | lr=3.80e-05 | t=40.3s
|
| 368 |
+
step=445400 | total=1.1142 | leaf=1.1142 | ancestor=0.0000 | lr=3.79e-05 | t=40.3s
|
| 369 |
+
step=445500 | total=1.1727 | leaf=1.1727 | ancestor=0.0000 | lr=3.79e-05 | t=40.3s
|
| 370 |
+
step=445600 | total=0.9856 | leaf=0.9856 | ancestor=0.0000 | lr=3.79e-05 | t=40.3s
|
| 371 |
+
step=445700 | total=1.3444 | leaf=1.3444 | ancestor=0.0000 | lr=3.78e-05 | t=40.3s
|
| 372 |
+
step=445800 | total=1.5482 | leaf=1.5482 | ancestor=0.0000 | lr=3.78e-05 | t=40.3s
|
| 373 |
+
step=445900 | total=1.2607 | leaf=1.2607 | ancestor=0.0000 | lr=3.78e-05 | t=40.3s
|
| 374 |
+
step=446000 | total=1.4733 | leaf=1.4733 | ancestor=0.0000 | lr=3.78e-05 | t=40.3s
|
| 375 |
+
step=446100 | total=1.4045 | leaf=1.4045 | ancestor=0.0000 | lr=3.77e-05 | t=40.3s
|
| 376 |
+
step=446200 | total=1.4546 | leaf=1.4546 | ancestor=0.0000 | lr=3.77e-05 | t=40.3s
|
| 377 |
+
step=446300 | total=1.4040 | leaf=1.4040 | ancestor=0.0000 | lr=3.77e-05 | t=40.3s
|
| 378 |
+
step=446400 | total=1.1792 | leaf=1.1792 | ancestor=0.0000 | lr=3.76e-05 | t=40.3s
|
| 379 |
+
step=446500 | total=1.2761 | leaf=1.2761 | ancestor=0.0000 | lr=3.76e-05 | t=40.3s
|
| 380 |
+
step=446600 | total=1.4740 | leaf=1.4740 | ancestor=0.0000 | lr=3.76e-05 | t=40.3s
|
| 381 |
+
step=446700 | total=0.8015 | leaf=0.8015 | ancestor=0.0000 | lr=3.76e-05 | t=40.3s
|
| 382 |
+
step=446800 | total=1.4113 | leaf=1.4113 | ancestor=0.0000 | lr=3.75e-05 | t=40.3s
|
| 383 |
+
step=446900 | total=1.2629 | leaf=1.2629 | ancestor=0.0000 | lr=3.75e-05 | t=40.3s
|
| 384 |
+
step=447000 | total=1.1258 | leaf=1.1258 | ancestor=0.0000 | lr=3.75e-05 | t=40.3s
|
| 385 |
+
step=447100 | total=1.0549 | leaf=1.0549 | ancestor=0.0000 | lr=3.74e-05 | t=40.3s
|
| 386 |
+
step=447200 | total=1.0760 | leaf=1.0760 | ancestor=0.0000 | lr=3.74e-05 | t=40.3s
|
| 387 |
+
step=447300 | total=1.1495 | leaf=1.1495 | ancestor=0.0000 | lr=3.74e-05 | t=40.3s
|
| 388 |
+
step=447400 | total=0.9082 | leaf=0.9082 | ancestor=0.0000 | lr=3.74e-05 | t=40.3s
|
| 389 |
+
step=447500 | total=1.2279 | leaf=1.2279 | ancestor=0.0000 | lr=3.73e-05 | t=40.3s
|
| 390 |
+
step=447600 | total=1.2246 | leaf=1.2246 | ancestor=0.0000 | lr=3.73e-05 | t=40.3s
|
| 391 |
+
step=447700 | total=1.0533 | leaf=1.0533 | ancestor=0.0000 | lr=3.73e-05 | t=40.3s
|
| 392 |
+
step=447800 | total=1.2798 | leaf=1.2798 | ancestor=0.0000 | lr=3.73e-05 | t=40.3s
|
| 393 |
+
step=447900 | total=1.0699 | leaf=1.0699 | ancestor=0.0000 | lr=3.72e-05 | t=40.3s
|
| 394 |
+
step=448000 | total=1.6639 | leaf=1.6639 | ancestor=0.0000 | lr=3.72e-05 | t=40.3s
|
| 395 |
+
step=448100 | total=1.4433 | leaf=1.4433 | ancestor=0.0000 | lr=3.72e-05 | t=40.3s
|
| 396 |
+
step=448200 | total=1.0875 | leaf=1.0875 | ancestor=0.0000 | lr=3.71e-05 | t=40.3s
|
| 397 |
+
step=448300 | total=1.3449 | leaf=1.3449 | ancestor=0.0000 | lr=3.71e-05 | t=40.3s
|
| 398 |
+
step=448400 | total=1.4151 | leaf=1.4151 | ancestor=0.0000 | lr=3.71e-05 | t=40.3s
|
| 399 |
+
step=448500 | total=1.0956 | leaf=1.0956 | ancestor=0.0000 | lr=3.71e-05 | t=40.3s
|
| 400 |
+
step=448600 | total=1.2530 | leaf=1.2530 | ancestor=0.0000 | lr=3.70e-05 | t=40.3s
|
| 401 |
+
step=448700 | total=1.2827 | leaf=1.2827 | ancestor=0.0000 | lr=3.70e-05 | t=40.3s
|
| 402 |
+
step=448800 | total=1.1704 | leaf=1.1704 | ancestor=0.0000 | lr=3.70e-05 | t=40.3s
|
| 403 |
+
step=448900 | total=1.3378 | leaf=1.3378 | ancestor=0.0000 | lr=3.70e-05 | t=40.3s
|
| 404 |
+
step=449000 | total=1.3851 | leaf=1.3851 | ancestor=0.0000 | lr=3.69e-05 | t=40.3s
|
| 405 |
+
step=449100 | total=1.4329 | leaf=1.4329 | ancestor=0.0000 | lr=3.69e-05 | t=40.3s
|
| 406 |
+
step=449200 | total=0.8953 | leaf=0.8953 | ancestor=0.0000 | lr=3.69e-05 | t=40.3s
|
| 407 |
+
step=449300 | total=1.3241 | leaf=1.3241 | ancestor=0.0000 | lr=3.68e-05 | t=40.3s
|
| 408 |
+
step=449400 | total=1.0976 | leaf=1.0976 | ancestor=0.0000 | lr=3.68e-05 | t=40.3s
|
| 409 |
+
step=449500 | total=1.4353 | leaf=1.4353 | ancestor=0.0000 | lr=3.68e-05 | t=40.2s
|
| 410 |
+
step=449600 | total=1.2150 | leaf=1.2150 | ancestor=0.0000 | lr=3.68e-05 | t=40.3s
|
| 411 |
+
step=449700 | total=1.4042 | leaf=1.4042 | ancestor=0.0000 | lr=3.67e-05 | t=40.3s
|
| 412 |
+
step=449800 | total=1.5845 | leaf=1.5845 | ancestor=0.0000 | lr=3.67e-05 | t=40.3s
|
| 413 |
+
step=449900 | total=1.2661 | leaf=1.2661 | ancestor=0.0000 | lr=3.67e-05 | t=40.3s
|
| 414 |
+
step=450000 | total=1.3156 | leaf=1.3156 | ancestor=0.0000 | lr=3.67e-05 | t=40.2s
|
| 415 |
+
VAL | loss_leaf=1.1985 | loss_ancestor=0.0000 | loss_total=1.1985
|
| 416 |
+
Saved checkpoint: outputs/sad_block_ar_b32_top2/ckpt_450000.pt
|
| 417 |
+
step=450100 | total=1.3212 | leaf=1.3212 | ancestor=0.0000 | lr=3.66e-05 | t=56.7s
|
| 418 |
+
step=450200 | total=1.0623 | leaf=1.0623 | ancestor=0.0000 | lr=3.66e-05 | t=40.3s
|
| 419 |
+
step=450300 | total=1.2462 | leaf=1.2462 | ancestor=0.0000 | lr=3.66e-05 | t=40.3s
|
| 420 |
+
step=450400 | total=0.8678 | leaf=0.8678 | ancestor=0.0000 | lr=3.66e-05 | t=40.3s
|
| 421 |
+
step=450500 | total=1.2235 | leaf=1.2235 | ancestor=0.0000 | lr=3.65e-05 | t=40.3s
|
| 422 |
+
step=450600 | total=1.1379 | leaf=1.1379 | ancestor=0.0000 | lr=3.65e-05 | t=40.3s
|
| 423 |
+
step=450700 | total=1.2904 | leaf=1.2904 | ancestor=0.0000 | lr=3.65e-05 | t=40.3s
|
| 424 |
+
step=450800 | total=1.1404 | leaf=1.1404 | ancestor=0.0000 | lr=3.65e-05 | t=40.3s
|
| 425 |
+
step=450900 | total=1.1783 | leaf=1.1783 | ancestor=0.0000 | lr=3.64e-05 | t=40.3s
|
| 426 |
+
step=451000 | total=1.1305 | leaf=1.1305 | ancestor=0.0000 | lr=3.64e-05 | t=40.3s
|
| 427 |
+
step=451100 | total=1.2071 | leaf=1.2071 | ancestor=0.0000 | lr=3.64e-05 | t=40.3s
|
| 428 |
+
step=451200 | total=1.1491 | leaf=1.1491 | ancestor=0.0000 | lr=3.63e-05 | t=40.3s
|
| 429 |
+
step=451300 | total=1.0906 | leaf=1.0906 | ancestor=0.0000 | lr=3.63e-05 | t=40.3s
|
| 430 |
+
step=451400 | total=1.6005 | leaf=1.6005 | ancestor=0.0000 | lr=3.63e-05 | t=40.3s
|
| 431 |
+
step=451500 | total=1.0016 | leaf=1.0016 | ancestor=0.0000 | lr=3.63e-05 | t=40.3s
|
| 432 |
+
step=451600 | total=1.1340 | leaf=1.1340 | ancestor=0.0000 | lr=3.62e-05 | t=40.3s
|
| 433 |
+
step=451700 | total=0.8937 | leaf=0.8937 | ancestor=0.0000 | lr=3.62e-05 | t=40.3s
|
| 434 |
+
step=451800 | total=1.3327 | leaf=1.3327 | ancestor=0.0000 | lr=3.62e-05 | t=40.3s
|
| 435 |
+
step=451900 | total=1.2990 | leaf=1.2990 | ancestor=0.0000 | lr=3.62e-05 | t=40.3s
|
| 436 |
+
step=452000 | total=1.3462 | leaf=1.3462 | ancestor=0.0000 | lr=3.61e-05 | t=40.3s
|
| 437 |
+
step=452100 | total=1.4127 | leaf=1.4127 | ancestor=0.0000 | lr=3.61e-05 | t=40.3s
|
| 438 |
+
step=452200 | total=1.6536 | leaf=1.6536 | ancestor=0.0000 | lr=3.61e-05 | t=40.3s
|
| 439 |
+
step=452300 | total=1.0208 | leaf=1.0208 | ancestor=0.0000 | lr=3.61e-05 | t=40.3s
|
| 440 |
+
step=452400 | total=1.0947 | leaf=1.0947 | ancestor=0.0000 | lr=3.60e-05 | t=40.3s
|
| 441 |
+
step=452500 | total=1.5023 | leaf=1.5023 | ancestor=0.0000 | lr=3.60e-05 | t=40.3s
|
| 442 |
+
step=452600 | total=1.1538 | leaf=1.1538 | ancestor=0.0000 | lr=3.60e-05 | t=40.3s
|
| 443 |
+
step=452700 | total=1.2930 | leaf=1.2930 | ancestor=0.0000 | lr=3.60e-05 | t=40.3s
|
| 444 |
+
step=452800 | total=1.2546 | leaf=1.2546 | ancestor=0.0000 | lr=3.59e-05 | t=40.3s
|
| 445 |
+
step=452900 | total=1.1675 | leaf=1.1675 | ancestor=0.0000 | lr=3.59e-05 | t=40.3s
|
| 446 |
+
step=453000 | total=0.8554 | leaf=0.8554 | ancestor=0.0000 | lr=3.59e-05 | t=40.3s
|
| 447 |
+
step=453100 | total=1.3108 | leaf=1.3108 | ancestor=0.0000 | lr=3.59e-05 | t=40.3s
|
| 448 |
+
step=453200 | total=1.1131 | leaf=1.1131 | ancestor=0.0000 | lr=3.58e-05 | t=40.3s
|
| 449 |
+
step=453300 | total=1.1412 | leaf=1.1412 | ancestor=0.0000 | lr=3.58e-05 | t=40.3s
|
| 450 |
+
step=453400 | total=1.1088 | leaf=1.1088 | ancestor=0.0000 | lr=3.58e-05 | t=40.3s
|
| 451 |
+
step=453500 | total=0.9238 | leaf=0.9238 | ancestor=0.0000 | lr=3.58e-05 | t=40.3s
|
| 452 |
+
step=453600 | total=1.2744 | leaf=1.2744 | ancestor=0.0000 | lr=3.57e-05 | t=40.3s
|
| 453 |
+
step=453700 | total=1.0480 | leaf=1.0480 | ancestor=0.0000 | lr=3.57e-05 | t=40.3s
|
| 454 |
+
step=453800 | total=1.3003 | leaf=1.3003 | ancestor=0.0000 | lr=3.57e-05 | t=40.3s
|
| 455 |
+
step=453900 | total=1.5034 | leaf=1.5034 | ancestor=0.0000 | lr=3.57e-05 | t=40.3s
|
| 456 |
+
step=454000 | total=1.1938 | leaf=1.1938 | ancestor=0.0000 | lr=3.56e-05 | t=40.3s
|
| 457 |
+
step=454100 | total=1.3812 | leaf=1.3812 | ancestor=0.0000 | lr=3.56e-05 | t=40.3s
|
| 458 |
+
step=454200 | total=1.1304 | leaf=1.1304 | ancestor=0.0000 | lr=3.56e-05 | t=40.3s
|
| 459 |
+
step=454300 | total=1.0112 | leaf=1.0112 | ancestor=0.0000 | lr=3.56e-05 | t=40.3s
|
| 460 |
+
step=454400 | total=1.2711 | leaf=1.2711 | ancestor=0.0000 | lr=3.55e-05 | t=40.3s
|
| 461 |
+
step=454500 | total=1.0733 | leaf=1.0733 | ancestor=0.0000 | lr=3.55e-05 | t=40.3s
|
| 462 |
+
step=454600 | total=0.9125 | leaf=0.9125 | ancestor=0.0000 | lr=3.55e-05 | t=40.3s
|
| 463 |
+
step=454700 | total=1.3884 | leaf=1.3884 | ancestor=0.0000 | lr=3.55e-05 | t=40.3s
|
| 464 |
+
step=454800 | total=1.2066 | leaf=1.2066 | ancestor=0.0000 | lr=3.55e-05 | t=40.3s
|
| 465 |
+
step=454900 | total=1.2995 | leaf=1.2995 | ancestor=0.0000 | lr=3.54e-05 | t=40.3s
|
| 466 |
+
step=455000 | total=0.9928 | leaf=0.9928 | ancestor=0.0000 | lr=3.54e-05 | t=40.3s
|
| 467 |
+
VAL | loss_leaf=1.2805 | loss_ancestor=0.0000 | loss_total=1.2805
|
| 468 |
+
step=455100 | total=1.5429 | leaf=1.5429 | ancestor=0.0000 | lr=3.54e-05 | t=47.7s
|
| 469 |
+
step=455200 | total=1.0063 | leaf=1.0063 | ancestor=0.0000 | lr=3.54e-05 | t=40.3s
|
| 470 |
+
step=455300 | total=1.1632 | leaf=1.1632 | ancestor=0.0000 | lr=3.53e-05 | t=40.3s
|
| 471 |
+
step=455400 | total=1.0120 | leaf=1.0120 | ancestor=0.0000 | lr=3.53e-05 | t=40.3s
|
| 472 |
+
step=455500 | total=1.1293 | leaf=1.1293 | ancestor=0.0000 | lr=3.53e-05 | t=40.3s
|
| 473 |
+
step=455600 | total=1.3286 | leaf=1.3286 | ancestor=0.0000 | lr=3.53e-05 | t=40.3s
|
| 474 |
+
step=455700 | total=1.3481 | leaf=1.3481 | ancestor=0.0000 | lr=3.52e-05 | t=40.3s
|
| 475 |
+
step=455800 | total=1.7148 | leaf=1.7148 | ancestor=0.0000 | lr=3.52e-05 | t=40.3s
|
| 476 |
+
step=455900 | total=1.3264 | leaf=1.3264 | ancestor=0.0000 | lr=3.52e-05 | t=40.3s
|
| 477 |
+
step=456000 | total=1.2417 | leaf=1.2417 | ancestor=0.0000 | lr=3.52e-05 | t=40.3s
|
| 478 |
+
step=456100 | total=1.0333 | leaf=1.0333 | ancestor=0.0000 | lr=3.51e-05 | t=40.3s
|
| 479 |
+
step=456200 | total=1.4334 | leaf=1.4334 | ancestor=0.0000 | lr=3.51e-05 | t=40.3s
|
| 480 |
+
step=456300 | total=1.3431 | leaf=1.3431 | ancestor=0.0000 | lr=3.51e-05 | t=40.3s
|
| 481 |
+
step=456400 | total=1.0626 | leaf=1.0626 | ancestor=0.0000 | lr=3.51e-05 | t=40.4s
|
| 482 |
+
step=456500 | total=1.3605 | leaf=1.3605 | ancestor=0.0000 | lr=3.51e-05 | t=40.3s
|
| 483 |
+
step=456600 | total=1.3519 | leaf=1.3519 | ancestor=0.0000 | lr=3.50e-05 | t=40.3s
|
| 484 |
+
step=456700 | total=1.2289 | leaf=1.2289 | ancestor=0.0000 | lr=3.50e-05 | t=40.3s
|
| 485 |
+
step=456800 | total=1.3607 | leaf=1.3607 | ancestor=0.0000 | lr=3.50e-05 | t=40.3s
|
| 486 |
+
step=456900 | total=1.3050 | leaf=1.3050 | ancestor=0.0000 | lr=3.50e-05 | t=40.3s
|
| 487 |
+
step=457000 | total=1.2144 | leaf=1.2144 | ancestor=0.0000 | lr=3.49e-05 | t=40.3s
|
| 488 |
+
step=457100 | total=1.3327 | leaf=1.3327 | ancestor=0.0000 | lr=3.49e-05 | t=40.3s
|
| 489 |
+
step=457200 | total=1.5496 | leaf=1.5496 | ancestor=0.0000 | lr=3.49e-05 | t=40.2s
|
| 490 |
+
step=457300 | total=1.1525 | leaf=1.1525 | ancestor=0.0000 | lr=3.49e-05 | t=40.3s
|
| 491 |
+
step=457400 | total=1.5401 | leaf=1.5401 | ancestor=0.0000 | lr=3.48e-05 | t=40.3s
|
| 492 |
+
step=457500 | total=1.2758 | leaf=1.2758 | ancestor=0.0000 | lr=3.48e-05 | t=40.3s
|
| 493 |
+
step=457600 | total=0.9489 | leaf=0.9489 | ancestor=0.0000 | lr=3.48e-05 | t=40.3s
|
| 494 |
+
step=457700 | total=1.6225 | leaf=1.6225 | ancestor=0.0000 | lr=3.48e-05 | t=40.3s
|
| 495 |
+
step=457800 | total=1.3073 | leaf=1.3073 | ancestor=0.0000 | lr=3.48e-05 | t=40.3s
|
| 496 |
+
step=457900 | total=1.3077 | leaf=1.3077 | ancestor=0.0000 | lr=3.47e-05 | t=40.3s
|
| 497 |
+
step=458000 | total=1.5553 | leaf=1.5553 | ancestor=0.0000 | lr=3.47e-05 | t=40.3s
|
| 498 |
+
step=458100 | total=1.4344 | leaf=1.4344 | ancestor=0.0000 | lr=3.47e-05 | t=40.3s
|
| 499 |
+
step=458200 | total=0.9570 | leaf=0.9570 | ancestor=0.0000 | lr=3.47e-05 | t=40.3s
|
| 500 |
+
step=458300 | total=1.2048 | leaf=1.2048 | ancestor=0.0000 | lr=3.46e-05 | t=40.3s
|
| 501 |
+
step=458400 | total=1.0758 | leaf=1.0758 | ancestor=0.0000 | lr=3.46e-05 | t=40.3s
|
| 502 |
+
step=458500 | total=1.4770 | leaf=1.4770 | ancestor=0.0000 | lr=3.46e-05 | t=40.3s
|
| 503 |
+
step=458600 | total=1.1195 | leaf=1.1195 | ancestor=0.0000 | lr=3.46e-05 | t=40.3s
|
| 504 |
+
step=458700 | total=1.2767 | leaf=1.2767 | ancestor=0.0000 | lr=3.46e-05 | t=40.3s
|
| 505 |
+
step=458800 | total=1.0367 | leaf=1.0367 | ancestor=0.0000 | lr=3.45e-05 | t=40.3s
|
| 506 |
+
step=458900 | total=1.4008 | leaf=1.4008 | ancestor=0.0000 | lr=3.45e-05 | t=40.3s
|
| 507 |
+
step=459000 | total=1.2898 | leaf=1.2898 | ancestor=0.0000 | lr=3.45e-05 | t=40.3s
|
| 508 |
+
step=459100 | total=1.0777 | leaf=1.0777 | ancestor=0.0000 | lr=3.45e-05 | t=40.3s
|
| 509 |
+
step=459200 | total=1.0790 | leaf=1.0790 | ancestor=0.0000 | lr=3.44e-05 | t=40.3s
|
| 510 |
+
step=459300 | total=0.9234 | leaf=0.9234 | ancestor=0.0000 | lr=3.44e-05 | t=40.3s
|
| 511 |
+
step=459400 | total=1.1893 | leaf=1.1893 | ancestor=0.0000 | lr=3.44e-05 | t=40.3s
|
| 512 |
+
step=459500 | total=1.2767 | leaf=1.2767 | ancestor=0.0000 | lr=3.44e-05 | t=40.3s
|
| 513 |
+
step=459600 | total=1.2044 | leaf=1.2044 | ancestor=0.0000 | lr=3.44e-05 | t=40.3s
|
| 514 |
+
step=459700 | total=1.0538 | leaf=1.0538 | ancestor=0.0000 | lr=3.43e-05 | t=40.3s
|
| 515 |
+
step=459800 | total=1.1889 | leaf=1.1889 | ancestor=0.0000 | lr=3.43e-05 | t=40.3s
|
| 516 |
+
step=459900 | total=1.3613 | leaf=1.3613 | ancestor=0.0000 | lr=3.43e-05 | t=40.3s
|
| 517 |
+
step=460000 | total=1.0465 | leaf=1.0465 | ancestor=0.0000 | lr=3.43e-05 | t=40.3s
|
| 518 |
+
VAL | loss_leaf=1.2419 | loss_ancestor=0.0000 | loss_total=1.2419
|
| 519 |
+
Saved checkpoint: outputs/sad_block_ar_b32_top2/ckpt_460000.pt
|
| 520 |
+
step=460100 | total=1.0895 | leaf=1.0895 | ancestor=0.0000 | lr=3.43e-05 | t=56.7s
|
| 521 |
+
step=460200 | total=1.2064 | leaf=1.2064 | ancestor=0.0000 | lr=3.42e-05 | t=40.3s
|
| 522 |
+
step=460300 | total=1.2714 | leaf=1.2714 | ancestor=0.0000 | lr=3.42e-05 | t=40.3s
|
| 523 |
+
step=460400 | total=1.3947 | leaf=1.3947 | ancestor=0.0000 | lr=3.42e-05 | t=40.3s
|
| 524 |
+
step=460500 | total=1.2918 | leaf=1.2918 | ancestor=0.0000 | lr=3.42e-05 | t=40.3s
|
| 525 |
+
step=460600 | total=1.4216 | leaf=1.4216 | ancestor=0.0000 | lr=3.41e-05 | t=40.3s
|
| 526 |
+
step=460700 | total=1.2013 | leaf=1.2013 | ancestor=0.0000 | lr=3.41e-05 | t=40.3s
|
| 527 |
+
step=460800 | total=1.1770 | leaf=1.1770 | ancestor=0.0000 | lr=3.41e-05 | t=40.3s
|
| 528 |
+
step=460900 | total=1.1890 | leaf=1.1890 | ancestor=0.0000 | lr=3.41e-05 | t=40.3s
|
| 529 |
+
step=461000 | total=1.1817 | leaf=1.1817 | ancestor=0.0000 | lr=3.41e-05 | t=40.3s
|
| 530 |
+
step=461100 | total=1.0745 | leaf=1.0745 | ancestor=0.0000 | lr=3.40e-05 | t=40.3s
|
| 531 |
+
step=461200 | total=1.1344 | leaf=1.1344 | ancestor=0.0000 | lr=3.40e-05 | t=40.3s
|
| 532 |
+
step=461300 | total=1.4840 | leaf=1.4840 | ancestor=0.0000 | lr=3.40e-05 | t=40.3s
|
| 533 |
+
step=461400 | total=1.3444 | leaf=1.3444 | ancestor=0.0000 | lr=3.40e-05 | t=40.3s
|
| 534 |
+
step=461500 | total=1.1792 | leaf=1.1792 | ancestor=0.0000 | lr=3.40e-05 | t=40.3s
|
| 535 |
+
step=461600 | total=1.1199 | leaf=1.1199 | ancestor=0.0000 | lr=3.39e-05 | t=40.3s
|
| 536 |
+
step=461700 | total=1.2163 | leaf=1.2163 | ancestor=0.0000 | lr=3.39e-05 | t=40.3s
|
| 537 |
+
step=461800 | total=1.3772 | leaf=1.3772 | ancestor=0.0000 | lr=3.39e-05 | t=40.3s
|
| 538 |
+
step=461900 | total=1.1294 | leaf=1.1294 | ancestor=0.0000 | lr=3.39e-05 | t=40.3s
|
| 539 |
+
step=462000 | total=1.1798 | leaf=1.1798 | ancestor=0.0000 | lr=3.39e-05 | t=40.3s
|
| 540 |
+
step=462100 | total=1.0373 | leaf=1.0373 | ancestor=0.0000 | lr=3.38e-05 | t=40.3s
|
| 541 |
+
step=462200 | total=1.0556 | leaf=1.0556 | ancestor=0.0000 | lr=3.38e-05 | t=40.3s
|
| 542 |
+
step=462300 | total=1.0494 | leaf=1.0494 | ancestor=0.0000 | lr=3.38e-05 | t=40.3s
|
| 543 |
+
step=462400 | total=1.5292 | leaf=1.5292 | ancestor=0.0000 | lr=3.38e-05 | t=40.3s
|
| 544 |
+
step=462500 | total=1.3763 | leaf=1.3763 | ancestor=0.0000 | lr=3.38e-05 | t=40.3s
|
| 545 |
+
step=462600 | total=1.0243 | leaf=1.0243 | ancestor=0.0000 | lr=3.37e-05 | t=40.3s
|
| 546 |
+
step=462700 | total=0.8646 | leaf=0.8646 | ancestor=0.0000 | lr=3.37e-05 | t=40.3s
|
| 547 |
+
step=462800 | total=1.3226 | leaf=1.3226 | ancestor=0.0000 | lr=3.37e-05 | t=40.3s
|
| 548 |
+
step=462900 | total=1.2515 | leaf=1.2515 | ancestor=0.0000 | lr=3.37e-05 | t=40.3s
|
| 549 |
+
step=463000 | total=0.9903 | leaf=0.9903 | ancestor=0.0000 | lr=3.37e-05 | t=40.3s
|
| 550 |
+
step=463100 | total=0.8500 | leaf=0.8500 | ancestor=0.0000 | lr=3.36e-05 | t=40.3s
|
| 551 |
+
step=463200 | total=1.0696 | leaf=1.0696 | ancestor=0.0000 | lr=3.36e-05 | t=40.3s
|
| 552 |
+
step=463300 | total=1.0906 | leaf=1.0906 | ancestor=0.0000 | lr=3.36e-05 | t=40.3s
|
| 553 |
+
step=463400 | total=1.2012 | leaf=1.2012 | ancestor=0.0000 | lr=3.36e-05 | t=40.3s
|
| 554 |
+
step=463500 | total=1.2335 | leaf=1.2335 | ancestor=0.0000 | lr=3.36e-05 | t=40.3s
|
| 555 |
+
step=463600 | total=0.9654 | leaf=0.9654 | ancestor=0.0000 | lr=3.35e-05 | t=40.3s
|
| 556 |
+
step=463700 | total=1.1793 | leaf=1.1793 | ancestor=0.0000 | lr=3.35e-05 | t=40.3s
|
| 557 |
+
step=463800 | total=1.5509 | leaf=1.5509 | ancestor=0.0000 | lr=3.35e-05 | t=40.3s
|
| 558 |
+
step=463900 | total=0.9974 | leaf=0.9974 | ancestor=0.0000 | lr=3.35e-05 | t=40.3s
|
| 559 |
+
step=464000 | total=1.0959 | leaf=1.0959 | ancestor=0.0000 | lr=3.35e-05 | t=40.3s
|
| 560 |
+
step=464100 | total=1.3032 | leaf=1.3032 | ancestor=0.0000 | lr=3.34e-05 | t=40.3s
|
| 561 |
+
step=464200 | total=1.2510 | leaf=1.2510 | ancestor=0.0000 | lr=3.34e-05 | t=40.3s
|
| 562 |
+
step=464300 | total=1.1357 | leaf=1.1357 | ancestor=0.0000 | lr=3.34e-05 | t=40.3s
|
| 563 |
+
step=464400 | total=1.3500 | leaf=1.3500 | ancestor=0.0000 | lr=3.34e-05 | t=40.3s
|
| 564 |
+
step=464500 | total=1.0143 | leaf=1.0143 | ancestor=0.0000 | lr=3.34e-05 | t=40.3s
|
| 565 |
+
step=464600 | total=1.6722 | leaf=1.6722 | ancestor=0.0000 | lr=3.34e-05 | t=40.3s
|
| 566 |
+
step=464700 | total=0.8911 | leaf=0.8911 | ancestor=0.0000 | lr=3.33e-05 | t=40.3s
|
| 567 |
+
step=464800 | total=1.1161 | leaf=1.1161 | ancestor=0.0000 | lr=3.33e-05 | t=40.3s
|
| 568 |
+
step=464900 | total=1.5329 | leaf=1.5329 | ancestor=0.0000 | lr=3.33e-05 | t=40.3s
|
| 569 |
+
step=465000 | total=1.1260 | leaf=1.1260 | ancestor=0.0000 | lr=3.33e-05 | t=40.3s
|
| 570 |
+
VAL | loss_leaf=1.2194 | loss_ancestor=0.0000 | loss_total=1.2194
|
| 571 |
+
step=465100 | total=1.3872 | leaf=1.3872 | ancestor=0.0000 | lr=3.33e-05 | t=47.6s
|
| 572 |
+
step=465200 | total=1.4603 | leaf=1.4603 | ancestor=0.0000 | lr=3.32e-05 | t=40.3s
|
| 573 |
+
step=465300 | total=1.5965 | leaf=1.5965 | ancestor=0.0000 | lr=3.32e-05 | t=40.3s
|
| 574 |
+
step=465400 | total=1.1805 | leaf=1.1805 | ancestor=0.0000 | lr=3.32e-05 | t=40.3s
|
| 575 |
+
step=465500 | total=1.3356 | leaf=1.3356 | ancestor=0.0000 | lr=3.32e-05 | t=40.3s
|
| 576 |
+
step=465600 | total=1.3428 | leaf=1.3428 | ancestor=0.0000 | lr=3.32e-05 | t=40.3s
|
| 577 |
+
step=465700 | total=1.2450 | leaf=1.2450 | ancestor=0.0000 | lr=3.31e-05 | t=40.3s
|
| 578 |
+
step=465800 | total=1.1503 | leaf=1.1503 | ancestor=0.0000 | lr=3.31e-05 | t=40.3s
|
| 579 |
+
step=465900 | total=1.5136 | leaf=1.5136 | ancestor=0.0000 | lr=3.31e-05 | t=40.3s
|
| 580 |
+
step=466000 | total=1.2782 | leaf=1.2782 | ancestor=0.0000 | lr=3.31e-05 | t=40.3s
|
| 581 |
+
step=466100 | total=1.0429 | leaf=1.0429 | ancestor=0.0000 | lr=3.31e-05 | t=40.3s
|
| 582 |
+
step=466200 | total=1.0551 | leaf=1.0551 | ancestor=0.0000 | lr=3.31e-05 | t=40.3s
|
| 583 |
+
step=466300 | total=1.0487 | leaf=1.0487 | ancestor=0.0000 | lr=3.30e-05 | t=40.3s
|
| 584 |
+
step=466400 | total=1.1359 | leaf=1.1359 | ancestor=0.0000 | lr=3.30e-05 | t=40.3s
|
| 585 |
+
step=466500 | total=1.3912 | leaf=1.3912 | ancestor=0.0000 | lr=3.30e-05 | t=40.3s
|
| 586 |
+
step=466600 | total=1.3337 | leaf=1.3337 | ancestor=0.0000 | lr=3.30e-05 | t=40.3s
|
| 587 |
+
step=466700 | total=1.1182 | leaf=1.1182 | ancestor=0.0000 | lr=3.30e-05 | t=40.3s
|
| 588 |
+
step=466800 | total=0.8895 | leaf=0.8895 | ancestor=0.0000 | lr=3.30e-05 | t=40.3s
|
| 589 |
+
step=466900 | total=1.3773 | leaf=1.3773 | ancestor=0.0000 | lr=3.29e-05 | t=40.3s
|
| 590 |
+
step=467000 | total=1.3978 | leaf=1.3978 | ancestor=0.0000 | lr=3.29e-05 | t=40.3s
|
| 591 |
+
step=467100 | total=1.2229 | leaf=1.2229 | ancestor=0.0000 | lr=3.29e-05 | t=40.3s
|
| 592 |
+
step=467200 | total=1.3018 | leaf=1.3018 | ancestor=0.0000 | lr=3.29e-05 | t=40.3s
|
| 593 |
+
step=467300 | total=1.1367 | leaf=1.1367 | ancestor=0.0000 | lr=3.29e-05 | t=40.3s
|
| 594 |
+
step=467400 | total=1.2648 | leaf=1.2648 | ancestor=0.0000 | lr=3.28e-05 | t=40.3s
|
| 595 |
+
step=467500 | total=1.2703 | leaf=1.2703 | ancestor=0.0000 | lr=3.28e-05 | t=40.3s
|
| 596 |
+
step=467600 | total=1.0693 | leaf=1.0693 | ancestor=0.0000 | lr=3.28e-05 | t=40.3s
|
| 597 |
+
step=467700 | total=1.3922 | leaf=1.3922 | ancestor=0.0000 | lr=3.28e-05 | t=40.3s
|
| 598 |
+
step=467800 | total=1.6418 | leaf=1.6418 | ancestor=0.0000 | lr=3.28e-05 | t=40.3s
|
| 599 |
+
step=467900 | total=1.5046 | leaf=1.5046 | ancestor=0.0000 | lr=3.28e-05 | t=40.3s
|
| 600 |
+
step=468000 | total=1.2261 | leaf=1.2261 | ancestor=0.0000 | lr=3.27e-05 | t=40.3s
|
| 601 |
+
step=468100 | total=1.1036 | leaf=1.1036 | ancestor=0.0000 | lr=3.27e-05 | t=40.3s
|
| 602 |
+
step=468200 | total=0.9881 | leaf=0.9881 | ancestor=0.0000 | lr=3.27e-05 | t=40.3s
|
| 603 |
+
step=468300 | total=1.4085 | leaf=1.4085 | ancestor=0.0000 | lr=3.27e-05 | t=40.3s
|
| 604 |
+
step=468400 | total=1.0510 | leaf=1.0510 | ancestor=0.0000 | lr=3.27e-05 | t=40.3s
|
| 605 |
+
step=468500 | total=1.3033 | leaf=1.3033 | ancestor=0.0000 | lr=3.27e-05 | t=40.3s
|
| 606 |
+
step=468600 | total=1.4307 | leaf=1.4307 | ancestor=0.0000 | lr=3.26e-05 | t=40.2s
|
| 607 |
+
step=468700 | total=0.9554 | leaf=0.9554 | ancestor=0.0000 | lr=3.26e-05 | t=40.3s
|
| 608 |
+
step=468800 | total=1.4369 | leaf=1.4369 | ancestor=0.0000 | lr=3.26e-05 | t=40.3s
|
| 609 |
+
step=468900 | total=1.2176 | leaf=1.2176 | ancestor=0.0000 | lr=3.26e-05 | t=40.3s
|
| 610 |
+
step=469000 | total=1.1445 | leaf=1.1445 | ancestor=0.0000 | lr=3.26e-05 | t=40.3s
|
| 611 |
+
step=469100 | total=0.9817 | leaf=0.9817 | ancestor=0.0000 | lr=3.26e-05 | t=40.3s
|
| 612 |
+
step=469200 | total=1.2272 | leaf=1.2272 | ancestor=0.0000 | lr=3.25e-05 | t=40.3s
|
| 613 |
+
step=469300 | total=0.9642 | leaf=0.9642 | ancestor=0.0000 | lr=3.25e-05 | t=40.3s
|
| 614 |
+
step=469400 | total=1.2749 | leaf=1.2749 | ancestor=0.0000 | lr=3.25e-05 | t=40.3s
|
| 615 |
+
step=469500 | total=1.1264 | leaf=1.1264 | ancestor=0.0000 | lr=3.25e-05 | t=40.3s
|
| 616 |
+
step=469600 | total=1.0131 | leaf=1.0131 | ancestor=0.0000 | lr=3.25e-05 | t=40.3s
|
| 617 |
+
step=469700 | total=1.1251 | leaf=1.1251 | ancestor=0.0000 | lr=3.25e-05 | t=40.3s
|
| 618 |
+
step=469800 | total=1.0974 | leaf=1.0974 | ancestor=0.0000 | lr=3.24e-05 | t=40.3s
|
| 619 |
+
step=469900 | total=1.1753 | leaf=1.1753 | ancestor=0.0000 | lr=3.24e-05 | t=40.3s
|
| 620 |
+
step=470000 | total=1.3797 | leaf=1.3797 | ancestor=0.0000 | lr=3.24e-05 | t=40.3s
|
| 621 |
+
VAL | loss_leaf=1.1935 | loss_ancestor=0.0000 | loss_total=1.1935
|
| 622 |
+
Saved checkpoint: outputs/sad_block_ar_b32_top2/ckpt_470000.pt
|
| 623 |
+
step=470100 | total=0.8281 | leaf=0.8281 | ancestor=0.0000 | lr=3.24e-05 | t=56.6s
|
| 624 |
+
step=470200 | total=1.3157 | leaf=1.3157 | ancestor=0.0000 | lr=3.24e-05 | t=40.3s
|
| 625 |
+
step=470300 | total=1.0804 | leaf=1.0804 | ancestor=0.0000 | lr=3.24e-05 | t=40.3s
|
| 626 |
+
step=470400 | total=1.0784 | leaf=1.0784 | ancestor=0.0000 | lr=3.23e-05 | t=40.3s
|
| 627 |
+
step=470500 | total=1.3093 | leaf=1.3093 | ancestor=0.0000 | lr=3.23e-05 | t=40.3s
|
| 628 |
+
step=470600 | total=1.0963 | leaf=1.0963 | ancestor=0.0000 | lr=3.23e-05 | t=40.3s
|
| 629 |
+
step=470700 | total=1.3988 | leaf=1.3988 | ancestor=0.0000 | lr=3.23e-05 | t=40.3s
|
| 630 |
+
step=470800 | total=1.3596 | leaf=1.3596 | ancestor=0.0000 | lr=3.23e-05 | t=40.3s
|
| 631 |
+
step=470900 | total=1.4056 | leaf=1.4056 | ancestor=0.0000 | lr=3.23e-05 | t=40.3s
|
| 632 |
+
step=471000 | total=1.4822 | leaf=1.4822 | ancestor=0.0000 | lr=3.23e-05 | t=40.3s
|
| 633 |
+
step=471100 | total=1.2821 | leaf=1.2821 | ancestor=0.0000 | lr=3.22e-05 | t=40.3s
|
| 634 |
+
step=471200 | total=1.1788 | leaf=1.1788 | ancestor=0.0000 | lr=3.22e-05 | t=40.3s
|
| 635 |
+
step=471300 | total=1.5031 | leaf=1.5031 | ancestor=0.0000 | lr=3.22e-05 | t=40.3s
|
| 636 |
+
step=471400 | total=1.3076 | leaf=1.3076 | ancestor=0.0000 | lr=3.22e-05 | t=40.3s
|
| 637 |
+
step=471500 | total=1.3963 | leaf=1.3963 | ancestor=0.0000 | lr=3.22e-05 | t=40.3s
|
| 638 |
+
step=471600 | total=1.2121 | leaf=1.2121 | ancestor=0.0000 | lr=3.22e-05 | t=40.3s
|
| 639 |
+
step=471700 | total=1.2303 | leaf=1.2303 | ancestor=0.0000 | lr=3.21e-05 | t=40.3s
|
| 640 |
+
step=471800 | total=1.1363 | leaf=1.1363 | ancestor=0.0000 | lr=3.21e-05 | t=40.3s
|
| 641 |
+
step=471900 | total=1.6114 | leaf=1.6114 | ancestor=0.0000 | lr=3.21e-05 | t=40.3s
|
| 642 |
+
step=472000 | total=1.1337 | leaf=1.1337 | ancestor=0.0000 | lr=3.21e-05 | t=40.3s
|
| 643 |
+
step=472100 | total=1.4178 | leaf=1.4178 | ancestor=0.0000 | lr=3.21e-05 | t=40.3s
|
| 644 |
+
step=472200 | total=1.1852 | leaf=1.1852 | ancestor=0.0000 | lr=3.21e-05 | t=40.3s
|
| 645 |
+
step=472300 | total=1.3056 | leaf=1.3056 | ancestor=0.0000 | lr=3.21e-05 | t=40.3s
|
| 646 |
+
step=472400 | total=1.2814 | leaf=1.2814 | ancestor=0.0000 | lr=3.20e-05 | t=40.3s
|
| 647 |
+
step=472500 | total=1.2617 | leaf=1.2617 | ancestor=0.0000 | lr=3.20e-05 | t=40.3s
|
| 648 |
+
step=472600 | total=1.5637 | leaf=1.5637 | ancestor=0.0000 | lr=3.20e-05 | t=40.3s
|
| 649 |
+
step=472700 | total=1.1234 | leaf=1.1234 | ancestor=0.0000 | lr=3.20e-05 | t=40.3s
|
| 650 |
+
step=472800 | total=1.2236 | leaf=1.2236 | ancestor=0.0000 | lr=3.20e-05 | t=40.3s
|
| 651 |
+
step=472900 | total=1.3280 | leaf=1.3280 | ancestor=0.0000 | lr=3.20e-05 | t=40.3s
|
| 652 |
+
step=473000 | total=1.2197 | leaf=1.2197 | ancestor=0.0000 | lr=3.20e-05 | t=40.3s
|
| 653 |
+
step=473100 | total=1.2109 | leaf=1.2109 | ancestor=0.0000 | lr=3.19e-05 | t=40.3s
|
| 654 |
+
step=473200 | total=1.2149 | leaf=1.2149 | ancestor=0.0000 | lr=3.19e-05 | t=40.3s
|
| 655 |
+
step=473300 | total=1.3544 | leaf=1.3544 | ancestor=0.0000 | lr=3.19e-05 | t=40.3s
|
| 656 |
+
step=473400 | total=0.8801 | leaf=0.8801 | ancestor=0.0000 | lr=3.19e-05 | t=40.3s
|
| 657 |
+
step=473500 | total=1.4509 | leaf=1.4509 | ancestor=0.0000 | lr=3.19e-05 | t=40.3s
|
| 658 |
+
step=473600 | total=0.8773 | leaf=0.8773 | ancestor=0.0000 | lr=3.19e-05 | t=40.3s
|
| 659 |
+
step=473700 | total=1.0851 | leaf=1.0851 | ancestor=0.0000 | lr=3.19e-05 | t=40.3s
|
| 660 |
+
step=473800 | total=1.1469 | leaf=1.1469 | ancestor=0.0000 | lr=3.18e-05 | t=40.3s
|
| 661 |
+
step=473900 | total=1.1391 | leaf=1.1391 | ancestor=0.0000 | lr=3.18e-05 | t=40.3s
|
| 662 |
+
step=474000 | total=1.2749 | leaf=1.2749 | ancestor=0.0000 | lr=3.18e-05 | t=40.3s
|
| 663 |
+
step=474100 | total=1.0404 | leaf=1.0404 | ancestor=0.0000 | lr=3.18e-05 | t=40.3s
|
| 664 |
+
step=474200 | total=1.3091 | leaf=1.3091 | ancestor=0.0000 | lr=3.18e-05 | t=40.3s
|
| 665 |
+
step=474300 | total=0.9925 | leaf=0.9925 | ancestor=0.0000 | lr=3.18e-05 | t=40.3s
|
| 666 |
+
step=474400 | total=1.2602 | leaf=1.2602 | ancestor=0.0000 | lr=3.18e-05 | t=40.3s
|
| 667 |
+
step=474500 | total=1.1486 | leaf=1.1486 | ancestor=0.0000 | lr=3.17e-05 | t=40.3s
|
| 668 |
+
step=474600 | total=1.1085 | leaf=1.1085 | ancestor=0.0000 | lr=3.17e-05 | t=40.3s
|
| 669 |
+
step=474700 | total=1.1736 | leaf=1.1736 | ancestor=0.0000 | lr=3.17e-05 | t=40.3s
|
| 670 |
+
step=474800 | total=1.4460 | leaf=1.4460 | ancestor=0.0000 | lr=3.17e-05 | t=40.3s
|
| 671 |
+
step=474900 | total=1.5334 | leaf=1.5334 | ancestor=0.0000 | lr=3.17e-05 | t=40.3s
|
| 672 |
+
step=475000 | total=1.3169 | leaf=1.3169 | ancestor=0.0000 | lr=3.17e-05 | t=40.3s
|
| 673 |
+
VAL | loss_leaf=1.1921 | loss_ancestor=0.0000 | loss_total=1.1921
|
| 674 |
+
step=475100 | total=1.0717 | leaf=1.0717 | ancestor=0.0000 | lr=3.17e-05 | t=47.7s
|
| 675 |
+
step=475200 | total=1.2372 | leaf=1.2372 | ancestor=0.0000 | lr=3.16e-05 | t=40.3s
|
| 676 |
+
step=475300 | total=1.1789 | leaf=1.1789 | ancestor=0.0000 | lr=3.16e-05 | t=40.3s
|
| 677 |
+
step=475400 | total=1.3897 | leaf=1.3897 | ancestor=0.0000 | lr=3.16e-05 | t=40.3s
|
| 678 |
+
step=475500 | total=1.3503 | leaf=1.3503 | ancestor=0.0000 | lr=3.16e-05 | t=40.3s
|
| 679 |
+
step=475600 | total=1.1828 | leaf=1.1828 | ancestor=0.0000 | lr=3.16e-05 | t=40.3s
|
| 680 |
+
step=475700 | total=0.8215 | leaf=0.8215 | ancestor=0.0000 | lr=3.16e-05 | t=40.3s
|
| 681 |
+
step=475800 | total=1.2010 | leaf=1.2010 | ancestor=0.0000 | lr=3.16e-05 | t=40.3s
|
| 682 |
+
step=475900 | total=1.2097 | leaf=1.2097 | ancestor=0.0000 | lr=3.16e-05 | t=40.3s
|
| 683 |
+
step=476000 | total=1.3043 | leaf=1.3043 | ancestor=0.0000 | lr=3.15e-05 | t=40.3s
|
| 684 |
+
step=476100 | total=1.3303 | leaf=1.3303 | ancestor=0.0000 | lr=3.15e-05 | t=40.3s
|
| 685 |
+
step=476200 | total=1.0480 | leaf=1.0480 | ancestor=0.0000 | lr=3.15e-05 | t=40.3s
|
| 686 |
+
step=476300 | total=1.4794 | leaf=1.4794 | ancestor=0.0000 | lr=3.15e-05 | t=40.3s
|
| 687 |
+
step=476400 | total=1.5732 | leaf=1.5732 | ancestor=0.0000 | lr=3.15e-05 | t=40.3s
|
| 688 |
+
step=476500 | total=1.5136 | leaf=1.5136 | ancestor=0.0000 | lr=3.15e-05 | t=40.3s
|
| 689 |
+
step=476600 | total=1.0726 | leaf=1.0726 | ancestor=0.0000 | lr=3.15e-05 | t=40.3s
|
| 690 |
+
step=476700 | total=1.3099 | leaf=1.3099 | ancestor=0.0000 | lr=3.15e-05 | t=40.3s
|
| 691 |
+
step=476800 | total=1.2762 | leaf=1.2762 | ancestor=0.0000 | lr=3.14e-05 | t=40.3s
|
| 692 |
+
step=476900 | total=1.1131 | leaf=1.1131 | ancestor=0.0000 | lr=3.14e-05 | t=40.3s
|
| 693 |
+
step=477000 | total=1.0690 | leaf=1.0690 | ancestor=0.0000 | lr=3.14e-05 | t=40.3s
|
| 694 |
+
step=477100 | total=1.1985 | leaf=1.1985 | ancestor=0.0000 | lr=3.14e-05 | t=40.3s
|
| 695 |
+
step=477200 | total=1.2274 | leaf=1.2274 | ancestor=0.0000 | lr=3.14e-05 | t=40.3s
|
| 696 |
+
step=477300 | total=1.3631 | leaf=1.3631 | ancestor=0.0000 | lr=3.14e-05 | t=40.3s
|
| 697 |
+
step=477400 | total=1.3889 | leaf=1.3889 | ancestor=0.0000 | lr=3.14e-05 | t=40.3s
|
| 698 |
+
step=477500 | total=1.4023 | leaf=1.4023 | ancestor=0.0000 | lr=3.14e-05 | t=40.3s
|
| 699 |
+
step=477600 | total=1.3075 | leaf=1.3075 | ancestor=0.0000 | lr=3.13e-05 | t=40.3s
|
| 700 |
+
step=477700 | total=1.2908 | leaf=1.2908 | ancestor=0.0000 | lr=3.13e-05 | t=40.3s
|
| 701 |
+
step=477800 | total=0.9994 | leaf=0.9994 | ancestor=0.0000 | lr=3.13e-05 | t=40.3s
|
| 702 |
+
step=477900 | total=1.3720 | leaf=1.3720 | ancestor=0.0000 | lr=3.13e-05 | t=40.3s
|
| 703 |
+
step=478000 | total=1.4297 | leaf=1.4297 | ancestor=0.0000 | lr=3.13e-05 | t=40.3s
|
| 704 |
+
step=478100 | total=1.2324 | leaf=1.2324 | ancestor=0.0000 | lr=3.13e-05 | t=40.3s
|
| 705 |
+
step=478200 | total=1.3844 | leaf=1.3844 | ancestor=0.0000 | lr=3.13e-05 | t=40.3s
|
| 706 |
+
step=478300 | total=1.0865 | leaf=1.0865 | ancestor=0.0000 | lr=3.13e-05 | t=40.3s
|
| 707 |
+
step=478400 | total=1.0903 | leaf=1.0903 | ancestor=0.0000 | lr=3.13e-05 | t=40.3s
|
| 708 |
+
step=478500 | total=0.5629 | leaf=0.5629 | ancestor=0.0000 | lr=3.12e-05 | t=40.3s
|
| 709 |
+
step=478600 | total=0.7747 | leaf=0.7747 | ancestor=0.0000 | lr=3.12e-05 | t=40.3s
|
| 710 |
+
step=478700 | total=1.0142 | leaf=1.0142 | ancestor=0.0000 | lr=3.12e-05 | t=40.3s
|
| 711 |
+
step=478800 | total=1.3919 | leaf=1.3919 | ancestor=0.0000 | lr=3.12e-05 | t=40.3s
|
| 712 |
+
step=478900 | total=0.9374 | leaf=0.9374 | ancestor=0.0000 | lr=3.12e-05 | t=40.3s
|
| 713 |
+
step=479000 | total=1.3680 | leaf=1.3680 | ancestor=0.0000 | lr=3.12e-05 | t=40.3s
|
| 714 |
+
step=479100 | total=1.3355 | leaf=1.3355 | ancestor=0.0000 | lr=3.12e-05 | t=40.3s
|
| 715 |
+
step=479200 | total=1.1743 | leaf=1.1743 | ancestor=0.0000 | lr=3.12e-05 | t=40.3s
|
| 716 |
+
step=479300 | total=0.8512 | leaf=0.8512 | ancestor=0.0000 | lr=3.11e-05 | t=40.3s
|
| 717 |
+
step=479400 | total=0.9796 | leaf=0.9796 | ancestor=0.0000 | lr=3.11e-05 | t=40.3s
|
| 718 |
+
step=479500 | total=1.2952 | leaf=1.2952 | ancestor=0.0000 | lr=3.11e-05 | t=40.3s
|
| 719 |
+
step=479600 | total=1.1286 | leaf=1.1286 | ancestor=0.0000 | lr=3.11e-05 | t=40.3s
|
| 720 |
+
step=479700 | total=1.1199 | leaf=1.1199 | ancestor=0.0000 | lr=3.11e-05 | t=40.3s
|
| 721 |
+
step=479800 | total=1.4124 | leaf=1.4124 | ancestor=0.0000 | lr=3.11e-05 | t=40.3s
|
| 722 |
+
step=479900 | total=0.9806 | leaf=0.9806 | ancestor=0.0000 | lr=3.11e-05 | t=40.3s
|
| 723 |
+
step=480000 | total=1.3615 | leaf=1.3615 | ancestor=0.0000 | lr=3.11e-05 | t=40.3s
|
| 724 |
+
VAL | loss_leaf=1.2410 | loss_ancestor=0.0000 | loss_total=1.2410
|
| 725 |
+
Saved checkpoint: outputs/sad_block_ar_b32_top2/ckpt_480000.pt
|
| 726 |
+
step=480100 | total=1.1244 | leaf=1.1244 | ancestor=0.0000 | lr=3.11e-05 | t=56.9s
|
| 727 |
+
step=480200 | total=1.1044 | leaf=1.1044 | ancestor=0.0000 | lr=3.11e-05 | t=40.3s
|
| 728 |
+
step=480300 | total=0.9236 | leaf=0.9236 | ancestor=0.0000 | lr=3.10e-05 | t=40.3s
|
| 729 |
+
step=480400 | total=1.3275 | leaf=1.3275 | ancestor=0.0000 | lr=3.10e-05 | t=40.3s
|
| 730 |
+
step=480500 | total=1.0416 | leaf=1.0416 | ancestor=0.0000 | lr=3.10e-05 | t=40.3s
|
| 731 |
+
step=480600 | total=1.0571 | leaf=1.0571 | ancestor=0.0000 | lr=3.10e-05 | t=40.3s
|
| 732 |
+
step=480700 | total=1.2307 | leaf=1.2307 | ancestor=0.0000 | lr=3.10e-05 | t=40.3s
|
| 733 |
+
step=480800 | total=0.9892 | leaf=0.9892 | ancestor=0.0000 | lr=3.10e-05 | t=40.3s
|
| 734 |
+
step=480900 | total=1.2168 | leaf=1.2168 | ancestor=0.0000 | lr=3.10e-05 | t=40.3s
|
| 735 |
+
step=481000 | total=1.7248 | leaf=1.7248 | ancestor=0.0000 | lr=3.10e-05 | t=40.3s
|
| 736 |
+
step=481100 | total=1.2145 | leaf=1.2145 | ancestor=0.0000 | lr=3.10e-05 | t=40.3s
|
| 737 |
+
step=481200 | total=0.9723 | leaf=0.9723 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 738 |
+
step=481300 | total=1.2684 | leaf=1.2684 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 739 |
+
step=481400 | total=1.4375 | leaf=1.4375 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 740 |
+
step=481500 | total=1.1808 | leaf=1.1808 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 741 |
+
step=481600 | total=0.9313 | leaf=0.9313 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 742 |
+
step=481700 | total=1.0295 | leaf=1.0295 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 743 |
+
step=481800 | total=1.4370 | leaf=1.4370 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 744 |
+
step=481900 | total=1.1836 | leaf=1.1836 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 745 |
+
step=482000 | total=1.5445 | leaf=1.5445 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 746 |
+
step=482100 | total=1.0373 | leaf=1.0373 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 747 |
+
step=482200 | total=1.1129 | leaf=1.1129 | ancestor=0.0000 | lr=3.09e-05 | t=40.3s
|
| 748 |
+
step=482300 | total=1.1680 | leaf=1.1680 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 749 |
+
step=482400 | total=1.5140 | leaf=1.5140 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 750 |
+
step=482500 | total=1.1126 | leaf=1.1126 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 751 |
+
step=482600 | total=1.1101 | leaf=1.1101 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 752 |
+
step=482700 | total=1.2094 | leaf=1.2094 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 753 |
+
step=482800 | total=1.0922 | leaf=1.0922 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 754 |
+
step=482900 | total=0.9776 | leaf=0.9776 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 755 |
+
step=483000 | total=1.4841 | leaf=1.4841 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 756 |
+
step=483100 | total=1.1955 | leaf=1.1955 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 757 |
+
step=483200 | total=1.2435 | leaf=1.2435 | ancestor=0.0000 | lr=3.08e-05 | t=40.3s
|
| 758 |
+
step=483300 | total=1.0243 | leaf=1.0243 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 759 |
+
step=483400 | total=0.8837 | leaf=0.8837 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 760 |
+
step=483500 | total=1.0121 | leaf=1.0121 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 761 |
+
step=483600 | total=0.9647 | leaf=0.9647 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 762 |
+
step=483700 | total=1.3023 | leaf=1.3023 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 763 |
+
step=483800 | total=1.1267 | leaf=1.1267 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 764 |
+
step=483900 | total=0.9271 | leaf=0.9271 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 765 |
+
step=484000 | total=0.8685 | leaf=0.8685 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 766 |
+
step=484100 | total=1.2233 | leaf=1.2233 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 767 |
+
step=484200 | total=1.0682 | leaf=1.0682 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 768 |
+
step=484300 | total=1.4540 | leaf=1.4540 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 769 |
+
step=484400 | total=1.1744 | leaf=1.1744 | ancestor=0.0000 | lr=3.07e-05 | t=40.3s
|
| 770 |
+
step=484500 | total=1.1404 | leaf=1.1404 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 771 |
+
step=484600 | total=0.8756 | leaf=0.8756 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 772 |
+
step=484700 | total=1.2686 | leaf=1.2686 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 773 |
+
step=484800 | total=1.4919 | leaf=1.4919 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 774 |
+
step=484900 | total=1.2911 | leaf=1.2911 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 775 |
+
step=485000 | total=1.0848 | leaf=1.0848 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 776 |
+
VAL | loss_leaf=1.2364 | loss_ancestor=0.0000 | loss_total=1.2364
|
| 777 |
+
step=485100 | total=1.2446 | leaf=1.2446 | ancestor=0.0000 | lr=3.06e-05 | t=47.6s
|
| 778 |
+
step=485200 | total=1.2006 | leaf=1.2006 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 779 |
+
step=485300 | total=1.3791 | leaf=1.3791 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 780 |
+
step=485400 | total=1.0525 | leaf=1.0525 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 781 |
+
step=485500 | total=1.2458 | leaf=1.2458 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 782 |
+
step=485600 | total=1.3566 | leaf=1.3566 | ancestor=0.0000 | lr=3.06e-05 | t=40.3s
|
| 783 |
+
step=485700 | total=1.2545 | leaf=1.2545 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 784 |
+
step=485800 | total=1.0466 | leaf=1.0466 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 785 |
+
step=485900 | total=1.3910 | leaf=1.3910 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 786 |
+
step=486000 | total=1.0124 | leaf=1.0124 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 787 |
+
step=486100 | total=1.3613 | leaf=1.3613 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 788 |
+
step=486200 | total=1.0080 | leaf=1.0080 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 789 |
+
step=486300 | total=1.0217 | leaf=1.0217 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 790 |
+
step=486400 | total=1.2746 | leaf=1.2746 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 791 |
+
step=486500 | total=1.3356 | leaf=1.3356 | ancestor=0.0000 | lr=3.05e-05 | t=40.2s
|
| 792 |
+
step=486600 | total=1.4412 | leaf=1.4412 | ancestor=0.0000 | lr=3.05e-05 | t=40.2s
|
| 793 |
+
step=486700 | total=1.4709 | leaf=1.4709 | ancestor=0.0000 | lr=3.05e-05 | t=40.2s
|
| 794 |
+
step=486800 | total=1.2203 | leaf=1.2203 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 795 |
+
step=486900 | total=1.0824 | leaf=1.0824 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 796 |
+
step=487000 | total=0.9329 | leaf=0.9329 | ancestor=0.0000 | lr=3.05e-05 | t=40.3s
|
| 797 |
+
step=487100 | total=0.9386 | leaf=0.9386 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 798 |
+
step=487200 | total=1.2990 | leaf=1.2990 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 799 |
+
step=487300 | total=0.9541 | leaf=0.9541 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 800 |
+
step=487400 | total=0.9577 | leaf=0.9577 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 801 |
+
step=487500 | total=1.6272 | leaf=1.6272 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 802 |
+
step=487600 | total=1.5025 | leaf=1.5025 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 803 |
+
step=487700 | total=1.1703 | leaf=1.1703 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 804 |
+
step=487800 | total=1.1564 | leaf=1.1564 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 805 |
+
step=487900 | total=1.1937 | leaf=1.1937 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 806 |
+
step=488000 | total=1.1178 | leaf=1.1178 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 807 |
+
step=488100 | total=1.1207 | leaf=1.1207 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 808 |
+
step=488200 | total=1.2899 | leaf=1.2899 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 809 |
+
step=488300 | total=1.1507 | leaf=1.1507 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 810 |
+
step=488400 | total=1.6816 | leaf=1.6816 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 811 |
+
step=488500 | total=1.0011 | leaf=1.0011 | ancestor=0.0000 | lr=3.04e-05 | t=40.3s
|
| 812 |
+
step=488600 | total=1.3281 | leaf=1.3281 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 813 |
+
step=488700 | total=1.3342 | leaf=1.3342 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 814 |
+
step=488800 | total=1.3828 | leaf=1.3828 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 815 |
+
step=488900 | total=1.3022 | leaf=1.3022 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 816 |
+
step=489000 | total=1.4060 | leaf=1.4060 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 817 |
+
step=489100 | total=1.0900 | leaf=1.0900 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 818 |
+
step=489200 | total=0.9213 | leaf=0.9213 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 819 |
+
step=489300 | total=0.9266 | leaf=0.9266 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 820 |
+
step=489400 | total=0.8804 | leaf=0.8804 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 821 |
+
step=489500 | total=1.4033 | leaf=1.4033 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 822 |
+
step=489600 | total=1.4494 | leaf=1.4494 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 823 |
+
step=489700 | total=1.2536 | leaf=1.2536 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 824 |
+
step=489800 | total=1.4020 | leaf=1.4020 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 825 |
+
step=489900 | total=1.3716 | leaf=1.3716 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 826 |
+
step=490000 | total=1.1236 | leaf=1.1236 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 827 |
+
VAL | loss_leaf=1.1893 | loss_ancestor=0.0000 | loss_total=1.1893
|
| 828 |
+
Saved checkpoint: outputs/sad_block_ar_b32_top2/ckpt_490000.pt
|
| 829 |
+
step=490100 | total=1.0777 | leaf=1.0777 | ancestor=0.0000 | lr=3.03e-05 | t=56.6s
|
| 830 |
+
step=490200 | total=1.5164 | leaf=1.5164 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 831 |
+
step=490300 | total=1.3328 | leaf=1.3328 | ancestor=0.0000 | lr=3.03e-05 | t=40.3s
|
| 832 |
+
step=490400 | total=1.0856 | leaf=1.0856 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 833 |
+
step=490500 | total=0.9337 | leaf=0.9337 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 834 |
+
step=490600 | total=1.2668 | leaf=1.2668 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 835 |
+
step=490700 | total=1.4018 | leaf=1.4018 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 836 |
+
step=490800 | total=1.0989 | leaf=1.0989 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 837 |
+
step=490900 | total=1.3649 | leaf=1.3649 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 838 |
+
step=491000 | total=1.2155 | leaf=1.2155 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 839 |
+
step=491100 | total=1.0642 | leaf=1.0642 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 840 |
+
step=491200 | total=1.3769 | leaf=1.3769 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 841 |
+
step=491300 | total=1.5088 | leaf=1.5088 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 842 |
+
step=491400 | total=0.9645 | leaf=0.9645 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 843 |
+
step=491500 | total=0.9416 | leaf=0.9416 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 844 |
+
step=491600 | total=1.1153 | leaf=1.1153 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 845 |
+
step=491700 | total=1.6065 | leaf=1.6065 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 846 |
+
step=491800 | total=1.6258 | leaf=1.6258 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 847 |
+
step=491900 | total=1.1287 | leaf=1.1287 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 848 |
+
step=492000 | total=1.3389 | leaf=1.3389 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 849 |
+
step=492100 | total=1.1923 | leaf=1.1923 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 850 |
+
step=492200 | total=1.0298 | leaf=1.0298 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 851 |
+
step=492300 | total=0.9409 | leaf=0.9409 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 852 |
+
step=492400 | total=1.2225 | leaf=1.2225 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 853 |
+
step=492500 | total=0.9809 | leaf=0.9809 | ancestor=0.0000 | lr=3.02e-05 | t=40.3s
|
| 854 |
+
step=492600 | total=0.9142 | leaf=0.9142 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 855 |
+
step=492700 | total=0.8419 | leaf=0.8419 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 856 |
+
step=492800 | total=1.2533 | leaf=1.2533 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 857 |
+
step=492900 | total=1.2324 | leaf=1.2324 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 858 |
+
step=493000 | total=1.4736 | leaf=1.4736 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 859 |
+
step=493100 | total=1.1599 | leaf=1.1599 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 860 |
+
step=493200 | total=1.2237 | leaf=1.2237 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 861 |
+
step=493300 | total=1.1120 | leaf=1.1120 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 862 |
+
step=493400 | total=0.9884 | leaf=0.9884 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 863 |
+
step=493500 | total=1.1557 | leaf=1.1557 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 864 |
+
step=493600 | total=1.0021 | leaf=1.0021 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 865 |
+
step=493700 | total=0.9874 | leaf=0.9874 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 866 |
+
step=493800 | total=1.1684 | leaf=1.1684 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 867 |
+
step=493900 | total=1.0083 | leaf=1.0083 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 868 |
+
step=494000 | total=1.2214 | leaf=1.2214 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 869 |
+
step=494100 | total=1.1041 | leaf=1.1041 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 870 |
+
step=494200 | total=1.2712 | leaf=1.2712 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 871 |
+
step=494300 | total=1.0128 | leaf=1.0128 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 872 |
+
step=494400 | total=1.7351 | leaf=1.7351 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 873 |
+
step=494500 | total=1.0953 | leaf=1.0953 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 874 |
+
step=494600 | total=1.3251 | leaf=1.3251 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 875 |
+
step=494700 | total=1.2632 | leaf=1.2632 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 876 |
+
step=494800 | total=1.2024 | leaf=1.2024 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 877 |
+
step=494900 | total=1.4773 | leaf=1.4773 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 878 |
+
step=495000 | total=0.8134 | leaf=0.8134 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 879 |
+
VAL | loss_leaf=1.2542 | loss_ancestor=0.0000 | loss_total=1.2542
|
| 880 |
+
step=495100 | total=1.1714 | leaf=1.1714 | ancestor=0.0000 | lr=3.01e-05 | t=47.6s
|
| 881 |
+
step=495200 | total=1.1596 | leaf=1.1596 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 882 |
+
step=495300 | total=1.3913 | leaf=1.3913 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 883 |
+
step=495400 | total=1.2536 | leaf=1.2536 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 884 |
+
step=495500 | total=1.4143 | leaf=1.4143 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 885 |
+
step=495600 | total=1.2160 | leaf=1.2160 | ancestor=0.0000 | lr=3.01e-05 | t=40.3s
|
| 886 |
+
step=495700 | total=1.2344 | leaf=1.2344 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 887 |
+
step=495800 | total=0.9286 | leaf=0.9286 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 888 |
+
step=495900 | total=1.0642 | leaf=1.0642 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 889 |
+
step=496000 | total=1.6705 | leaf=1.6705 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 890 |
+
step=496100 | total=0.9871 | leaf=0.9871 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 891 |
+
step=496200 | total=1.1816 | leaf=1.1816 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 892 |
+
step=496300 | total=1.4845 | leaf=1.4845 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 893 |
+
step=496400 | total=1.3518 | leaf=1.3518 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 894 |
+
step=496500 | total=1.5566 | leaf=1.5566 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 895 |
+
step=496600 | total=1.1714 | leaf=1.1714 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 896 |
+
step=496700 | total=1.2405 | leaf=1.2405 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 897 |
+
step=496800 | total=1.2672 | leaf=1.2672 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 898 |
+
step=496900 | total=1.0933 | leaf=1.0933 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 899 |
+
step=497000 | total=1.6770 | leaf=1.6770 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 900 |
+
step=497100 | total=1.1702 | leaf=1.1702 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 901 |
+
step=497200 | total=1.4897 | leaf=1.4897 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 902 |
+
step=497300 | total=1.0193 | leaf=1.0193 | ancestor=0.0000 | lr=3.00e-05 | t=40.2s
|
| 903 |
+
step=497400 | total=1.4697 | leaf=1.4697 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 904 |
+
step=497500 | total=1.1861 | leaf=1.1861 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 905 |
+
step=497600 | total=1.4180 | leaf=1.4180 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 906 |
+
step=497700 | total=0.9310 | leaf=0.9310 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 907 |
+
step=497800 | total=1.3913 | leaf=1.3913 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 908 |
+
step=497900 | total=1.1903 | leaf=1.1903 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 909 |
+
step=498000 | total=1.2765 | leaf=1.2765 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 910 |
+
step=498100 | total=0.9634 | leaf=0.9634 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 911 |
+
step=498200 | total=1.3592 | leaf=1.3592 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 912 |
+
step=498300 | total=1.2247 | leaf=1.2247 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 913 |
+
step=498400 | total=1.4671 | leaf=1.4671 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 914 |
+
step=498500 | total=0.9464 | leaf=0.9464 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 915 |
+
step=498600 | total=1.4500 | leaf=1.4500 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 916 |
+
step=498700 | total=1.1107 | leaf=1.1107 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 917 |
+
step=498800 | total=1.3573 | leaf=1.3573 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 918 |
+
step=498900 | total=1.5379 | leaf=1.5379 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 919 |
+
step=499000 | total=0.9553 | leaf=0.9553 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 920 |
+
step=499100 | total=1.5346 | leaf=1.5346 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 921 |
+
step=499200 | total=1.0802 | leaf=1.0802 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 922 |
+
step=499300 | total=1.2069 | leaf=1.2069 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 923 |
+
step=499400 | total=1.2949 | leaf=1.2949 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 924 |
+
step=499500 | total=1.0799 | leaf=1.0799 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 925 |
+
step=499600 | total=1.1038 | leaf=1.1038 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 926 |
+
step=499700 | total=0.9365 | leaf=0.9365 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 927 |
+
step=499800 | total=1.3700 | leaf=1.3700 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 928 |
+
step=499900 | total=1.3399 | leaf=1.3399 | ancestor=0.0000 | lr=3.00e-05 | t=40.3s
|
| 929 |
+
Saved checkpoint: outputs/sad_block_ar_b32_top2/ckpt_500000.pt
|
| 930 |
+
Training complete.
|
wandb/run-20260428_173540-bbc05m6k/logs/debug-core.log
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-28T17:35:40.293902849Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprtimbivr/port-4021893.txt","pid":4021893,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-28T17:35:40.294356336Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":4021893}
|
| 3 |
+
{"time":"2026-04-28T17:35:40.294351271Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4021893-4027855-2291999168/socket","Net":"unix"}}
|
| 4 |
+
{"time":"2026-04-28T17:35:40.475769839Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-28T17:35:40.481366757Z","level":"INFO","msg":"handleInformInit: received","streamId":"bbc05m6k","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-28T17:35:41.392919812Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"bbc05m6k","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-28T17:35:47.277835604Z","level":"INFO","msg":"connection: cancelling request","id":"1(@)","requestId":"e2tlgwc6lghm"}
|
| 8 |
+
{"time":"2026-04-29T03:44:56.315397615Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-29T03:44:56.315508197Z","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2026-04-29T03:44:56.315484662Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-29T03:44:56.315670266Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4021893-4027855-2291999168/socket","Net":"unix"}}
|
| 12 |
+
{"time":"2026-04-29T03:44:56.316439056Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 13 |
+
{"time":"2026-04-29T03:47:11.586817206Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 14 |
+
{"time":"2026-04-29T03:47:11.586887121Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 15 |
+
{"time":"2026-04-29T03:47:11.586916411Z","level":"INFO","msg":"server is closed"}
|
wandb/run-20260428_173540-bbc05m6k/logs/debug-internal.log
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
wandb/run-20260428_173540-bbc05m6k/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-28 17:35:40,258 INFO MainThread:4021893 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-28 17:35:40,258 INFO MainThread:4021893 [wandb_setup.py:_flush():81] Configure stats pid to 4021893
|
| 3 |
+
2026-04-28 17:35:40,258 INFO MainThread:4021893 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-28 17:35:40,258 INFO MainThread:4021893 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260428_173540-bbc05m6k/logs/debug.log
|
| 5 |
+
2026-04-28 17:35:40,259 INFO MainThread:4021893 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260428_173540-bbc05m6k/logs/debug-internal.log
|
| 6 |
+
2026-04-28 17:35:40,259 INFO MainThread:4021893 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-28 17:35:40,259 INFO MainThread:4021893 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 32, 'dropout': 0.0, 'num_levels': 2, 'level_sizes': [50258, 128]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128_top2_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 500000, 'lr': 0.0003, 'lr_min': 3e-05, 'warmup_steps': 2000, 'weight_decay': 0.01, 'grad_clip': 1.0, 'dtype': 'bf16', 'compile': 'default', 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_block_ar_b32_top2', 'save_dir': 'outputs/sad_block_ar_b32_top2'}, '_wandb': {}}
|
| 9 |
+
2026-04-28 17:35:40,259 INFO MainThread:4021893 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-28 17:35:40,475 INFO MainThread:4021893 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-28 17:35:40,479 INFO MainThread:4021893 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-28 17:35:40,481 INFO MainThread:4021893 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-28 17:35:40,484 INFO MainThread:4021893 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-28 17:35:42,105 INFO MainThread:4021893 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-28 17:35:42,225 INFO MainThread:4021893 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-28 17:35:42,225 INFO MainThread:4021893 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-28 17:35:42,225 INFO MainThread:4021893 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-28 17:35:42,225 INFO MainThread:4021893 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-28 17:35:42,230 INFO MainThread:4021893 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-29 03:44:56,315 INFO wandb-AsyncioManager-main:4021893 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-29 03:44:56,316 INFO wandb-AsyncioManager-main:4021893 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|
wandb/run-20260428_183729-tv4p94fz/files/output.log
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
block-ar training: 0it [00:01, ?it/s]
|
| 2 |
+
Saved checkpoint: outputs/sad_block_ar_b32_h3_mixed/ckpt_0.pt
|
| 3 |
+
Training complete.
|
wandb/run-20260428_183729-tv4p94fz/logs/debug-core.log
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-28T18:37:33.241206156Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp1xx5d9d4/port-4071139.txt","pid":4071139,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
|
| 2 |
+
{"time":"2026-04-28T18:37:33.241658379Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-4071139-4072440-3629784724/socket","Net":"unix"}}
|
| 3 |
+
{"time":"2026-04-28T18:37:33.241702989Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":4071139}
|
| 4 |
+
{"time":"2026-04-28T18:37:33.420634023Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
|
| 5 |
+
{"time":"2026-04-28T18:37:33.433066724Z","level":"INFO","msg":"handleInformInit: received","streamId":"tv4p94fz","id":"1(@)"}
|
| 6 |
+
{"time":"2026-04-28T18:37:34.293834213Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"tv4p94fz","id":"1(@)"}
|
| 7 |
+
{"time":"2026-04-28T18:37:36.717969869Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
|
| 8 |
+
{"time":"2026-04-28T18:37:36.718031626Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
|
| 9 |
+
{"time":"2026-04-28T18:37:36.71804588Z","level":"INFO","msg":"server is shutting down"}
|
| 10 |
+
{"time":"2026-04-28T18:37:36.718075922Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
|
| 11 |
+
{"time":"2026-04-28T18:37:36.718142597Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-4071139-4072440-3629784724/socket","Net":"unix"}}
|
| 12 |
+
{"time":"2026-04-28T18:37:39.70467358Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
|
| 13 |
+
{"time":"2026-04-28T18:37:39.704699051Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
|
| 14 |
+
{"time":"2026-04-28T18:37:39.704710625Z","level":"INFO","msg":"server is closed"}
|
wandb/run-20260428_183729-tv4p94fz/logs/debug-internal.log
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{"time":"2026-04-28T18:37:33.433454641Z","level":"INFO","msg":"wandb-core"}
|
| 2 |
+
{"time":"2026-04-28T18:37:33.436547314Z","level":"INFO","msg":"stream: starting","core version":"0.25.1"}
|
| 3 |
+
{"time":"2026-04-28T18:37:34.293406977Z","level":"INFO","msg":"stream: created new stream","id":"tv4p94fz"}
|
| 4 |
+
{"time":"2026-04-28T18:37:34.293569554Z","level":"INFO","msg":"handler: started"}
|
| 5 |
+
{"time":"2026-04-28T18:37:34.293815456Z","level":"INFO","msg":"stream: started"}
|
| 6 |
+
{"time":"2026-04-28T18:37:34.293861715Z","level":"INFO","msg":"sender: started"}
|
| 7 |
+
{"time":"2026-04-28T18:37:34.29385459Z","level":"INFO","msg":"writer: started","stream_id":"tv4p94fz"}
|
| 8 |
+
{"time":"2026-04-28T18:37:35.068869782Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
|
| 9 |
+
{"time":"2026-04-28T18:37:35.833729246Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 10 |
+
{"time":"2026-04-28T18:37:36.718034644Z","level":"INFO","msg":"stream: closing"}
|
| 11 |
+
{"time":"2026-04-28T18:37:39.264625771Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
|
| 12 |
+
{"time":"2026-04-28T18:37:39.264740589Z","level":"INFO","msg":"filestream: sending request","total_files":2,"console_offset":0,"console_lines":3,"uploaded_len":5,"complete":true,"exit_code":0}
|
| 13 |
+
{"time":"2026-04-28T18:37:39.703629851Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
|
| 14 |
+
{"time":"2026-04-28T18:37:39.703770374Z","level":"INFO","msg":"handler: closed"}
|
| 15 |
+
{"time":"2026-04-28T18:37:39.703816572Z","level":"INFO","msg":"sender: closed"}
|
| 16 |
+
{"time":"2026-04-28T18:37:39.703823557Z","level":"INFO","msg":"stream: closed"}
|
wandb/run-20260428_183729-tv4p94fz/logs/debug.log
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-04-28 18:37:29,201 INFO MainThread:4071139 [wandb_setup.py:_flush():81] Current SDK version is 0.25.1
|
| 2 |
+
2026-04-28 18:37:29,201 INFO MainThread:4071139 [wandb_setup.py:_flush():81] Configure stats pid to 4071139
|
| 3 |
+
2026-04-28 18:37:29,201 INFO MainThread:4071139 [wandb_setup.py:_flush():81] Loading settings from environment variables
|
| 4 |
+
2026-04-28 18:37:29,201 INFO MainThread:4071139 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /home/sunhc/diffusion/sad/wandb/run-20260428_183729-tv4p94fz/logs/debug.log
|
| 5 |
+
2026-04-28 18:37:29,201 INFO MainThread:4071139 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /home/sunhc/diffusion/sad/wandb/run-20260428_183729-tv4p94fz/logs/debug-internal.log
|
| 6 |
+
2026-04-28 18:37:29,201 INFO MainThread:4071139 [wandb_init.py:init():844] calling init triggers
|
| 7 |
+
2026-04-28 18:37:29,201 INFO MainThread:4071139 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
|
| 8 |
+
config: {'model': {'vocab_size': 50258, 'hidden_size': 768, 'n_blocks': 12, 'n_heads': 12, 'cond_dim': 128, 'max_seq_len': 512, 'block_size': 32, 'dropout': 0.0, 'num_levels': 4, 'level_sizes': [50258, 128, 32, 8]}, 'ancestor': {'lut_path': 'data/ancestor_lut_50257-128-32-8_leaf_to_each_level_topk_3-2-1_t1.0.pt', 'proto_path': 'data/hierarchy_prototypes_50257-128-32-8.pt'}, 'hierarchy': {'prototype_cache': 'data/hierarchy_prototypes_50257-128-32-8.pt'}, 'loss': {'lambda_ancestor': 0.0, 'mask_only': True}, 'training': {'seed': 0, 'batch_size': 64, 'num_steps': 0, 'lr': 0.0003, 'lr_min': 3e-05, 'warmup_steps': 2000, 'weight_decay': 0.01, 'grad_clip': 1.0, 'dtype': 'bf16', 'compile': 'default', 'log_interval': 100, 'eval_interval': 5000, 'save_interval': 10000}, 'data': {'dataset': 'openwebtext', 'seq_len': 512, 'cache_dir': 'data/owt_cache', 'num_workers': 4, 'max_train_samples': None, 'max_val_samples': 100000, 'mode': 'subsample'}, 'logging': {'use_wandb': True, 'project': 'sad_block_ar_b32_h3_mixed', 'save_dir': 'outputs/sad_block_ar_b32_h3_mixed'}, '_wandb': {}}
|
| 9 |
+
2026-04-28 18:37:29,201 INFO MainThread:4071139 [wandb_init.py:init():892] starting backend
|
| 10 |
+
2026-04-28 18:37:33,421 INFO MainThread:4071139 [wandb_init.py:init():895] sending inform_init request
|
| 11 |
+
2026-04-28 18:37:33,428 INFO MainThread:4071139 [wandb_init.py:init():903] backend started and connected
|
| 12 |
+
2026-04-28 18:37:33,432 INFO MainThread:4071139 [wandb_init.py:init():973] updated telemetry
|
| 13 |
+
2026-04-28 18:37:33,440 INFO MainThread:4071139 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
|
| 14 |
+
2026-04-28 18:37:34,942 INFO MainThread:4071139 [wandb_init.py:init():1042] starting run threads in backend
|
| 15 |
+
2026-04-28 18:37:35,063 INFO MainThread:4071139 [wandb_run.py:_console_start():2524] atexit reg
|
| 16 |
+
2026-04-28 18:37:35,063 INFO MainThread:4071139 [wandb_run.py:_redirect():2373] redirect: wrap_raw
|
| 17 |
+
2026-04-28 18:37:35,063 INFO MainThread:4071139 [wandb_run.py:_redirect():2442] Wrapping output streams.
|
| 18 |
+
2026-04-28 18:37:35,063 INFO MainThread:4071139 [wandb_run.py:_redirect():2465] Redirects installed.
|
| 19 |
+
2026-04-28 18:37:35,066 INFO MainThread:4071139 [wandb_init.py:init():1082] run started, returning control to user process
|
| 20 |
+
2026-04-28 18:37:36,718 INFO wandb-AsyncioManager-main:4071139 [service_client.py:_forward_responses():134] Reached EOF.
|
| 21 |
+
2026-04-28 18:37:36,718 INFO wandb-AsyncioManager-main:4071139 [mailbox.py:close():155] Closing mailbox, abandoning 1 handles.
|