Hugging Face's logo

rubentium
/
sparse-checkpoints

Model card Files
xet
 Community
rubentium commited on
Commit
28658dc
·
verified ·
1 Parent(s): af3e39f

EPFL final checkpoints and W&B history 1251-1300 of 1463

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +8 -0
  2. epfl_final/SparseAtt/wandb/run-20260827_163926-8ynn1we6/files/output.log +1109 -0
  3. epfl_final/SparseAtt/wandb/run-20260827_163926-8ynn1we6/files/requirements.txt +142 -0
  4. epfl_final/SparseAtt/wandb/run-20260827_163926-8ynn1we6/run-8ynn1we6.wandb +3 -0
  5. epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/files/output.log +20 -0
  6. epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/files/requirements.txt +142 -0
  7. epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/files/wandb-metadata.json +65 -0
  8. epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/logs/debug-internal.log +94 -0
  9. epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/logs/debug.log +20 -0
  10. epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/run-z1i60278.wandb +3 -0
  11. epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/files/output.log +530 -0
  12. epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/files/requirements.txt +142 -0
  13. epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/files/wandb-metadata.json +65 -0
  14. epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/logs/debug-internal.log +0 -0
  15. epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/logs/debug.log +20 -0
  16. epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/run-z1i60278.wandb +3 -0
  17. epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/files/output.log +1170 -0
  18. epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/files/requirements.txt +142 -0
  19. epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/files/wandb-metadata.json +65 -0
  20. epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/logs/debug-internal.log +0 -0
  21. epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/logs/debug.log +20 -0
  22. epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/run-z1i60278.wandb +3 -0
  23. epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/files/output.log +87 -0
  24. epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/files/requirements.txt +142 -0
  25. epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/files/wandb-metadata.json +65 -0
  26. epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/logs/debug-internal.log +440 -0
  27. epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/logs/debug.log +20 -0
  28. epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/run-8ynn1we6.wandb +3 -0
  29. epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/files/output.log +234 -0
  30. epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/files/requirements.txt +142 -0
  31. epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/files/wandb-metadata.json +65 -0
  32. epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/logs/debug-internal.log +0 -0
  33. epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/logs/debug.log +20 -0
  34. epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/run-z1i60278.wandb +3 -0
  35. epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/files/output.log +587 -0
  36. epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/files/requirements.txt +142 -0
  37. epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/files/wandb-metadata.json +65 -0
  38. epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/logs/debug-internal.log +0 -0
  39. epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/logs/debug.log +20 -0
  40. epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/run-8ynn1we6.wandb +3 -0
  41. epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/files/output.log +191 -0
  42. epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/files/requirements.txt +142 -0
  43. epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/files/wandb-metadata.json +65 -0
  44. epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/logs/debug-internal.log +0 -0
  45. epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/logs/debug.log +20 -0
  46. epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/run-8ynn1we6.wandb +3 -0
  47. epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/files/output.log +676 -0
  48. epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/files/requirements.txt +142 -0
  49. epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/files/wandb-metadata.json +65 -0
  50. epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/logs/debug-internal.log +0 -0
.gitattributes CHANGED
@@ -160,3 +160,11 @@ epfl_final/SparseAtt/wandb/run-20260827_123419-z1i60278/run-z1i60278.wandb filte
160
  epfl_final/SparseAtt/wandb/run-20260827_123421-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
161
  epfl_final/SparseAtt/wandb/run-20260827_140233-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
162
  epfl_final/SparseAtt/wandb/run-20260827_145841-z1i60278/run-z1i60278.wandb filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
160
  epfl_final/SparseAtt/wandb/run-20260827_123421-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
161
  epfl_final/SparseAtt/wandb/run-20260827_140233-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
162
  epfl_final/SparseAtt/wandb/run-20260827_145841-z1i60278/run-z1i60278.wandb filter=lfs diff=lfs merge=lfs -text
163
+ epfl_final/SparseAtt/wandb/run-20260827_163926-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
164
+ epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/run-z1i60278.wandb filter=lfs diff=lfs merge=lfs -text
165
+ epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/run-z1i60278.wandb filter=lfs diff=lfs merge=lfs -text
166
+ epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/run-z1i60278.wandb filter=lfs diff=lfs merge=lfs -text
167
+ epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
168
+ epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/run-z1i60278.wandb filter=lfs diff=lfs merge=lfs -text
169
+ epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
170
+ epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/run-8ynn1we6.wandb filter=lfs diff=lfs merge=lfs -text
epfl_final/SparseAtt/wandb/run-20260827_163926-8ynn1we6/files/output.log ADDED
@@ -0,0 +1,1109 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 18500: Loss = 2.6389, Perplexity = 14.00
3
+ wandb: WARNING Tried to log to step 18500 that is less than the current step 18521. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 18501: sparse_stochastic (configured sparse attention)
5
+ Step 18510/50462 | Loss: 2.6643 | LR: 3.58e-04 | 52277.4ms/step | 1.1 samples/s
6
+ wandb: WARNING Tried to log to step 18510 that is less than the current step 18521. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 18520/50462 | Loss: 2.6259 | LR: 3.58e-04 | 3989.8ms/step | 15.0 samples/s
8
+ wandb: WARNING Tried to log to step 18520 that is less than the current step 18521. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 18530/50462 | Loss: 2.6686 | LR: 3.58e-04 | 3994.7ms/step | 15.0 samples/s
10
+ Step 18540/50462 | Loss: 2.6322 | LR: 3.58e-04 | 4006.0ms/step | 15.0 samples/s
11
+ Step 18550/50462 | Loss: 2.6708 | LR: 3.58e-04 | 3992.8ms/step | 15.0 samples/s
12
+ Step 18560/50462 | Loss: 2.6275 | LR: 3.58e-04 | 4005.6ms/step | 15.0 samples/s
13
+ Step 18570/50462 | Loss: 2.6423 | LR: 3.58e-04 | 4005.2ms/step | 15.0 samples/s
14
+ Step 18580/50462 | Loss: 2.6655 | LR: 3.57e-04 | 4013.4ms/step | 14.9 samples/s
15
+ Step 18590/50462 | Loss: 2.6601 | LR: 3.57e-04 | 3995.5ms/step | 15.0 samples/s
16
+ Step 18600/50462 | Loss: 2.6698 | LR: 3.57e-04 | 4033.6ms/step | 14.9 samples/s
17
+ Step 18610/50462 | Loss: 2.6166 | LR: 3.57e-04 | 4036.1ms/step | 14.9 samples/s
18
+ Step 18620/50462 | Loss: 2.6345 | LR: 3.57e-04 | 4021.5ms/step | 14.9 samples/s
19
+ Step 18630/50462 | Loss: 2.6748 | LR: 3.57e-04 | 4022.4ms/step | 14.9 samples/s
20
+ Step 18640/50462 | Loss: 2.6566 | LR: 3.57e-04 | 4032.8ms/step | 14.9 samples/s
21
+ Step 18650/50462 | Loss: 2.6878 | LR: 3.56e-04 | 4033.7ms/step | 14.9 samples/s
22
+ Step 18660/50462 | Loss: 2.6329 | LR: 3.56e-04 | 4020.1ms/step | 14.9 samples/s
23
+ Step 18670/50462 | Loss: 2.6736 | LR: 3.56e-04 | 4025.8ms/step | 14.9 samples/s
24
+ Step 18680/50462 | Loss: 2.6302 | LR: 3.56e-04 | 4024.7ms/step | 14.9 samples/s
25
+ Step 18690/50462 | Loss: 2.6372 | LR: 3.56e-04 | 4011.3ms/step | 15.0 samples/s
26
+ Step 18700/50462 | Loss: 2.6551 | LR: 3.56e-04 | 4019.2ms/step | 14.9 samples/s
27
+ Step 18710/50462 | Loss: 2.6391 | LR: 3.56e-04 | 4009.2ms/step | 15.0 samples/s
28
+ Step 18720/50462 | Loss: 2.6546 | LR: 3.55e-04 | 4028.5ms/step | 14.9 samples/s
29
+ Step 18730/50462 | Loss: 2.6122 | LR: 3.55e-04 | 4021.8ms/step | 14.9 samples/s
30
+ Step 18740/50462 | Loss: 2.6474 | LR: 3.55e-04 | 4034.5ms/step | 14.9 samples/s
31
+ Step 18750/50462 | Loss: 2.6390 | LR: 3.55e-04 | 4023.9ms/step | 14.9 samples/s
32
+ Validation at step 18750: Loss = 2.6365, Perplexity = 13.96
33
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-18750
34
+ Step 18760/50462 | Loss: 2.6173 | LR: 3.55e-04 | 5949.8ms/step | 10.1 samples/s
35
+ Step 18770/50462 | Loss: 2.6279 | LR: 3.55e-04 | 4035.1ms/step | 14.9 samples/s
36
+ Step 18780/50462 | Loss: 2.6384 | LR: 3.55e-04 | 4023.4ms/step | 14.9 samples/s
37
+ Step 18790/50462 | Loss: 2.6470 | LR: 3.54e-04 | 4036.3ms/step | 14.9 samples/s
38
+ Step 18800/50462 | Loss: 2.6169 | LR: 3.54e-04 | 4025.0ms/step | 14.9 samples/s
39
+ Step 18810/50462 | Loss: 2.6928 | LR: 3.54e-04 | 4028.2ms/step | 14.9 samples/s
40
+ Step 18820/50462 | Loss: 2.6530 | LR: 3.54e-04 | 4030.4ms/step | 14.9 samples/s
41
+ Step 18830/50462 | Loss: 2.6309 | LR: 3.54e-04 | 4023.8ms/step | 14.9 samples/s
42
+ Step 18840/50462 | Loss: 2.6579 | LR: 3.54e-04 | 4033.0ms/step | 14.9 samples/s
43
+ Step 18850/50462 | Loss: 2.6426 | LR: 3.54e-04 | 4021.6ms/step | 14.9 samples/s
44
+ Step 18860/50462 | Loss: 2.6338 | LR: 3.53e-04 | 4035.7ms/step | 14.9 samples/s
45
+ Step 18870/50462 | Loss: 2.6348 | LR: 3.53e-04 | 4016.5ms/step | 14.9 samples/s
46
+ Step 18880/50462 | Loss: 2.6410 | LR: 3.53e-04 | 4010.6ms/step | 15.0 samples/s
47
+ Step 18890/50462 | Loss: 2.6323 | LR: 3.53e-04 | 4023.2ms/step | 14.9 samples/s
48
+ Step 18900/50462 | Loss: 2.6601 | LR: 3.53e-04 | 4015.1ms/step | 14.9 samples/s
49
+ Step 18910/50462 | Loss: 2.6415 | LR: 3.53e-04 | 4013.8ms/step | 14.9 samples/s
50
+ Step 18920/50462 | Loss: 2.6828 | LR: 3.53e-04 | 4021.1ms/step | 14.9 samples/s
51
+ Step 18930/50462 | Loss: 2.6442 | LR: 3.52e-04 | 4015.5ms/step | 14.9 samples/s
52
+ Step 18940/50462 | Loss: 2.6618 | LR: 3.52e-04 | 4022.3ms/step | 14.9 samples/s
53
+ Step 18950/50462 | Loss: 2.6533 | LR: 3.52e-04 | 4020.6ms/step | 14.9 samples/s
54
+ Step 18960/50462 | Loss: 2.6386 | LR: 3.52e-04 | 4015.6ms/step | 14.9 samples/s
55
+ Step 18970/50462 | Loss: 2.6006 | LR: 3.52e-04 | 4017.5ms/step | 14.9 samples/s
56
+ Step 18980/50462 | Loss: 2.6317 | LR: 3.52e-04 | 4019.4ms/step | 14.9 samples/s
57
+ Step 18990/50462 | Loss: 2.6413 | LR: 3.52e-04 | 4017.2ms/step | 14.9 samples/s
58
+ Step 19000/50462 | Loss: 2.6418 | LR: 3.51e-04 | 4017.4ms/step | 14.9 samples/s
59
+ Validation at step 19000: Loss = 2.6340, Perplexity = 13.93
60
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-19000
61
+ Step 19010/50462 | Loss: 2.6141 | LR: 3.51e-04 | 5913.7ms/step | 10.1 samples/s
62
+ Step 19020/50462 | Loss: 2.6582 | LR: 3.51e-04 | 4018.6ms/step | 14.9 samples/s
63
+ Step 19030/50462 | Loss: 2.6319 | LR: 3.51e-04 | 4017.9ms/step | 14.9 samples/s
64
+ Step 19040/50462 | Loss: 2.6287 | LR: 3.51e-04 | 4010.6ms/step | 15.0 samples/s
65
+ Step 19050/50462 | Loss: 2.6512 | LR: 3.51e-04 | 4019.0ms/step | 14.9 samples/s
66
+ Step 19060/50462 | Loss: 2.6346 | LR: 3.50e-04 | 4011.6ms/step | 15.0 samples/s
67
+ Step 19070/50462 | Loss: 2.6830 | LR: 3.50e-04 | 4016.1ms/step | 14.9 samples/s
68
+ Step 19080/50462 | Loss: 2.6788 | LR: 3.50e-04 | 4015.6ms/step | 14.9 samples/s
69
+ Step 19090/50462 | Loss: 2.6411 | LR: 3.50e-04 | 4015.3ms/step | 14.9 samples/s
70
+ Step 19100/50462 | Loss: 2.6209 | LR: 3.50e-04 | 4023.5ms/step | 14.9 samples/s
71
+ Step 19110/50462 | Loss: 2.6643 | LR: 3.50e-04 | 4011.4ms/step | 15.0 samples/s
72
+ Step 19120/50462 | Loss: 2.6579 | LR: 3.50e-04 | 4020.6ms/step | 14.9 samples/s
73
+ Step 19130/50462 | Loss: 2.6168 | LR: 3.49e-04 | 4024.4ms/step | 14.9 samples/s
74
+ Step 19140/50462 | Loss: 2.6538 | LR: 3.49e-04 | 4013.2ms/step | 15.0 samples/s
75
+ Step 19150/50462 | Loss: 2.6573 | LR: 3.49e-04 | 4034.7ms/step | 14.9 samples/s
76
+ Step 19160/50462 | Loss: 2.6371 | LR: 3.49e-04 | 4005.2ms/step | 15.0 samples/s
77
+ Step 19170/50462 | Loss: 2.6617 | LR: 3.49e-04 | 4015.2ms/step | 14.9 samples/s
78
+ Step 19180/50462 | Loss: 2.6376 | LR: 3.49e-04 | 4013.9ms/step | 14.9 samples/s
79
+ Step 19190/50462 | Loss: 2.5998 | LR: 3.49e-04 | 4020.5ms/step | 14.9 samples/s
80
+ Step 19200/50462 | Loss: 2.6378 | LR: 3.48e-04 | 4022.0ms/step | 14.9 samples/s
81
+ Step 19210/50462 | Loss: 2.6421 | LR: 3.48e-04 | 4028.2ms/step | 14.9 samples/s
82
+ Step 19220/50462 | Loss: 2.6430 | LR: 3.48e-04 | 4015.5ms/step | 14.9 samples/s
83
+ Step 19230/50462 | Loss: 2.6266 | LR: 3.48e-04 | 4006.2ms/step | 15.0 samples/s
84
+ Step 19240/50462 | Loss: 2.6589 | LR: 3.48e-04 | 4006.1ms/step | 15.0 samples/s
85
+ Step 19250/50462 | Loss: 2.6737 | LR: 3.48e-04 | 4013.4ms/step | 15.0 samples/s
86
+ Validation at step 19250: Loss = 2.6317, Perplexity = 13.90
87
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-19250
88
+ Step 19260/50462 | Loss: 2.6678 | LR: 3.48e-04 | 5914.9ms/step | 10.1 samples/s
89
+ Step 19270/50462 | Loss: 2.6350 | LR: 3.47e-04 | 3999.9ms/step | 15.0 samples/s
90
+ Step 19280/50462 | Loss: 2.6335 | LR: 3.47e-04 | 3994.7ms/step | 15.0 samples/s
91
+ Step 19290/50462 | Loss: 2.6327 | LR: 3.47e-04 | 4003.7ms/step | 15.0 samples/s
92
+ Step 19300/50462 | Loss: 2.6231 | LR: 3.47e-04 | 4007.2ms/step | 15.0 samples/s
93
+ Step 19310/50462 | Loss: 2.6618 | LR: 3.47e-04 | 4020.5ms/step | 14.9 samples/s
94
+ Step 19320/50462 | Loss: 2.6668 | LR: 3.47e-04 | 4017.6ms/step | 14.9 samples/s
95
+ Step 19330/50462 | Loss: 2.6280 | LR: 3.47e-04 | 4020.5ms/step | 14.9 samples/s
96
+ Step 19340/50462 | Loss: 2.6047 | LR: 3.46e-04 | 4004.8ms/step | 15.0 samples/s
97
+ Step 19350/50462 | Loss: 2.6717 | LR: 3.46e-04 | 4008.3ms/step | 15.0 samples/s
98
+ Step 19360/50462 | Loss: 2.6392 | LR: 3.46e-04 | 4003.1ms/step | 15.0 samples/s
99
+ Step 19370/50462 | Loss: 2.6679 | LR: 3.46e-04 | 4004.9ms/step | 15.0 samples/s
100
+ Step 19380/50462 | Loss: 2.6566 | LR: 3.46e-04 | 4012.3ms/step | 15.0 samples/s
101
+ Step 19390/50462 | Loss: 2.6530 | LR: 3.46e-04 | 4011.1ms/step | 15.0 samples/s
102
+ Step 19400/50462 | Loss: 2.6254 | LR: 3.46e-04 | 4004.5ms/step | 15.0 samples/s
103
+ Step 19410/50462 | Loss: 2.6302 | LR: 3.45e-04 | 4014.7ms/step | 14.9 samples/s
104
+ Step 19420/50462 | Loss: 2.6381 | LR: 3.45e-04 | 4018.8ms/step | 14.9 samples/s
105
+ Step 19430/50462 | Loss: 2.6430 | LR: 3.45e-04 | 4010.1ms/step | 15.0 samples/s
106
+ Step 19440/50462 | Loss: 2.6403 | LR: 3.45e-04 | 4018.8ms/step | 14.9 samples/s
107
+ Step 19450/50462 | Loss: 2.6222 | LR: 3.45e-04 | 4019.0ms/step | 14.9 samples/s
108
+ Step 19460/50462 | Loss: 2.6196 | LR: 3.45e-04 | 4015.0ms/step | 14.9 samples/s
109
+ Step 19470/50462 | Loss: 2.6332 | LR: 3.45e-04 | 4018.2ms/step | 14.9 samples/s
110
+ Step 19480/50462 | Loss: 2.6343 | LR: 3.44e-04 | 4011.8ms/step | 15.0 samples/s
111
+ Step 19490/50462 | Loss: 2.5977 | LR: 3.44e-04 | 4025.6ms/step | 14.9 samples/s
112
+ Step 19500/50462 | Loss: 2.6276 | LR: 3.44e-04 | 4025.8ms/step | 14.9 samples/s
113
+ Validation at step 19500: Loss = 2.6302, Perplexity = 13.88
114
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-19500
115
+ Step 19510/50462 | Loss: 2.6717 | LR: 3.44e-04 | 5925.4ms/step | 10.1 samples/s
116
+ Step 19520/50462 | Loss: 2.6294 | LR: 3.44e-04 | 4006.3ms/step | 15.0 samples/s
117
+ Step 19530/50462 | Loss: 2.6318 | LR: 3.44e-04 | 4021.8ms/step | 14.9 samples/s
118
+ Step 19540/50462 | Loss: 2.6595 | LR: 3.44e-04 | 4021.6ms/step | 14.9 samples/s
119
+ Step 19550/50462 | Loss: 2.6410 | LR: 3.43e-04 | 4021.5ms/step | 14.9 samples/s
120
+ Step 19560/50462 | Loss: 2.6446 | LR: 3.43e-04 | 4010.7ms/step | 15.0 samples/s
121
+ Step 19570/50462 | Loss: 2.6194 | LR: 3.43e-04 | 4012.6ms/step | 15.0 samples/s
122
+ Step 19580/50462 | Loss: 2.6526 | LR: 3.43e-04 | 4022.9ms/step | 14.9 samples/s
123
+ Step 19590/50462 | Loss: 2.6699 | LR: 3.43e-04 | 4014.2ms/step | 14.9 samples/s
124
+ Step 19600/50462 | Loss: 2.6666 | LR: 3.43e-04 | 4020.8ms/step | 14.9 samples/s
125
+ Step 19610/50462 | Loss: 2.6399 | LR: 3.42e-04 | 4011.7ms/step | 15.0 samples/s
126
+ Step 19620/50462 | Loss: 2.6274 | LR: 3.42e-04 | 4018.6ms/step | 14.9 samples/s
127
+ Step 19630/50462 | Loss: 2.6456 | LR: 3.42e-04 | 4023.4ms/step | 14.9 samples/s
128
+ Step 19640/50462 | Loss: 2.6330 | LR: 3.42e-04 | 4017.8ms/step | 14.9 samples/s
129
+ Step 19650/50462 | Loss: 2.6440 | LR: 3.42e-04 | 4024.9ms/step | 14.9 samples/s
130
+ Step 19660/50462 | Loss: 2.6137 | LR: 3.42e-04 | 4025.9ms/step | 14.9 samples/s
131
+ Step 19670/50462 | Loss: 2.6452 | LR: 3.42e-04 | 4018.2ms/step | 14.9 samples/s
132
+ Step 19680/50462 | Loss: 2.6283 | LR: 3.41e-04 | 4021.0ms/step | 14.9 samples/s
133
+ Step 19690/50462 | Loss: 2.6286 | LR: 3.41e-04 | 4031.2ms/step | 14.9 samples/s
134
+ Step 19700/50462 | Loss: 2.6438 | LR: 3.41e-04 | 4005.9ms/step | 15.0 samples/s
135
+ Step 19710/50462 | Loss: 2.6610 | LR: 3.41e-04 | 4023.2ms/step | 14.9 samples/s
136
+ Step 19720/50462 | Loss: 2.6239 | LR: 3.41e-04 | 4006.0ms/step | 15.0 samples/s
137
+ Step 19730/50462 | Loss: 2.6205 | LR: 3.41e-04 | 4009.7ms/step | 15.0 samples/s
138
+ Step 19740/50462 | Loss: 2.6315 | LR: 3.41e-04 | 4007.4ms/step | 15.0 samples/s
139
+ Step 19750/50462 | Loss: 2.6700 | LR: 3.40e-04 | 4014.5ms/step | 14.9 samples/s
140
+ Validation at step 19750: Loss = 2.6260, Perplexity = 13.82
141
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-19750
142
+ Step 19760/50462 | Loss: 2.6124 | LR: 3.40e-04 | 5929.9ms/step | 10.1 samples/s
143
+ Step 19770/50462 | Loss: 2.6481 | LR: 3.40e-04 | 4009.0ms/step | 15.0 samples/s
144
+ Step 19780/50462 | Loss: 2.6344 | LR: 3.40e-04 | 4019.2ms/step | 14.9 samples/s
145
+ Step 19790/50462 | Loss: 2.6706 | LR: 3.40e-04 | 4015.7ms/step | 14.9 samples/s
146
+ Step 19800/50462 | Loss: 2.6234 | LR: 3.40e-04 | 4009.1ms/step | 15.0 samples/s
147
+ Step 19810/50462 | Loss: 2.6170 | LR: 3.40e-04 | 4018.4ms/step | 14.9 samples/s
148
+ Step 19820/50462 | Loss: 2.6692 | LR: 3.39e-04 | 4012.9ms/step | 15.0 samples/s
149
+ Step 19830/50462 | Loss: 2.6121 | LR: 3.39e-04 | 4005.8ms/step | 15.0 samples/s
150
+ Step 19840/50462 | Loss: 2.6464 | LR: 3.39e-04 | 4010.1ms/step | 15.0 samples/s
151
+ Step 19850/50462 | Loss: 2.6817 | LR: 3.39e-04 | 4016.1ms/step | 14.9 samples/s
152
+ Step 19860/50462 | Loss: 2.5996 | LR: 3.39e-04 | 4008.2ms/step | 15.0 samples/s
153
+ Step 19870/50462 | Loss: 2.6218 | LR: 3.39e-04 | 4016.3ms/step | 14.9 samples/s
154
+ Step 19880/50462 | Loss: 2.6380 | LR: 3.39e-04 | 4015.3ms/step | 14.9 samples/s
155
+ Step 19890/50462 | Loss: 2.6561 | LR: 3.38e-04 | 4015.0ms/step | 14.9 samples/s
156
+ Step 19900/50462 | Loss: 2.6387 | LR: 3.38e-04 | 4015.0ms/step | 14.9 samples/s
157
+ Step 19910/50462 | Loss: 2.6296 | LR: 3.38e-04 | 4020.3ms/step | 14.9 samples/s
158
+ Step 19920/50462 | Loss: 2.6369 | LR: 3.38e-04 | 4014.0ms/step | 14.9 samples/s
159
+ Step 19930/50462 | Loss: 2.6273 | LR: 3.38e-04 | 4010.2ms/step | 15.0 samples/s
160
+ Step 19940/50462 | Loss: 2.6548 | LR: 3.38e-04 | 4012.6ms/step | 15.0 samples/s
161
+ Step 19950/50462 | Loss: 2.6452 | LR: 3.37e-04 | 4006.3ms/step | 15.0 samples/s
162
+ Step 19960/50462 | Loss: 2.6349 | LR: 3.37e-04 | 4014.0ms/step | 14.9 samples/s
163
+ Step 19970/50462 | Loss: 2.6351 | LR: 3.37e-04 | 4004.5ms/step | 15.0 samples/s
164
+ Step 19980/50462 | Loss: 2.6499 | LR: 3.37e-04 | 4001.1ms/step | 15.0 samples/s
165
+ Step 19990/50462 | Loss: 2.6250 | LR: 3.37e-04 | 4008.9ms/step | 15.0 samples/s
166
+ Step 20000/50462 | Loss: 2.6262 | LR: 3.37e-04 | 4011.1ms/step | 15.0 samples/s
167
+ Validation at step 20000: Loss = 2.6235, Perplexity = 13.78
168
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-20000
169
+ Step 20010/50462 | Loss: 2.6525 | LR: 3.37e-04 | 5914.5ms/step | 10.1 samples/s
170
+ Step 20020/50462 | Loss: 2.6511 | LR: 3.36e-04 | 4008.5ms/step | 15.0 samples/s
171
+ Step 20030/50462 | Loss: 2.6379 | LR: 3.36e-04 | 4011.1ms/step | 15.0 samples/s
172
+ Step 20040/50462 | Loss: 2.6188 | LR: 3.36e-04 | 4003.6ms/step | 15.0 samples/s
173
+ Step 20050/50462 | Loss: 2.6450 | LR: 3.36e-04 | 3991.7ms/step | 15.0 samples/s
174
+ Step 20060/50462 | Loss: 2.6345 | LR: 3.36e-04 | 4018.8ms/step | 14.9 samples/s
175
+ Step 20070/50462 | Loss: 2.6438 | LR: 3.36e-04 | 3999.9ms/step | 15.0 samples/s
176
+ Step 20080/50462 | Loss: 2.6234 | LR: 3.36e-04 | 4013.5ms/step | 14.9 samples/s
177
+ Step 20090/50462 | Loss: 2.6089 | LR: 3.35e-04 | 4005.6ms/step | 15.0 samples/s
178
+ Step 20100/50462 | Loss: 2.6505 | LR: 3.35e-04 | 4011.6ms/step | 15.0 samples/s
179
+ Step 20110/50462 | Loss: 2.6243 | LR: 3.35e-04 | 4004.8ms/step | 15.0 samples/s
180
+ Step 20120/50462 | Loss: 2.6663 | LR: 3.35e-04 | 4005.5ms/step | 15.0 samples/s
181
+ Step 20130/50462 | Loss: 2.6281 | LR: 3.35e-04 | 4001.9ms/step | 15.0 samples/s
182
+ Step 20140/50462 | Loss: 2.5954 | LR: 3.35e-04 | 4000.0ms/step | 15.0 samples/s
183
+ Step 20150/50462 | Loss: 2.6343 | LR: 3.35e-04 | 4000.7ms/step | 15.0 samples/s
184
+ Step 20160/50462 | Loss: 2.6569 | LR: 3.34e-04 | 4008.5ms/step | 15.0 samples/s
185
+ Step 20170/50462 | Loss: 2.6478 | LR: 3.34e-04 | 4008.3ms/step | 15.0 samples/s
186
+ Step 20180/50462 | Loss: 2.6469 | LR: 3.34e-04 | 4008.2ms/step | 15.0 samples/s
187
+ Step 20190/50462 | Loss: 2.5980 | LR: 3.34e-04 | 4010.0ms/step | 15.0 samples/s
188
+ Step 20200/50462 | Loss: 2.6288 | LR: 3.34e-04 | 4018.9ms/step | 14.9 samples/s
189
+ Step 20210/50462 | Loss: 2.6125 | LR: 3.34e-04 | 4007.5ms/step | 15.0 samples/s
190
+ Step 20220/50462 | Loss: 2.6596 | LR: 3.33e-04 | 4010.2ms/step | 15.0 samples/s
191
+ Step 20230/50462 | Loss: 2.6436 | LR: 3.33e-04 | 4013.1ms/step | 15.0 samples/s
192
+ Step 20240/50462 | Loss: 2.6295 | LR: 3.33e-04 | 4017.2ms/step | 14.9 samples/s
193
+ Step 20250/50462 | Loss: 2.6532 | LR: 3.33e-04 | 4006.3ms/step | 15.0 samples/s
194
+ Validation at step 20250: Loss = 2.6213, Perplexity = 13.75
195
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-20250
196
+ Step 20260/50462 | Loss: 2.6556 | LR: 3.33e-04 | 5925.9ms/step | 10.1 samples/s
197
+ Step 20270/50462 | Loss: 2.6097 | LR: 3.33e-04 | 4002.1ms/step | 15.0 samples/s
198
+ Step 20280/50462 | Loss: 2.6397 | LR: 3.33e-04 | 4008.2ms/step | 15.0 samples/s
199
+ Step 20290/50462 | Loss: 2.6172 | LR: 3.32e-04 | 4008.3ms/step | 15.0 samples/s
200
+ Step 20300/50462 | Loss: 2.6254 | LR: 3.32e-04 | 4011.5ms/step | 15.0 samples/s
201
+ Step 20310/50462 | Loss: 2.5995 | LR: 3.32e-04 | 4006.1ms/step | 15.0 samples/s
202
+ Step 20320/50462 | Loss: 2.6405 | LR: 3.32e-04 | 4002.9ms/step | 15.0 samples/s
203
+ Step 20330/50462 | Loss: 2.6580 | LR: 3.32e-04 | 4011.2ms/step | 15.0 samples/s
204
+ Step 20340/50462 | Loss: 2.6144 | LR: 3.32e-04 | 4020.9ms/step | 14.9 samples/s
205
+ Step 20350/50462 | Loss: 2.6290 | LR: 3.32e-04 | 3997.5ms/step | 15.0 samples/s
206
+ Step 20360/50462 | Loss: 2.6079 | LR: 3.31e-04 | 3999.1ms/step | 15.0 samples/s
207
+ Step 20370/50462 | Loss: 2.6209 | LR: 3.31e-04 | 4006.3ms/step | 15.0 samples/s
208
+ Step 20380/50462 | Loss: 2.6494 | LR: 3.31e-04 | 4005.3ms/step | 15.0 samples/s
209
+ Step 20390/50462 | Loss: 2.6424 | LR: 3.31e-04 | 3997.7ms/step | 15.0 samples/s
210
+ Step 20400/50462 | Loss: 2.6355 | LR: 3.31e-04 | 4001.3ms/step | 15.0 samples/s
211
+ Step 20410/50462 | Loss: 2.6353 | LR: 3.31e-04 | 4005.1ms/step | 15.0 samples/s
212
+ Step 20420/50462 | Loss: 2.6157 | LR: 3.30e-04 | 4012.1ms/step | 15.0 samples/s
213
+ Step 20430/50462 | Loss: 2.6418 | LR: 3.30e-04 | 4009.0ms/step | 15.0 samples/s
214
+ Step 20440/50462 | Loss: 2.6432 | LR: 3.30e-04 | 4005.5ms/step | 15.0 samples/s
215
+ Step 20450/50462 | Loss: 2.6344 | LR: 3.30e-04 | 4000.8ms/step | 15.0 samples/s
216
+ Step 20460/50462 | Loss: 2.6084 | LR: 3.30e-04 | 4019.5ms/step | 14.9 samples/s
217
+ Step 20470/50462 | Loss: 2.6154 | LR: 3.30e-04 | 4016.9ms/step | 14.9 samples/s
218
+ Step 20480/50462 | Loss: 2.6195 | LR: 3.30e-04 | 4007.4ms/step | 15.0 samples/s
219
+ Step 20490/50462 | Loss: 2.6401 | LR: 3.29e-04 | 4012.1ms/step | 15.0 samples/s
220
+ Step 20500/50462 | Loss: 2.6479 | LR: 3.29e-04 | 4007.2ms/step | 15.0 samples/s
221
+ Validation at step 20500: Loss = 2.6180, Perplexity = 13.71
222
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-20500
223
+ Step 20510/50462 | Loss: 2.6510 | LR: 3.29e-04 | 5975.2ms/step | 10.0 samples/s
224
+ Step 20520/50462 | Loss: 2.5941 | LR: 3.29e-04 | 4010.9ms/step | 15.0 samples/s
225
+ Step 20530/50462 | Loss: 2.6363 | LR: 3.29e-04 | 4004.1ms/step | 15.0 samples/s
226
+ Step 20540/50462 | Loss: 2.6331 | LR: 3.29e-04 | 4000.0ms/step | 15.0 samples/s
227
+ Step 20550/50462 | Loss: 2.6145 | LR: 3.29e-04 | 4013.4ms/step | 14.9 samples/s
228
+ Step 20560/50462 | Loss: 2.6335 | LR: 3.28e-04 | 4011.4ms/step | 15.0 samples/s
229
+ Step 20570/50462 | Loss: 2.6310 | LR: 3.28e-04 | 3999.8ms/step | 15.0 samples/s
230
+ Step 20580/50462 | Loss: 2.6300 | LR: 3.28e-04 | 4009.1ms/step | 15.0 samples/s
231
+ Step 20590/50462 | Loss: 2.6375 | LR: 3.28e-04 | 4003.2ms/step | 15.0 samples/s
232
+ Step 20600/50462 | Loss: 2.6206 | LR: 3.28e-04 | 4009.8ms/step | 15.0 samples/s
233
+ Step 20610/50462 | Loss: 2.6491 | LR: 3.28e-04 | 4014.4ms/step | 14.9 samples/s
234
+ Step 20620/50462 | Loss: 2.6234 | LR: 3.27e-04 | 3996.0ms/step | 15.0 samples/s
235
+ Step 20630/50462 | Loss: 2.6171 | LR: 3.27e-04 | 4014.0ms/step | 14.9 samples/s
236
+ Step 20640/50462 | Loss: 2.6292 | LR: 3.27e-04 | 4008.6ms/step | 15.0 samples/s
237
+ Step 20650/50462 | Loss: 2.6510 | LR: 3.27e-04 | 3993.6ms/step | 15.0 samples/s
238
+ Step 20660/50462 | Loss: 2.6303 | LR: 3.27e-04 | 4005.3ms/step | 15.0 samples/s
239
+ Step 20670/50462 | Loss: 2.6390 | LR: 3.27e-04 | 3999.0ms/step | 15.0 samples/s
240
+ Step 20680/50462 | Loss: 2.6194 | LR: 3.27e-04 | 3998.7ms/step | 15.0 samples/s
241
+ Step 20690/50462 | Loss: 2.6174 | LR: 3.26e-04 | 4000.6ms/step | 15.0 samples/s
242
+ Step 20700/50462 | Loss: 2.6218 | LR: 3.26e-04 | 4002.7ms/step | 15.0 samples/s
243
+ Step 20710/50462 | Loss: 2.6608 | LR: 3.26e-04 | 4003.9ms/step | 15.0 samples/s
244
+ Step 20720/50462 | Loss: 2.6196 | LR: 3.26e-04 | 4013.8ms/step | 14.9 samples/s
245
+ Step 20730/50462 | Loss: 2.6310 | LR: 3.26e-04 | 4011.0ms/step | 15.0 samples/s
246
+ Step 20740/50462 | Loss: 2.6121 | LR: 3.26e-04 | 4003.5ms/step | 15.0 samples/s
247
+ Step 20750/50462 | Loss: 2.6776 | LR: 3.26e-04 | 4011.6ms/step | 15.0 samples/s
248
+ Validation at step 20750: Loss = 2.6158, Perplexity = 13.68
249
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-20750
250
+ Step 20760/50462 | Loss: 2.6520 | LR: 3.25e-04 | 5899.3ms/step | 10.2 samples/s
251
+ Step 20770/50462 | Loss: 2.6207 | LR: 3.25e-04 | 3997.3ms/step | 15.0 samples/s
252
+ Step 20780/50462 | Loss: 2.6147 | LR: 3.25e-04 | 4005.3ms/step | 15.0 samples/s
253
+ Step 20790/50462 | Loss: 2.6244 | LR: 3.25e-04 | 3997.5ms/step | 15.0 samples/s
254
+ Step 20800/50462 | Loss: 2.6431 | LR: 3.25e-04 | 3991.3ms/step | 15.0 samples/s
255
+ Step 20810/50462 | Loss: 2.6276 | LR: 3.25e-04 | 3997.6ms/step | 15.0 samples/s
256
+ Step 20820/50462 | Loss: 2.6253 | LR: 3.24e-04 | 4002.3ms/step | 15.0 samples/s
257
+ Step 20830/50462 | Loss: 2.6361 | LR: 3.24e-04 | 4004.5ms/step | 15.0 samples/s
258
+ Step 20840/50462 | Loss: 2.6385 | LR: 3.24e-04 | 4006.5ms/step | 15.0 samples/s
259
+ Step 20850/50462 | Loss: 2.6475 | LR: 3.24e-04 | 4007.1ms/step | 15.0 samples/s
260
+ Step 20860/50462 | Loss: 2.6530 | LR: 3.24e-04 | 4015.0ms/step | 14.9 samples/s
261
+ Step 20870/50462 | Loss: 2.6556 | LR: 3.24e-04 | 4008.3ms/step | 15.0 samples/s
262
+ Step 20880/50462 | Loss: 2.6326 | LR: 3.24e-04 | 4001.0ms/step | 15.0 samples/s
263
+ Step 20890/50462 | Loss: 2.6427 | LR: 3.23e-04 | 4003.7ms/step | 15.0 samples/s
264
+ Step 20900/50462 | Loss: 2.6179 | LR: 3.23e-04 | 4002.0ms/step | 15.0 samples/s
265
+ Step 20910/50462 | Loss: 2.6563 | LR: 3.23e-04 | 3999.4ms/step | 15.0 samples/s
266
+ Step 20920/50462 | Loss: 2.6224 | LR: 3.23e-04 | 4009.4ms/step | 15.0 samples/s
267
+ Step 20930/50462 | Loss: 2.6344 | LR: 3.23e-04 | 4007.1ms/step | 15.0 samples/s
268
+ Step 20940/50462 | Loss: 2.6838 | LR: 3.23e-04 | 4000.9ms/step | 15.0 samples/s
269
+ Step 20950/50462 | Loss: 2.5799 | LR: 3.23e-04 | 3995.8ms/step | 15.0 samples/s
270
+ Step 20960/50462 | Loss: 2.6385 | LR: 3.22e-04 | 4015.9ms/step | 14.9 samples/s
271
+ Step 20970/50462 | Loss: 2.6148 | LR: 3.22e-04 | 4007.1ms/step | 15.0 samples/s
272
+ Step 20980/50462 | Loss: 2.6134 | LR: 3.22e-04 | 3993.7ms/step | 15.0 samples/s
273
+ Step 20990/50462 | Loss: 2.6015 | LR: 3.22e-04 | 4019.9ms/step | 14.9 samples/s
274
+ Step 21000/50462 | Loss: 2.6275 | LR: 3.22e-04 | 3998.2ms/step | 15.0 samples/s
275
+ Validation at step 21000: Loss = 2.6143, Perplexity = 13.66
276
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-21000
277
+ Step 21010/50462 | Loss: 2.6337 | LR: 3.22e-04 | 5912.7ms/step | 10.1 samples/s
278
+ Step 21020/50462 | Loss: 2.6116 | LR: 3.21e-04 | 3995.3ms/step | 15.0 samples/s
279
+ Step 21030/50462 | Loss: 2.6404 | LR: 3.21e-04 | 4004.5ms/step | 15.0 samples/s
280
+ Step 21040/50462 | Loss: 2.6264 | LR: 3.21e-04 | 3998.3ms/step | 15.0 samples/s
281
+ Step 21050/50462 | Loss: 2.6147 | LR: 3.21e-04 | 4010.0ms/step | 15.0 samples/s
282
+ Step 21060/50462 | Loss: 2.6030 | LR: 3.21e-04 | 3995.7ms/step | 15.0 samples/s
283
+ Step 21070/50462 | Loss: 2.6223 | LR: 3.21e-04 | 4004.7ms/step | 15.0 samples/s
284
+ Step 21080/50462 | Loss: 2.6186 | LR: 3.21e-04 | 4002.4ms/step | 15.0 samples/s
285
+ Step 21090/50462 | Loss: 2.6312 | LR: 3.20e-04 | 4012.5ms/step | 15.0 samples/s
286
+ Step 21100/50462 | Loss: 2.6250 | LR: 3.20e-04 | 3989.9ms/step | 15.0 samples/s
287
+ Step 21110/50462 | Loss: 2.6445 | LR: 3.20e-04 | 4005.5ms/step | 15.0 samples/s
288
+ Step 21120/50462 | Loss: 2.6439 | LR: 3.20e-04 | 3997.7ms/step | 15.0 samples/s
289
+ Step 21130/50462 | Loss: 2.6111 | LR: 3.20e-04 | 4009.6ms/step | 15.0 samples/s
290
+ Step 21140/50462 | Loss: 2.6416 | LR: 3.20e-04 | 4004.3ms/step | 15.0 samples/s
291
+ Step 21150/50462 | Loss: 2.6655 | LR: 3.19e-04 | 4006.1ms/step | 15.0 samples/s
292
+ Step 21160/50462 | Loss: 2.6440 | LR: 3.19e-04 | 4009.1ms/step | 15.0 samples/s
293
+ Step 21170/50462 | Loss: 2.6246 | LR: 3.19e-04 | 4006.7ms/step | 15.0 samples/s
294
+ Step 21180/50462 | Loss: 2.6195 | LR: 3.19e-04 | 4001.6ms/step | 15.0 samples/s
295
+ Step 21190/50462 | Loss: 2.6030 | LR: 3.19e-04 | 3995.4ms/step | 15.0 samples/s
296
+ Step 21200/50462 | Loss: 2.6561 | LR: 3.19e-04 | 4009.1ms/step | 15.0 samples/s
297
+ Step 21210/50462 | Loss: 2.6044 | LR: 3.19e-04 | 4001.8ms/step | 15.0 samples/s
298
+ Step 21220/50462 | Loss: 2.6511 | LR: 3.18e-04 | 4001.8ms/step | 15.0 samples/s
299
+ Step 21230/50462 | Loss: 2.6324 | LR: 3.18e-04 | 3994.0ms/step | 15.0 samples/s
300
+ Step 21240/50462 | Loss: 2.6102 | LR: 3.18e-04 | 3997.5ms/step | 15.0 samples/s
301
+ Step 21250/50462 | Loss: 2.6309 | LR: 3.18e-04 | 3984.8ms/step | 15.1 samples/s
302
+ Validation at step 21250: Loss = 2.6122, Perplexity = 13.63
303
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-21250
304
+ Step 21260/50462 | Loss: 2.5915 | LR: 3.18e-04 | 5915.1ms/step | 10.1 samples/s
305
+ Step 21270/50462 | Loss: 2.6654 | LR: 3.18e-04 | 4007.1ms/step | 15.0 samples/s
306
+ Step 21280/50462 | Loss: 2.6130 | LR: 3.17e-04 | 3997.5ms/step | 15.0 samples/s
307
+ Step 21290/50462 | Loss: 2.6082 | LR: 3.17e-04 | 3983.9ms/step | 15.1 samples/s
308
+ Step 21300/50462 | Loss: 2.6253 | LR: 3.17e-04 | 4002.1ms/step | 15.0 samples/s
309
+ Step 21310/50462 | Loss: 2.6433 | LR: 3.17e-04 | 3992.1ms/step | 15.0 samples/s
310
+ Step 21320/50462 | Loss: 2.6282 | LR: 3.17e-04 | 3988.3ms/step | 15.0 samples/s
311
+ Step 21330/50462 | Loss: 2.6252 | LR: 3.17e-04 | 3994.6ms/step | 15.0 samples/s
312
+ Step 21340/50462 | Loss: 2.6416 | LR: 3.17e-04 | 4000.9ms/step | 15.0 samples/s
313
+ Step 21350/50462 | Loss: 2.6022 | LR: 3.16e-04 | 4003.6ms/step | 15.0 samples/s
314
+ Step 21360/50462 | Loss: 2.5967 | LR: 3.16e-04 | 4003.6ms/step | 15.0 samples/s
315
+ Step 21370/50462 | Loss: 2.6328 | LR: 3.16e-04 | 4003.2ms/step | 15.0 samples/s
316
+ Step 21380/50462 | Loss: 2.6271 | LR: 3.16e-04 | 4000.9ms/step | 15.0 samples/s
317
+ Step 21390/50462 | Loss: 2.6405 | LR: 3.16e-04 | 4004.5ms/step | 15.0 samples/s
318
+ Step 21400/50462 | Loss: 2.6238 | LR: 3.16e-04 | 4007.7ms/step | 15.0 samples/s
319
+ Step 21410/50462 | Loss: 2.6182 | LR: 3.16e-04 | 4012.6ms/step | 15.0 samples/s
320
+ Step 21420/50462 | Loss: 2.6143 | LR: 3.15e-04 | 4004.4ms/step | 15.0 samples/s
321
+ Step 21430/50462 | Loss: 2.6412 | LR: 3.15e-04 | 4004.3ms/step | 15.0 samples/s
322
+ Step 21440/50462 | Loss: 2.6354 | LR: 3.15e-04 | 4003.7ms/step | 15.0 samples/s
323
+ Step 21450/50462 | Loss: 2.6456 | LR: 3.15e-04 | 4002.7ms/step | 15.0 samples/s
324
+ Step 21460/50462 | Loss: 2.5962 | LR: 3.15e-04 | 4006.6ms/step | 15.0 samples/s
325
+ Step 21470/50462 | Loss: 2.6550 | LR: 3.15e-04 | 4004.8ms/step | 15.0 samples/s
326
+ Step 21480/50462 | Loss: 2.6529 | LR: 3.14e-04 | 4003.6ms/step | 15.0 samples/s
327
+ Step 21490/50462 | Loss: 2.6217 | LR: 3.14e-04 | 4014.4ms/step | 14.9 samples/s
328
+ Step 21500/50462 | Loss: 2.5973 | LR: 3.14e-04 | 3997.6ms/step | 15.0 samples/s
329
+ Validation at step 21500: Loss = 2.6109, Perplexity = 13.61
330
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-21500
331
+ Step 21510/50462 | Loss: 2.6186 | LR: 3.14e-04 | 5945.9ms/step | 10.1 samples/s
332
+ Step 21520/50462 | Loss: 2.6293 | LR: 3.14e-04 | 3998.4ms/step | 15.0 samples/s
333
+ Step 21530/50462 | Loss: 2.6517 | LR: 3.14e-04 | 3993.9ms/step | 15.0 samples/s
334
+ Step 21540/50462 | Loss: 2.6426 | LR: 3.14e-04 | 3997.1ms/step | 15.0 samples/s
335
+ Step 21550/50462 | Loss: 2.5972 | LR: 3.13e-04 | 3989.9ms/step | 15.0 samples/s
336
+ Step 21560/50462 | Loss: 2.6109 | LR: 3.13e-04 | 4006.9ms/step | 15.0 samples/s
337
+ Step 21570/50462 | Loss: 2.5994 | LR: 3.13e-04 | 4008.6ms/step | 15.0 samples/s
338
+ Step 21580/50462 | Loss: 2.6239 | LR: 3.13e-04 | 4002.3ms/step | 15.0 samples/s
339
+ Step 21590/50462 | Loss: 2.6228 | LR: 3.13e-04 | 3995.9ms/step | 15.0 samples/s
340
+ Step 21600/50462 | Loss: 2.6171 | LR: 3.13e-04 | 4005.3ms/step | 15.0 samples/s
341
+ Step 21610/50462 | Loss: 2.5894 | LR: 3.12e-04 | 4000.5ms/step | 15.0 samples/s
342
+ Step 21620/50462 | Loss: 2.6516 | LR: 3.12e-04 | 3999.5ms/step | 15.0 samples/s
343
+ Step 21630/50462 | Loss: 2.6517 | LR: 3.12e-04 | 4005.3ms/step | 15.0 samples/s
344
+ Step 21640/50462 | Loss: 2.6178 | LR: 3.12e-04 | 3999.4ms/step | 15.0 samples/s
345
+ Step 21650/50462 | Loss: 2.6206 | LR: 3.12e-04 | 3998.6ms/step | 15.0 samples/s
346
+ Step 21660/50462 | Loss: 2.5801 | LR: 3.12e-04 | 4013.6ms/step | 14.9 samples/s
347
+ Step 21670/50462 | Loss: 2.6354 | LR: 3.12e-04 | 4007.9ms/step | 15.0 samples/s
348
+ Step 21680/50462 | Loss: 2.6285 | LR: 3.11e-04 | 4006.9ms/step | 15.0 samples/s
349
+ Step 21690/50462 | Loss: 2.6177 | LR: 3.11e-04 | 4007.4ms/step | 15.0 samples/s
350
+ Step 21700/50462 | Loss: 2.6161 | LR: 3.11e-04 | 4008.8ms/step | 15.0 samples/s
351
+ Step 21710/50462 | Loss: 2.6264 | LR: 3.11e-04 | 4001.8ms/step | 15.0 samples/s
352
+ Step 21720/50462 | Loss: 2.6411 | LR: 3.11e-04 | 4010.0ms/step | 15.0 samples/s
353
+ Step 21730/50462 | Loss: 2.5821 | LR: 3.11e-04 | 4003.8ms/step | 15.0 samples/s
354
+ Step 21740/50462 | Loss: 2.6234 | LR: 3.10e-04 | 4015.3ms/step | 14.9 samples/s
355
+ Step 21750/50462 | Loss: 2.5910 | LR: 3.10e-04 | 4010.6ms/step | 15.0 samples/s
356
+ Validation at step 21750: Loss = 2.6085, Perplexity = 13.58
357
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-21750
358
+ Step 21760/50462 | Loss: 2.5794 | LR: 3.10e-04 | 5931.8ms/step | 10.1 samples/s
359
+ Step 21770/50462 | Loss: 2.6210 | LR: 3.10e-04 | 4007.1ms/step | 15.0 samples/s
360
+ Step 21780/50462 | Loss: 2.6125 | LR: 3.10e-04 | 4012.9ms/step | 15.0 samples/s
361
+ Step 21790/50462 | Loss: 2.6176 | LR: 3.10e-04 | 4000.7ms/step | 15.0 samples/s
362
+ Step 21800/50462 | Loss: 2.6236 | LR: 3.10e-04 | 3994.2ms/step | 15.0 samples/s
363
+ Step 21810/50462 | Loss: 2.6054 | LR: 3.09e-04 | 3996.6ms/step | 15.0 samples/s
364
+ Step 21820/50462 | Loss: 2.6237 | LR: 3.09e-04 | 4000.7ms/step | 15.0 samples/s
365
+ Step 21830/50462 | Loss: 2.6223 | LR: 3.09e-04 | 3995.7ms/step | 15.0 samples/s
366
+ Step 21840/50462 | Loss: 2.6369 | LR: 3.09e-04 | 4013.9ms/step | 14.9 samples/s
367
+ Step 21850/50462 | Loss: 2.6448 | LR: 3.09e-04 | 3992.4ms/step | 15.0 samples/s
368
+ Step 21860/50462 | Loss: 2.5907 | LR: 3.09e-04 | 4008.0ms/step | 15.0 samples/s
369
+ Step 21870/50462 | Loss: 2.6121 | LR: 3.08e-04 | 4004.9ms/step | 15.0 samples/s
370
+ Step 21880/50462 | Loss: 2.6312 | LR: 3.08e-04 | 3993.8ms/step | 15.0 samples/s
371
+ Step 21890/50462 | Loss: 2.6007 | LR: 3.08e-04 | 3986.9ms/step | 15.0 samples/s
372
+ Step 21900/50462 | Loss: 2.5887 | LR: 3.08e-04 | 3992.5ms/step | 15.0 samples/s
373
+ Step 21910/50462 | Loss: 2.6116 | LR: 3.08e-04 | 3993.2ms/step | 15.0 samples/s
374
+ Step 21920/50462 | Loss: 2.5870 | LR: 3.08e-04 | 3999.3ms/step | 15.0 samples/s
375
+ Step 21930/50462 | Loss: 2.6212 | LR: 3.08e-04 | 4009.1ms/step | 15.0 samples/s
376
+ Step 21940/50462 | Loss: 2.6358 | LR: 3.07e-04 | 4007.0ms/step | 15.0 samples/s
377
+ Step 21950/50462 | Loss: 2.6051 | LR: 3.07e-04 | 4009.1ms/step | 15.0 samples/s
378
+ Step 21960/50462 | Loss: 2.6525 | LR: 3.07e-04 | 4009.2ms/step | 15.0 samples/s
379
+ Step 21970/50462 | Loss: 2.6023 | LR: 3.07e-04 | 4002.1ms/step | 15.0 samples/s
380
+ Step 21980/50462 | Loss: 2.6399 | LR: 3.07e-04 | 3997.2ms/step | 15.0 samples/s
381
+ Step 21990/50462 | Loss: 2.6154 | LR: 3.07e-04 | 3987.6ms/step | 15.0 samples/s
382
+ Step 22000/50462 | Loss: 2.6319 | LR: 3.06e-04 | 4006.7ms/step | 15.0 samples/s
383
+ Validation at step 22000: Loss = 2.6043, Perplexity = 13.52
384
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-22000
385
+ Step 22010/50462 | Loss: 2.5983 | LR: 3.06e-04 | 5938.1ms/step | 10.1 samples/s
386
+ Step 22020/50462 | Loss: 2.6244 | LR: 3.06e-04 | 4002.4ms/step | 15.0 samples/s
387
+ Step 22030/50462 | Loss: 2.6278 | LR: 3.06e-04 | 4000.9ms/step | 15.0 samples/s
388
+ Step 22040/50462 | Loss: 2.6408 | LR: 3.06e-04 | 4004.2ms/step | 15.0 samples/s
389
+ Step 22050/50462 | Loss: 2.6440 | LR: 3.06e-04 | 4004.6ms/step | 15.0 samples/s
390
+ Step 22060/50462 | Loss: 2.6213 | LR: 3.06e-04 | 3997.6ms/step | 15.0 samples/s
391
+ Step 22070/50462 | Loss: 2.6318 | LR: 3.05e-04 | 4002.0ms/step | 15.0 samples/s
392
+ Step 22080/50462 | Loss: 2.6002 | LR: 3.05e-04 | 3995.0ms/step | 15.0 samples/s
393
+ Step 22090/50462 | Loss: 2.6189 | LR: 3.05e-04 | 4009.7ms/step | 15.0 samples/s
394
+ Step 22100/50462 | Loss: 2.5943 | LR: 3.05e-04 | 4004.9ms/step | 15.0 samples/s
395
+ Step 22110/50462 | Loss: 2.6192 | LR: 3.05e-04 | 4016.2ms/step | 14.9 samples/s
396
+ Step 22120/50462 | Loss: 2.6075 | LR: 3.05e-04 | 4002.6ms/step | 15.0 samples/s
397
+ Step 22130/50462 | Loss: 2.6358 | LR: 3.04e-04 | 4000.2ms/step | 15.0 samples/s
398
+ Step 22140/50462 | Loss: 2.6139 | LR: 3.04e-04 | 4008.5ms/step | 15.0 samples/s
399
+ Step 22150/50462 | Loss: 2.6205 | LR: 3.04e-04 | 3996.8ms/step | 15.0 samples/s
400
+ Step 22160/50462 | Loss: 2.6169 | LR: 3.04e-04 | 4002.8ms/step | 15.0 samples/s
401
+ Step 22170/50462 | Loss: 2.6548 | LR: 3.04e-04 | 3997.4ms/step | 15.0 samples/s
402
+ Step 22180/50462 | Loss: 2.6267 | LR: 3.04e-04 | 4011.4ms/step | 15.0 samples/s
403
+ Step 22190/50462 | Loss: 2.5920 | LR: 3.04e-04 | 3998.3ms/step | 15.0 samples/s
404
+ Step 22200/50462 | Loss: 2.6071 | LR: 3.03e-04 | 3987.7ms/step | 15.0 samples/s
405
+ Step 22210/50462 | Loss: 2.5974 | LR: 3.03e-04 | 3992.6ms/step | 15.0 samples/s
406
+ Step 22220/50462 | Loss: 2.6401 | LR: 3.03e-04 | 3993.1ms/step | 15.0 samples/s
407
+ Step 22230/50462 | Loss: 2.6420 | LR: 3.03e-04 | 4004.9ms/step | 15.0 samples/s
408
+ Step 22240/50462 | Loss: 2.6123 | LR: 3.03e-04 | 4004.9ms/step | 15.0 samples/s
409
+ Step 22250/50462 | Loss: 2.5999 | LR: 3.03e-04 | 3997.7ms/step | 15.0 samples/s
410
+ Validation at step 22250: Loss = 2.6026, Perplexity = 13.50
411
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-22250
412
+ Step 22260/50462 | Loss: 2.6372 | LR: 3.02e-04 | 5919.1ms/step | 10.1 samples/s
413
+ Step 22270/50462 | Loss: 2.6056 | LR: 3.02e-04 | 3996.8ms/step | 15.0 samples/s
414
+ Step 22280/50462 | Loss: 2.5776 | LR: 3.02e-04 | 4010.2ms/step | 15.0 samples/s
415
+ Step 22290/50462 | Loss: 2.5660 | LR: 3.02e-04 | 3997.8ms/step | 15.0 samples/s
416
+ Step 22300/50462 | Loss: 2.6361 | LR: 3.02e-04 | 3995.9ms/step | 15.0 samples/s
417
+ Step 22310/50462 | Loss: 2.5880 | LR: 3.02e-04 | 4004.9ms/step | 15.0 samples/s
418
+ Step 22320/50462 | Loss: 2.6112 | LR: 3.02e-04 | 4001.3ms/step | 15.0 samples/s
419
+ Step 22330/50462 | Loss: 2.6164 | LR: 3.01e-04 | 4003.0ms/step | 15.0 samples/s
420
+ Step 22340/50462 | Loss: 2.5890 | LR: 3.01e-04 | 4004.5ms/step | 15.0 samples/s
421
+ Step 22350/50462 | Loss: 2.6269 | LR: 3.01e-04 | 4009.4ms/step | 15.0 samples/s
422
+ Step 22360/50462 | Loss: 2.5887 | LR: 3.01e-04 | 4010.1ms/step | 15.0 samples/s
423
+ Step 22370/50462 | Loss: 2.6033 | LR: 3.01e-04 | 4004.1ms/step | 15.0 samples/s
424
+ Step 22380/50462 | Loss: 2.6225 | LR: 3.01e-04 | 3998.7ms/step | 15.0 samples/s
425
+ Step 22390/50462 | Loss: 2.6090 | LR: 3.00e-04 | 4004.6ms/step | 15.0 samples/s
426
+ Step 22400/50462 | Loss: 2.5808 | LR: 3.00e-04 | 4012.1ms/step | 15.0 samples/s
427
+ Step 22410/50462 | Loss: 2.6251 | LR: 3.00e-04 | 4008.8ms/step | 15.0 samples/s
428
+ Step 22420/50462 | Loss: 2.6028 | LR: 3.00e-04 | 4017.7ms/step | 14.9 samples/s
429
+ Step 22430/50462 | Loss: 2.5981 | LR: 3.00e-04 | 3993.0ms/step | 15.0 samples/s
430
+ Step 22440/50462 | Loss: 2.6076 | LR: 3.00e-04 | 4004.9ms/step | 15.0 samples/s
431
+ Step 22450/50462 | Loss: 2.5987 | LR: 3.00e-04 | 3994.1ms/step | 15.0 samples/s
432
+ Step 22460/50462 | Loss: 2.6330 | LR: 2.99e-04 | 3995.4ms/step | 15.0 samples/s
433
+ Step 22470/50462 | Loss: 2.6153 | LR: 2.99e-04 | 4002.7ms/step | 15.0 samples/s
434
+ Step 22480/50462 | Loss: 2.5844 | LR: 2.99e-04 | 4011.7ms/step | 15.0 samples/s
435
+ Step 22490/50462 | Loss: 2.5966 | LR: 2.99e-04 | 4005.9ms/step | 15.0 samples/s
436
+ Step 22500/50462 | Loss: 2.6209 | LR: 2.99e-04 | 4010.4ms/step | 15.0 samples/s
437
+ Validation at step 22500: Loss = 2.5997, Perplexity = 13.46
438
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-22500
439
+ Step 22510/50462 | Loss: 2.6002 | LR: 2.99e-04 | 5924.5ms/step | 10.1 samples/s
440
+ Step 22520/50462 | Loss: 2.6142 | LR: 2.98e-04 | 4006.3ms/step | 15.0 samples/s
441
+ Step 22530/50462 | Loss: 2.5991 | LR: 2.98e-04 | 4004.3ms/step | 15.0 samples/s
442
+ Step 22540/50462 | Loss: 2.6126 | LR: 2.98e-04 | 4008.5ms/step | 15.0 samples/s
443
+ Step 22550/50462 | Loss: 2.6071 | LR: 2.98e-04 | 4007.2ms/step | 15.0 samples/s
444
+ Step 22560/50462 | Loss: 2.6166 | LR: 2.98e-04 | 4005.6ms/step | 15.0 samples/s
445
+ Step 22570/50462 | Loss: 2.6081 | LR: 2.98e-04 | 4006.4ms/step | 15.0 samples/s
446
+ Step 22580/50462 | Loss: 2.6249 | LR: 2.98e-04 | 3992.7ms/step | 15.0 samples/s
447
+ Step 22590/50462 | Loss: 2.6374 | LR: 2.97e-04 | 4007.9ms/step | 15.0 samples/s
448
+ Step 22600/50462 | Loss: 2.6285 | LR: 2.97e-04 | 3992.4ms/step | 15.0 samples/s
449
+ Step 22610/50462 | Loss: 2.6078 | LR: 2.97e-04 | 4015.2ms/step | 14.9 samples/s
450
+ Step 22620/50462 | Loss: 2.6361 | LR: 2.97e-04 | 4007.8ms/step | 15.0 samples/s
451
+ Step 22630/50462 | Loss: 2.6122 | LR: 2.97e-04 | 4012.5ms/step | 15.0 samples/s
452
+ Step 22640/50462 | Loss: 2.6002 | LR: 2.97e-04 | 4002.5ms/step | 15.0 samples/s
453
+ Step 22650/50462 | Loss: 2.6144 | LR: 2.96e-04 | 4002.2ms/step | 15.0 samples/s
454
+ Step 22660/50462 | Loss: 2.6097 | LR: 2.96e-04 | 4008.5ms/step | 15.0 samples/s
455
+ Step 22670/50462 | Loss: 2.6279 | LR: 2.96e-04 | 4018.0ms/step | 14.9 samples/s
456
+ Step 22680/50462 | Loss: 2.5981 | LR: 2.96e-04 | 4014.6ms/step | 14.9 samples/s
457
+ Step 22690/50462 | Loss: 2.5876 | LR: 2.96e-04 | 3994.0ms/step | 15.0 samples/s
458
+ Step 22700/50462 | Loss: 2.5706 | LR: 2.96e-04 | 3993.3ms/step | 15.0 samples/s
459
+ Step 22710/50462 | Loss: 2.5897 | LR: 2.95e-04 | 4001.6ms/step | 15.0 samples/s
460
+ Step 22720/50462 | Loss: 2.6051 | LR: 2.95e-04 | 3999.8ms/step | 15.0 samples/s
461
+ Step 22730/50462 | Loss: 2.5756 | LR: 2.95e-04 | 4011.9ms/step | 15.0 samples/s
462
+ Step 22740/50462 | Loss: 2.5679 | LR: 2.95e-04 | 4005.9ms/step | 15.0 samples/s
463
+ Step 22750/50462 | Loss: 2.6092 | LR: 2.95e-04 | 4002.2ms/step | 15.0 samples/s
464
+ Validation at step 22750: Loss = 2.5984, Perplexity = 13.44
465
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-22750
466
+ Step 22760/50462 | Loss: 2.6162 | LR: 2.95e-04 | 5909.5ms/step | 10.2 samples/s
467
+ Step 22770/50462 | Loss: 2.6108 | LR: 2.95e-04 | 4001.4ms/step | 15.0 samples/s
468
+ Step 22780/50462 | Loss: 2.5911 | LR: 2.94e-04 | 3990.9ms/step | 15.0 samples/s
469
+ Step 22790/50462 | Loss: 2.5963 | LR: 2.94e-04 | 4003.6ms/step | 15.0 samples/s
470
+ Step 22800/50462 | Loss: 2.5973 | LR: 2.94e-04 | 4000.5ms/step | 15.0 samples/s
471
+ Step 22810/50462 | Loss: 2.6453 | LR: 2.94e-04 | 4003.2ms/step | 15.0 samples/s
472
+ Step 22820/50462 | Loss: 2.5944 | LR: 2.94e-04 | 3995.3ms/step | 15.0 samples/s
473
+ Step 22830/50462 | Loss: 2.5974 | LR: 2.94e-04 | 4001.6ms/step | 15.0 samples/s
474
+ Step 22840/50462 | Loss: 2.6087 | LR: 2.93e-04 | 4003.0ms/step | 15.0 samples/s
475
+ Step 22850/50462 | Loss: 2.5930 | LR: 2.93e-04 | 4000.7ms/step | 15.0 samples/s
476
+ Step 22860/50462 | Loss: 2.6152 | LR: 2.93e-04 | 4006.2ms/step | 15.0 samples/s
477
+ Step 22870/50462 | Loss: 2.5838 | LR: 2.93e-04 | 4008.9ms/step | 15.0 samples/s
478
+ Step 22880/50462 | Loss: 2.5750 | LR: 2.93e-04 | 4000.7ms/step | 15.0 samples/s
479
+ Step 22890/50462 | Loss: 2.6314 | LR: 2.93e-04 | 3996.4ms/step | 15.0 samples/s
480
+ Step 22900/50462 | Loss: 2.6122 | LR: 2.93e-04 | 3998.2ms/step | 15.0 samples/s
481
+ Step 22910/50462 | Loss: 2.5877 | LR: 2.92e-04 | 4020.8ms/step | 14.9 samples/s
482
+ Step 22920/50462 | Loss: 2.5973 | LR: 2.92e-04 | 3990.3ms/step | 15.0 samples/s
483
+ Step 22930/50462 | Loss: 2.6424 | LR: 2.92e-04 | 4015.9ms/step | 14.9 samples/s
484
+ Step 22940/50462 | Loss: 2.6169 | LR: 2.92e-04 | 4001.8ms/step | 15.0 samples/s
485
+ Step 22950/50462 | Loss: 2.6220 | LR: 2.92e-04 | 3996.7ms/step | 15.0 samples/s
486
+ Step 22960/50462 | Loss: 2.5615 | LR: 2.92e-04 | 4006.3ms/step | 15.0 samples/s
487
+ Step 22970/50462 | Loss: 2.6135 | LR: 2.91e-04 | 4000.3ms/step | 15.0 samples/s
488
+ Step 22980/50462 | Loss: 2.6066 | LR: 2.91e-04 | 4010.7ms/step | 15.0 samples/s
489
+ Step 22990/50462 | Loss: 2.6174 | LR: 2.91e-04 | 3998.2ms/step | 15.0 samples/s
490
+ Step 23000/50462 | Loss: 2.5838 | LR: 2.91e-04 | 4010.7ms/step | 15.0 samples/s
491
+ Validation at step 23000: Loss = 2.5971, Perplexity = 13.43
492
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-23000
493
+ Step 23010/50462 | Loss: 2.6106 | LR: 2.91e-04 | 5919.3ms/step | 10.1 samples/s
494
+ Step 23020/50462 | Loss: 2.6048 | LR: 2.91e-04 | 4003.9ms/step | 15.0 samples/s
495
+ Step 23030/50462 | Loss: 2.5978 | LR: 2.91e-04 | 4018.5ms/step | 14.9 samples/s
496
+ Step 23040/50462 | Loss: 2.5952 | LR: 2.90e-04 | 4002.2ms/step | 15.0 samples/s
497
+ Step 23050/50462 | Loss: 2.6234 | LR: 2.90e-04 | 3999.1ms/step | 15.0 samples/s
498
+ Step 23060/50462 | Loss: 2.6459 | LR: 2.90e-04 | 4007.8ms/step | 15.0 samples/s
499
+ Step 23070/50462 | Loss: 2.5546 | LR: 2.90e-04 | 4007.9ms/step | 15.0 samples/s
500
+ Step 23080/50462 | Loss: 2.6116 | LR: 2.90e-04 | 4001.7ms/step | 15.0 samples/s
501
+ Step 23090/50462 | Loss: 2.5916 | LR: 2.90e-04 | 4001.3ms/step | 15.0 samples/s
502
+ Step 23100/50462 | Loss: 2.6124 | LR: 2.89e-04 | 4006.2ms/step | 15.0 samples/s
503
+ Step 23110/50462 | Loss: 2.6018 | LR: 2.89e-04 | 4001.1ms/step | 15.0 samples/s
504
+ Step 23120/50462 | Loss: 2.5946 | LR: 2.89e-04 | 4008.4ms/step | 15.0 samples/s
505
+ Step 23130/50462 | Loss: 2.5953 | LR: 2.89e-04 | 4000.1ms/step | 15.0 samples/s
506
+ Step 23140/50462 | Loss: 2.5763 | LR: 2.89e-04 | 4004.9ms/step | 15.0 samples/s
507
+ Step 23150/50462 | Loss: 2.6159 | LR: 2.89e-04 | 4004.5ms/step | 15.0 samples/s
508
+ Step 23160/50462 | Loss: 2.6388 | LR: 2.88e-04 | 4005.8ms/step | 15.0 samples/s
509
+ Step 23170/50462 | Loss: 2.6042 | LR: 2.88e-04 | 3999.7ms/step | 15.0 samples/s
510
+ Step 23180/50462 | Loss: 2.5867 | LR: 2.88e-04 | 3999.2ms/step | 15.0 samples/s
511
+ Step 23190/50462 | Loss: 2.6038 | LR: 2.88e-04 | 3993.4ms/step | 15.0 samples/s
512
+ Step 23200/50462 | Loss: 2.5875 | LR: 2.88e-04 | 4003.8ms/step | 15.0 samples/s
513
+ Step 23210/50462 | Loss: 2.6187 | LR: 2.88e-04 | 3998.9ms/step | 15.0 samples/s
514
+ Step 23220/50462 | Loss: 2.6123 | LR: 2.88e-04 | 3994.8ms/step | 15.0 samples/s
515
+ Step 23230/50462 | Loss: 2.6104 | LR: 2.87e-04 | 3989.7ms/step | 15.0 samples/s
516
+ Step 23240/50462 | Loss: 2.5994 | LR: 2.87e-04 | 3999.0ms/step | 15.0 samples/s
517
+ Step 23250/50462 | Loss: 2.6249 | LR: 2.87e-04 | 3995.0ms/step | 15.0 samples/s
518
+ Validation at step 23250: Loss = 2.5937, Perplexity = 13.38
519
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-23250
520
+ Step 23260/50462 | Loss: 2.5907 | LR: 2.87e-04 | 5932.7ms/step | 10.1 samples/s
521
+ Step 23270/50462 | Loss: 2.5883 | LR: 2.87e-04 | 3999.0ms/step | 15.0 samples/s
522
+ Step 23280/50462 | Loss: 2.5866 | LR: 2.87e-04 | 4015.4ms/step | 14.9 samples/s
523
+ Step 23290/50462 | Loss: 2.6456 | LR: 2.86e-04 | 4004.1ms/step | 15.0 samples/s
524
+ Step 23300/50462 | Loss: 2.5878 | LR: 2.86e-04 | 4007.9ms/step | 15.0 samples/s
525
+ Step 23310/50462 | Loss: 2.5966 | LR: 2.86e-04 | 3989.1ms/step | 15.0 samples/s
526
+ Step 23320/50462 | Loss: 2.6500 | LR: 2.86e-04 | 4004.0ms/step | 15.0 samples/s
527
+ Step 23330/50462 | Loss: 2.6148 | LR: 2.86e-04 | 4005.7ms/step | 15.0 samples/s
528
+ Step 23340/50462 | Loss: 2.6141 | LR: 2.86e-04 | 3996.6ms/step | 15.0 samples/s
529
+ Step 23350/50462 | Loss: 2.6043 | LR: 2.86e-04 | 3995.8ms/step | 15.0 samples/s
530
+ Step 23360/50462 | Loss: 2.5909 | LR: 2.85e-04 | 3992.6ms/step | 15.0 samples/s
531
+ Step 23370/50462 | Loss: 2.6066 | LR: 2.85e-04 | 3989.6ms/step | 15.0 samples/s
532
+ Step 23380/50462 | Loss: 2.6044 | LR: 2.85e-04 | 4000.9ms/step | 15.0 samples/s
533
+ Step 23390/50462 | Loss: 2.6189 | LR: 2.85e-04 | 4000.2ms/step | 15.0 samples/s
534
+ Step 23400/50462 | Loss: 2.6199 | LR: 2.85e-04 | 4001.9ms/step | 15.0 samples/s
535
+ Step 23410/50462 | Loss: 2.5935 | LR: 2.85e-04 | 4002.7ms/step | 15.0 samples/s
536
+ Step 23420/50462 | Loss: 2.6088 | LR: 2.84e-04 | 3996.1ms/step | 15.0 samples/s
537
+ Step 23430/50462 | Loss: 2.5940 | LR: 2.84e-04 | 4005.5ms/step | 15.0 samples/s
538
+ Step 23440/50462 | Loss: 2.5979 | LR: 2.84e-04 | 4008.3ms/step | 15.0 samples/s
539
+ Step 23450/50462 | Loss: 2.5981 | LR: 2.84e-04 | 4008.0ms/step | 15.0 samples/s
540
+ Step 23460/50462 | Loss: 2.6125 | LR: 2.84e-04 | 4004.3ms/step | 15.0 samples/s
541
+ Step 23470/50462 | Loss: 2.6046 | LR: 2.84e-04 | 4005.7ms/step | 15.0 samples/s
542
+ Step 23480/50462 | Loss: 2.5946 | LR: 2.83e-04 | 4005.4ms/step | 15.0 samples/s
543
+ Step 23490/50462 | Loss: 2.6359 | LR: 2.83e-04 | 3994.3ms/step | 15.0 samples/s
544
+ Step 23500/50462 | Loss: 2.6123 | LR: 2.83e-04 | 4010.2ms/step | 15.0 samples/s
545
+ Validation at step 23500: Loss = 2.5913, Perplexity = 13.35
546
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-23500
547
+ Step 23510/50462 | Loss: 2.6309 | LR: 2.83e-04 | 5946.4ms/step | 10.1 samples/s
548
+ Step 23520/50462 | Loss: 2.6054 | LR: 2.83e-04 | 4004.2ms/step | 15.0 samples/s
549
+ Step 23530/50462 | Loss: 2.5870 | LR: 2.83e-04 | 4019.6ms/step | 14.9 samples/s
550
+ Step 23540/50462 | Loss: 2.6232 | LR: 2.83e-04 | 4006.4ms/step | 15.0 samples/s
551
+ Step 23550/50462 | Loss: 2.6098 | LR: 2.82e-04 | 4000.4ms/step | 15.0 samples/s
552
+ Step 23560/50462 | Loss: 2.6469 | LR: 2.82e-04 | 4005.5ms/step | 15.0 samples/s
553
+ Step 23570/50462 | Loss: 2.6201 | LR: 2.82e-04 | 3993.1ms/step | 15.0 samples/s
554
+ Step 23580/50462 | Loss: 2.6240 | LR: 2.82e-04 | 4001.0ms/step | 15.0 samples/s
555
+ Step 23590/50462 | Loss: 2.5980 | LR: 2.82e-04 | 4008.4ms/step | 15.0 samples/s
556
+ Step 23600/50462 | Loss: 2.5937 | LR: 2.82e-04 | 4009.7ms/step | 15.0 samples/s
557
+ Step 23610/50462 | Loss: 2.5737 | LR: 2.81e-04 | 4009.0ms/step | 15.0 samples/s
558
+ Step 23620/50462 | Loss: 2.5981 | LR: 2.81e-04 | 4017.8ms/step | 14.9 samples/s
559
+ Step 23630/50462 | Loss: 2.6206 | LR: 2.81e-04 | 4008.7ms/step | 15.0 samples/s
560
+ Step 23640/50462 | Loss: 2.5891 | LR: 2.81e-04 | 4018.5ms/step | 14.9 samples/s
561
+ Step 23650/50462 | Loss: 2.5968 | LR: 2.81e-04 | 4006.5ms/step | 15.0 samples/s
562
+ Step 23660/50462 | Loss: 2.6270 | LR: 2.81e-04 | 4012.1ms/step | 15.0 samples/s
563
+ Step 23670/50462 | Loss: 2.6234 | LR: 2.81e-04 | 3995.2ms/step | 15.0 samples/s
564
+ Step 23680/50462 | Loss: 2.5748 | LR: 2.80e-04 | 3998.7ms/step | 15.0 samples/s
565
+ Step 23690/50462 | Loss: 2.5724 | LR: 2.80e-04 | 3994.9ms/step | 15.0 samples/s
566
+ Step 23700/50462 | Loss: 2.6361 | LR: 2.80e-04 | 3994.6ms/step | 15.0 samples/s
567
+ Step 23710/50462 | Loss: 2.5623 | LR: 2.80e-04 | 4009.8ms/step | 15.0 samples/s
568
+ Step 23720/50462 | Loss: 2.6134 | LR: 2.80e-04 | 3997.8ms/step | 15.0 samples/s
569
+ Step 23730/50462 | Loss: 2.5817 | LR: 2.80e-04 | 4001.9ms/step | 15.0 samples/s
570
+ Step 23740/50462 | Loss: 2.6119 | LR: 2.79e-04 | 4010.7ms/step | 15.0 samples/s
571
+ Step 23750/50462 | Loss: 2.5648 | LR: 2.79e-04 | 4001.3ms/step | 15.0 samples/s
572
+ Validation at step 23750: Loss = 2.5888, Perplexity = 13.31
573
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-23750
574
+ Step 23760/50462 | Loss: 2.5687 | LR: 2.79e-04 | 5926.4ms/step | 10.1 samples/s
575
+ Step 23770/50462 | Loss: 2.5940 | LR: 2.79e-04 | 3992.7ms/step | 15.0 samples/s
576
+ Step 23780/50462 | Loss: 2.5952 | LR: 2.79e-04 | 4010.6ms/step | 15.0 samples/s
577
+ Step 23790/50462 | Loss: 2.6062 | LR: 2.79e-04 | 3995.1ms/step | 15.0 samples/s
578
+ Step 23800/50462 | Loss: 2.6177 | LR: 2.78e-04 | 4004.9ms/step | 15.0 samples/s
579
+ Step 23810/50462 | Loss: 2.5969 | LR: 2.78e-04 | 3999.2ms/step | 15.0 samples/s
580
+ Step 23820/50462 | Loss: 2.6127 | LR: 2.78e-04 | 4001.9ms/step | 15.0 samples/s
581
+ Step 23830/50462 | Loss: 2.5904 | LR: 2.78e-04 | 4004.1ms/step | 15.0 samples/s
582
+ Step 23840/50462 | Loss: 2.6333 | LR: 2.78e-04 | 4006.9ms/step | 15.0 samples/s
583
+ Step 23850/50462 | Loss: 2.6652 | LR: 2.78e-04 | 3993.3ms/step | 15.0 samples/s
584
+ Step 23860/50462 | Loss: 2.5799 | LR: 2.78e-04 | 4005.7ms/step | 15.0 samples/s
585
+ Step 23870/50462 | Loss: 2.5788 | LR: 2.77e-04 | 4009.0ms/step | 15.0 samples/s
586
+ Step 23880/50462 | Loss: 2.5869 | LR: 2.77e-04 | 4005.9ms/step | 15.0 samples/s
587
+ Step 23890/50462 | Loss: 2.6370 | LR: 2.77e-04 | 4007.8ms/step | 15.0 samples/s
588
+ Step 23900/50462 | Loss: 2.6092 | LR: 2.77e-04 | 4011.2ms/step | 15.0 samples/s
589
+ Step 23910/50462 | Loss: 2.5914 | LR: 2.77e-04 | 4007.4ms/step | 15.0 samples/s
590
+ Step 23920/50462 | Loss: 2.6211 | LR: 2.77e-04 | 4010.5ms/step | 15.0 samples/s
591
+ Step 23930/50462 | Loss: 2.5718 | LR: 2.76e-04 | 4004.8ms/step | 15.0 samples/s
592
+ Step 23940/50462 | Loss: 2.6118 | LR: 2.76e-04 | 4003.0ms/step | 15.0 samples/s
593
+ Step 23950/50462 | Loss: 2.6229 | LR: 2.76e-04 | 4010.1ms/step | 15.0 samples/s
594
+ Step 23960/50462 | Loss: 2.5656 | LR: 2.76e-04 | 3996.8ms/step | 15.0 samples/s
595
+ Step 23970/50462 | Loss: 2.5712 | LR: 2.76e-04 | 4018.4ms/step | 14.9 samples/s
596
+ Step 23980/50462 | Loss: 2.5975 | LR: 2.76e-04 | 3990.5ms/step | 15.0 samples/s
597
+ Step 23990/50462 | Loss: 2.6125 | LR: 2.75e-04 | 4003.8ms/step | 15.0 samples/s
598
+ Step 24000/50462 | Loss: 2.5876 | LR: 2.75e-04 | 4004.8ms/step | 15.0 samples/s
599
+ Validation at step 24000: Loss = 2.5868, Perplexity = 13.29
600
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-24000
601
+ Step 24010/50462 | Loss: 2.6125 | LR: 2.75e-04 | 5910.3ms/step | 10.2 samples/s
602
+ Step 24020/50462 | Loss: 2.5996 | LR: 2.75e-04 | 4000.2ms/step | 15.0 samples/s
603
+ Step 24030/50462 | Loss: 2.6309 | LR: 2.75e-04 | 3999.4ms/step | 15.0 samples/s
604
+ Step 24040/50462 | Loss: 2.6127 | LR: 2.75e-04 | 3997.6ms/step | 15.0 samples/s
605
+ Step 24050/50462 | Loss: 2.5760 | LR: 2.75e-04 | 4013.3ms/step | 15.0 samples/s
606
+ Step 24060/50462 | Loss: 2.5872 | LR: 2.74e-04 | 3997.3ms/step | 15.0 samples/s
607
+ Step 24070/50462 | Loss: 2.5769 | LR: 2.74e-04 | 4010.3ms/step | 15.0 samples/s
608
+ Step 24080/50462 | Loss: 2.6128 | LR: 2.74e-04 | 4006.9ms/step | 15.0 samples/s
609
+ Step 24090/50462 | Loss: 2.5623 | LR: 2.74e-04 | 3998.8ms/step | 15.0 samples/s
610
+ Step 24100/50462 | Loss: 2.6007 | LR: 2.74e-04 | 4004.6ms/step | 15.0 samples/s
611
+ Step 24110/50462 | Loss: 2.6432 | LR: 2.74e-04 | 4006.8ms/step | 15.0 samples/s
612
+ Step 24120/50462 | Loss: 2.5837 | LR: 2.73e-04 | 3998.3ms/step | 15.0 samples/s
613
+ Step 24130/50462 | Loss: 2.6058 | LR: 2.73e-04 | 4009.8ms/step | 15.0 samples/s
614
+ Step 24140/50462 | Loss: 2.6032 | LR: 2.73e-04 | 4004.5ms/step | 15.0 samples/s
615
+ Step 24150/50462 | Loss: 2.5829 | LR: 2.73e-04 | 4002.1ms/step | 15.0 samples/s
616
+ Step 24160/50462 | Loss: 2.5778 | LR: 2.73e-04 | 4003.2ms/step | 15.0 samples/s
617
+ Step 24170/50462 | Loss: 2.5725 | LR: 2.73e-04 | 4010.4ms/step | 15.0 samples/s
618
+ Step 24180/50462 | Loss: 2.5440 | LR: 2.72e-04 | 4008.2ms/step | 15.0 samples/s
619
+ Step 24190/50462 | Loss: 2.5863 | LR: 2.72e-04 | 4002.5ms/step | 15.0 samples/s
620
+ Step 24200/50462 | Loss: 2.5578 | LR: 2.72e-04 | 4001.5ms/step | 15.0 samples/s
621
+ Step 24210/50462 | Loss: 2.5615 | LR: 2.72e-04 | 4008.9ms/step | 15.0 samples/s
622
+ Step 24220/50462 | Loss: 2.5386 | LR: 2.72e-04 | 3994.3ms/step | 15.0 samples/s
623
+ Step 24230/50462 | Loss: 2.5771 | LR: 2.72e-04 | 4003.0ms/step | 15.0 samples/s
624
+ Step 24240/50462 | Loss: 2.5720 | LR: 2.72e-04 | 4003.3ms/step | 15.0 samples/s
625
+ Step 24250/50462 | Loss: 2.5401 | LR: 2.71e-04 | 3996.8ms/step | 15.0 samples/s
626
+ Validation at step 24250: Loss = 2.5858, Perplexity = 13.27
627
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-24250
628
+ Step 24260/50462 | Loss: 2.5791 | LR: 2.71e-04 | 5912.3ms/step | 10.1 samples/s
629
+ Step 24270/50462 | Loss: 2.5692 | LR: 2.71e-04 | 4010.6ms/step | 15.0 samples/s
630
+ Step 24280/50462 | Loss: 2.5689 | LR: 2.71e-04 | 4011.9ms/step | 15.0 samples/s
631
+ Step 24290/50462 | Loss: 2.5517 | LR: 2.71e-04 | 4017.5ms/step | 14.9 samples/s
632
+ Step 24300/50462 | Loss: 2.5531 | LR: 2.71e-04 | 4031.2ms/step | 14.9 samples/s
633
+ Step 24310/50462 | Loss: 2.5563 | LR: 2.70e-04 | 4026.1ms/step | 14.9 samples/s
634
+ Step 24320/50462 | Loss: 2.5559 | LR: 2.70e-04 | 4011.3ms/step | 15.0 samples/s
635
+ Step 24330/50462 | Loss: 2.5524 | LR: 2.70e-04 | 4022.3ms/step | 14.9 samples/s
636
+ Step 24340/50462 | Loss: 2.5624 | LR: 2.70e-04 | 4019.3ms/step | 14.9 samples/s
637
+ Step 24350/50462 | Loss: 2.5445 | LR: 2.70e-04 | 4018.1ms/step | 14.9 samples/s
638
+ Step 24360/50462 | Loss: 2.5506 | LR: 2.70e-04 | 4018.3ms/step | 14.9 samples/s
639
+ Step 24370/50462 | Loss: 2.5949 | LR: 2.70e-04 | 4008.2ms/step | 15.0 samples/s
640
+ Step 24380/50462 | Loss: 2.5587 | LR: 2.69e-04 | 4010.0ms/step | 15.0 samples/s
641
+ Step 24390/50462 | Loss: 2.5685 | LR: 2.69e-04 | 4005.6ms/step | 15.0 samples/s
642
+ Step 24400/50462 | Loss: 2.5923 | LR: 2.69e-04 | 4008.5ms/step | 15.0 samples/s
643
+ Step 24410/50462 | Loss: 2.5362 | LR: 2.69e-04 | 4003.5ms/step | 15.0 samples/s
644
+ Step 24420/50462 | Loss: 2.5470 | LR: 2.69e-04 | 3990.8ms/step | 15.0 samples/s
645
+ Step 24430/50462 | Loss: 2.5719 | LR: 2.69e-04 | 4001.2ms/step | 15.0 samples/s
646
+ Step 24440/50462 | Loss: 2.6150 | LR: 2.68e-04 | 4003.2ms/step | 15.0 samples/s
647
+ Step 24450/50462 | Loss: 2.5566 | LR: 2.68e-04 | 4000.0ms/step | 15.0 samples/s
648
+ Step 24460/50462 | Loss: 2.5859 | LR: 2.68e-04 | 4004.0ms/step | 15.0 samples/s
649
+ Step 24470/50462 | Loss: 2.5502 | LR: 2.68e-04 | 4001.9ms/step | 15.0 samples/s
650
+ Step 24480/50462 | Loss: 2.5351 | LR: 2.68e-04 | 3994.3ms/step | 15.0 samples/s
651
+ Step 24490/50462 | Loss: 2.5339 | LR: 2.68e-04 | 4005.6ms/step | 15.0 samples/s
652
+ Step 24500/50462 | Loss: 2.5528 | LR: 2.67e-04 | 4006.6ms/step | 15.0 samples/s
653
+ Validation at step 24500: Loss = 2.5835, Perplexity = 13.24
654
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-24500
655
+ Step 24510/50462 | Loss: 2.5183 | LR: 2.67e-04 | 5939.8ms/step | 10.1 samples/s
656
+ Step 24520/50462 | Loss: 2.5537 | LR: 2.67e-04 | 4001.9ms/step | 15.0 samples/s
657
+ Step 24530/50462 | Loss: 2.5936 | LR: 2.67e-04 | 4001.6ms/step | 15.0 samples/s
658
+ Step 24540/50462 | Loss: 2.5606 | LR: 2.67e-04 | 4005.1ms/step | 15.0 samples/s
659
+ Step 24550/50462 | Loss: 2.5595 | LR: 2.67e-04 | 4009.6ms/step | 15.0 samples/s
660
+ Step 24560/50462 | Loss: 2.5823 | LR: 2.67e-04 | 4011.7ms/step | 15.0 samples/s
661
+ Step 24570/50462 | Loss: 2.5685 | LR: 2.66e-04 | 4008.1ms/step | 15.0 samples/s
662
+ Step 24580/50462 | Loss: 2.5711 | LR: 2.66e-04 | 4003.4ms/step | 15.0 samples/s
663
+ Step 24590/50462 | Loss: 2.5396 | LR: 2.66e-04 | 4011.8ms/step | 15.0 samples/s
664
+ Step 24600/50462 | Loss: 2.5580 | LR: 2.66e-04 | 4009.7ms/step | 15.0 samples/s
665
+ Step 24610/50462 | Loss: 2.5487 | LR: 2.66e-04 | 4006.9ms/step | 15.0 samples/s
666
+ Step 24620/50462 | Loss: 2.5518 | LR: 2.66e-04 | 4002.2ms/step | 15.0 samples/s
667
+ Step 24630/50462 | Loss: 2.5964 | LR: 2.65e-04 | 4003.4ms/step | 15.0 samples/s
668
+ Step 24640/50462 | Loss: 2.5582 | LR: 2.65e-04 | 4006.7ms/step | 15.0 samples/s
669
+ Step 24650/50462 | Loss: 2.5680 | LR: 2.65e-04 | 4001.2ms/step | 15.0 samples/s
670
+ Step 24660/50462 | Loss: 2.5743 | LR: 2.65e-04 | 4005.2ms/step | 15.0 samples/s
671
+ Step 24670/50462 | Loss: 2.5785 | LR: 2.65e-04 | 4000.6ms/step | 15.0 samples/s
672
+ Step 24680/50462 | Loss: 2.6000 | LR: 2.65e-04 | 4003.2ms/step | 15.0 samples/s
673
+ Step 24690/50462 | Loss: 2.5616 | LR: 2.64e-04 | 4012.2ms/step | 15.0 samples/s
674
+ Step 24700/50462 | Loss: 2.6001 | LR: 2.64e-04 | 4000.7ms/step | 15.0 samples/s
675
+ Step 24710/50462 | Loss: 2.5655 | LR: 2.64e-04 | 3993.5ms/step | 15.0 samples/s
676
+ Step 24720/50462 | Loss: 2.5920 | LR: 2.64e-04 | 4009.1ms/step | 15.0 samples/s
677
+ Step 24730/50462 | Loss: 2.5783 | LR: 2.64e-04 | 3993.3ms/step | 15.0 samples/s
678
+ Step 24740/50462 | Loss: 2.5743 | LR: 2.64e-04 | 4009.5ms/step | 15.0 samples/s
679
+ Step 24750/50462 | Loss: 2.6089 | LR: 2.64e-04 | 4014.9ms/step | 14.9 samples/s
680
+ Validation at step 24750: Loss = 2.5821, Perplexity = 13.22
681
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-24750
682
+ Step 24760/50462 | Loss: 2.5616 | LR: 2.63e-04 | 5923.2ms/step | 10.1 samples/s
683
+ Step 24770/50462 | Loss: 2.5939 | LR: 2.63e-04 | 4010.6ms/step | 15.0 samples/s
684
+ Step 24780/50462 | Loss: 2.6159 | LR: 2.63e-04 | 3999.6ms/step | 15.0 samples/s
685
+ Step 24790/50462 | Loss: 2.5832 | LR: 2.63e-04 | 4007.0ms/step | 15.0 samples/s
686
+ Step 24800/50462 | Loss: 2.6089 | LR: 2.63e-04 | 4001.5ms/step | 15.0 samples/s
687
+ Step 24810/50462 | Loss: 2.5770 | LR: 2.63e-04 | 4004.5ms/step | 15.0 samples/s
688
+ Step 24820/50462 | Loss: 2.5785 | LR: 2.62e-04 | 4017.1ms/step | 14.9 samples/s
689
+ Step 24830/50462 | Loss: 2.5735 | LR: 2.62e-04 | 4011.2ms/step | 15.0 samples/s
690
+ Step 24840/50462 | Loss: 2.5905 | LR: 2.62e-04 | 4006.9ms/step | 15.0 samples/s
691
+ Step 24850/50462 | Loss: 2.6022 | LR: 2.62e-04 | 4005.9ms/step | 15.0 samples/s
692
+ Step 24860/50462 | Loss: 2.5525 | LR: 2.62e-04 | 3995.0ms/step | 15.0 samples/s
693
+ Step 24870/50462 | Loss: 2.6093 | LR: 2.62e-04 | 3992.5ms/step | 15.0 samples/s
694
+ Step 24880/50462 | Loss: 2.5670 | LR: 2.61e-04 | 3993.7ms/step | 15.0 samples/s
695
+ Step 24890/50462 | Loss: 2.5961 | LR: 2.61e-04 | 4004.9ms/step | 15.0 samples/s
696
+ Step 24900/50462 | Loss: 2.5493 | LR: 2.61e-04 | 4001.3ms/step | 15.0 samples/s
697
+ Step 24910/50462 | Loss: 2.6070 | LR: 2.61e-04 | 4004.6ms/step | 15.0 samples/s
698
+ Step 24920/50462 | Loss: 2.5506 | LR: 2.61e-04 | 4021.5ms/step | 14.9 samples/s
699
+ Step 24930/50462 | Loss: 2.5995 | LR: 2.61e-04 | 4002.7ms/step | 15.0 samples/s
700
+ Step 24940/50462 | Loss: 2.5591 | LR: 2.61e-04 | 4014.7ms/step | 14.9 samples/s
701
+ Step 24950/50462 | Loss: 2.5734 | LR: 2.60e-04 | 4007.7ms/step | 15.0 samples/s
702
+ Step 24960/50462 | Loss: 2.5643 | LR: 2.60e-04 | 4006.8ms/step | 15.0 samples/s
703
+ Step 24970/50462 | Loss: 2.5831 | LR: 2.60e-04 | 4004.2ms/step | 15.0 samples/s
704
+ Step 24980/50462 | Loss: 2.5919 | LR: 2.60e-04 | 4003.9ms/step | 15.0 samples/s
705
+ Step 24990/50462 | Loss: 2.6171 | LR: 2.60e-04 | 4006.6ms/step | 15.0 samples/s
706
+ Step 25000/50462 | Loss: 2.5780 | LR: 2.60e-04 | 4008.3ms/step | 15.0 samples/s
707
+ Validation at step 25000: Loss = 2.5792, Perplexity = 13.19
708
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-25000
709
+ Step 25010/50462 | Loss: 2.5920 | LR: 2.59e-04 | 5960.2ms/step | 10.1 samples/s
710
+ Step 25020/50462 | Loss: 2.5587 | LR: 2.59e-04 | 3995.3ms/step | 15.0 samples/s
711
+ Step 25030/50462 | Loss: 2.5912 | LR: 2.59e-04 | 4013.4ms/step | 14.9 samples/s
712
+ Step 25040/50462 | Loss: 2.6122 | LR: 2.59e-04 | 4007.1ms/step | 15.0 samples/s
713
+ Step 25050/50462 | Loss: 2.5868 | LR: 2.59e-04 | 4012.5ms/step | 15.0 samples/s
714
+ Step 25060/50462 | Loss: 2.6046 | LR: 2.59e-04 | 4004.8ms/step | 15.0 samples/s
715
+ Step 25070/50462 | Loss: 2.5668 | LR: 2.58e-04 | 4018.6ms/step | 14.9 samples/s
716
+ Step 25080/50462 | Loss: 2.6025 | LR: 2.58e-04 | 4010.2ms/step | 15.0 samples/s
717
+ Step 25090/50462 | Loss: 2.6091 | LR: 2.58e-04 | 3997.1ms/step | 15.0 samples/s
718
+ Step 25100/50462 | Loss: 2.6176 | LR: 2.58e-04 | 4003.1ms/step | 15.0 samples/s
719
+ Step 25110/50462 | Loss: 2.5651 | LR: 2.58e-04 | 4000.8ms/step | 15.0 samples/s
720
+ Step 25120/50462 | Loss: 2.6041 | LR: 2.58e-04 | 4004.5ms/step | 15.0 samples/s
721
+ Step 25130/50462 | Loss: 2.6286 | LR: 2.58e-04 | 4000.7ms/step | 15.0 samples/s
722
+ Step 25140/50462 | Loss: 2.5481 | LR: 2.57e-04 | 4006.4ms/step | 15.0 samples/s
723
+ Step 25150/50462 | Loss: 2.5927 | LR: 2.57e-04 | 3999.7ms/step | 15.0 samples/s
724
+ Step 25160/50462 | Loss: 2.5810 | LR: 2.57e-04 | 4007.9ms/step | 15.0 samples/s
725
+ Step 25170/50462 | Loss: 2.5988 | LR: 2.57e-04 | 3999.1ms/step | 15.0 samples/s
726
+ Step 25180/50462 | Loss: 2.6007 | LR: 2.57e-04 | 4019.5ms/step | 14.9 samples/s
727
+ Step 25190/50462 | Loss: 2.5895 | LR: 2.57e-04 | 4007.5ms/step | 15.0 samples/s
728
+ Step 25200/50462 | Loss: 2.5646 | LR: 2.56e-04 | 4004.8ms/step | 15.0 samples/s
729
+ Step 25210/50462 | Loss: 2.6202 | LR: 2.56e-04 | 4003.8ms/step | 15.0 samples/s
730
+ Step 25220/50462 | Loss: 2.5924 | LR: 2.56e-04 | 4011.1ms/step | 15.0 samples/s
731
+ Step 25230/50462 | Loss: 2.6012 | LR: 2.56e-04 | 4001.1ms/step | 15.0 samples/s
732
+ Step 25240/50462 | Loss: 2.5833 | LR: 2.56e-04 | 3994.9ms/step | 15.0 samples/s
733
+ Step 25250/50462 | Loss: 2.5932 | LR: 2.56e-04 | 4005.5ms/step | 15.0 samples/s
734
+ Validation at step 25250: Loss = 2.5769, Perplexity = 13.16
735
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-25250
736
+ Step 25260/50462 | Loss: 2.5756 | LR: 2.55e-04 | 5926.9ms/step | 10.1 samples/s
737
+ Step 25270/50462 | Loss: 2.6086 | LR: 2.55e-04 | 3993.3ms/step | 15.0 samples/s
738
+ Step 25280/50462 | Loss: 2.5820 | LR: 2.55e-04 | 3992.7ms/step | 15.0 samples/s
739
+ Step 25290/50462 | Loss: 2.5725 | LR: 2.55e-04 | 3997.0ms/step | 15.0 samples/s
740
+ Step 25300/50462 | Loss: 2.6188 | LR: 2.55e-04 | 3992.2ms/step | 15.0 samples/s
741
+ Step 25310/50462 | Loss: 2.5883 | LR: 2.55e-04 | 4011.2ms/step | 15.0 samples/s
742
+ Step 25320/50462 | Loss: 2.6032 | LR: 2.55e-04 | 3995.6ms/step | 15.0 samples/s
743
+ Step 25330/50462 | Loss: 2.5982 | LR: 2.54e-04 | 4001.6ms/step | 15.0 samples/s
744
+ Step 25340/50462 | Loss: 2.5845 | LR: 2.54e-04 | 4008.9ms/step | 15.0 samples/s
745
+ Step 25350/50462 | Loss: 2.5751 | LR: 2.54e-04 | 4010.0ms/step | 15.0 samples/s
746
+ Step 25360/50462 | Loss: 2.6082 | LR: 2.54e-04 | 4004.1ms/step | 15.0 samples/s
747
+ Step 25370/50462 | Loss: 2.6269 | LR: 2.54e-04 | 4003.6ms/step | 15.0 samples/s
748
+ Step 25380/50462 | Loss: 2.5993 | LR: 2.54e-04 | 3994.2ms/step | 15.0 samples/s
749
+ Step 25390/50462 | Loss: 2.6022 | LR: 2.53e-04 | 3994.8ms/step | 15.0 samples/s
750
+ Step 25400/50462 | Loss: 2.5782 | LR: 2.53e-04 | 4002.8ms/step | 15.0 samples/s
751
+ Step 25410/50462 | Loss: 2.5797 | LR: 2.53e-04 | 4000.5ms/step | 15.0 samples/s
752
+ Step 25420/50462 | Loss: 2.5831 | LR: 2.53e-04 | 3998.6ms/step | 15.0 samples/s
753
+ Step 25430/50462 | Loss: 2.5703 | LR: 2.53e-04 | 3999.9ms/step | 15.0 samples/s
754
+ Step 25440/50462 | Loss: 2.5969 | LR: 2.53e-04 | 3997.5ms/step | 15.0 samples/s
755
+ Step 25450/50462 | Loss: 2.6078 | LR: 2.52e-04 | 3999.7ms/step | 15.0 samples/s
756
+ Step 25460/50462 | Loss: 2.5739 | LR: 2.52e-04 | 4009.7ms/step | 15.0 samples/s
757
+ Step 25470/50462 | Loss: 2.5879 | LR: 2.52e-04 | 4002.0ms/step | 15.0 samples/s
758
+ Step 25480/50462 | Loss: 2.6116 | LR: 2.52e-04 | 3995.5ms/step | 15.0 samples/s
759
+ Step 25490/50462 | Loss: 2.5844 | LR: 2.52e-04 | 3996.5ms/step | 15.0 samples/s
760
+ Step 25500/50462 | Loss: 2.5916 | LR: 2.52e-04 | 4000.4ms/step | 15.0 samples/s
761
+ Validation at step 25500: Loss = 2.5740, Perplexity = 13.12
762
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-25500
763
+ Step 25510/50462 | Loss: 2.5485 | LR: 2.52e-04 | 5908.1ms/step | 10.2 samples/s
764
+ Step 25520/50462 | Loss: 2.5931 | LR: 2.51e-04 | 4002.5ms/step | 15.0 samples/s
765
+ Step 25530/50462 | Loss: 2.6161 | LR: 2.51e-04 | 4009.1ms/step | 15.0 samples/s
766
+ Step 25540/50462 | Loss: 2.5804 | LR: 2.51e-04 | 4001.5ms/step | 15.0 samples/s
767
+ Step 25550/50462 | Loss: 2.6153 | LR: 2.51e-04 | 3994.0ms/step | 15.0 samples/s
768
+ Step 25560/50462 | Loss: 2.5404 | LR: 2.51e-04 | 4014.4ms/step | 14.9 samples/s
769
+ Step 25570/50462 | Loss: 2.5720 | LR: 2.51e-04 | 4003.6ms/step | 15.0 samples/s
770
+ Step 25580/50462 | Loss: 2.5869 | LR: 2.50e-04 | 4002.0ms/step | 15.0 samples/s
771
+ Step 25590/50462 | Loss: 2.5835 | LR: 2.50e-04 | 3998.3ms/step | 15.0 samples/s
772
+ Step 25600/50462 | Loss: 2.6064 | LR: 2.50e-04 | 4006.7ms/step | 15.0 samples/s
773
+ Step 25610/50462 | Loss: 2.5069 | LR: 2.50e-04 | 4009.6ms/step | 15.0 samples/s
774
+ Step 25620/50462 | Loss: 2.6138 | LR: 2.50e-04 | 4007.1ms/step | 15.0 samples/s
775
+ Step 25630/50462 | Loss: 2.5898 | LR: 2.50e-04 | 4006.5ms/step | 15.0 samples/s
776
+ Step 25640/50462 | Loss: 2.5810 | LR: 2.49e-04 | 3992.6ms/step | 15.0 samples/s
777
+ Step 25650/50462 | Loss: 2.5827 | LR: 2.49e-04 | 3997.7ms/step | 15.0 samples/s
778
+ Step 25660/50462 | Loss: 2.5800 | LR: 2.49e-04 | 3998.6ms/step | 15.0 samples/s
779
+ Step 25670/50462 | Loss: 2.5825 | LR: 2.49e-04 | 4003.6ms/step | 15.0 samples/s
780
+ Step 25680/50462 | Loss: 2.6057 | LR: 2.49e-04 | 3995.4ms/step | 15.0 samples/s
781
+ Step 25690/50462 | Loss: 2.5677 | LR: 2.49e-04 | 4010.3ms/step | 15.0 samples/s
782
+ Step 25700/50462 | Loss: 2.5578 | LR: 2.49e-04 | 4011.7ms/step | 15.0 samples/s
783
+ Step 25710/50462 | Loss: 2.5981 | LR: 2.48e-04 | 4005.8ms/step | 15.0 samples/s
784
+ Step 25720/50462 | Loss: 2.5837 | LR: 2.48e-04 | 4004.9ms/step | 15.0 samples/s
785
+ Step 25730/50462 | Loss: 2.5826 | LR: 2.48e-04 | 3997.3ms/step | 15.0 samples/s
786
+ Step 25740/50462 | Loss: 2.5964 | LR: 2.48e-04 | 4001.7ms/step | 15.0 samples/s
787
+ Step 25750/50462 | Loss: 2.6059 | LR: 2.48e-04 | 4011.8ms/step | 15.0 samples/s
788
+ Validation at step 25750: Loss = 2.5725, Perplexity = 13.10
789
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-25750
790
+ Step 25760/50462 | Loss: 2.5952 | LR: 2.48e-04 | 5918.1ms/step | 10.1 samples/s
791
+ Step 25770/50462 | Loss: 2.6106 | LR: 2.47e-04 | 3997.5ms/step | 15.0 samples/s
792
+ Step 25780/50462 | Loss: 2.5970 | LR: 2.47e-04 | 3997.6ms/step | 15.0 samples/s
793
+ Step 25790/50462 | Loss: 2.5877 | LR: 2.47e-04 | 3993.5ms/step | 15.0 samples/s
794
+ Step 25800/50462 | Loss: 2.6078 | LR: 2.47e-04 | 3988.0ms/step | 15.0 samples/s
795
+ Step 25810/50462 | Loss: 2.5785 | LR: 2.47e-04 | 4005.2ms/step | 15.0 samples/s
796
+ Step 25820/50462 | Loss: 2.5933 | LR: 2.47e-04 | 3998.5ms/step | 15.0 samples/s
797
+ Step 25830/50462 | Loss: 2.5866 | LR: 2.46e-04 | 3999.6ms/step | 15.0 samples/s
798
+ Step 25840/50462 | Loss: 2.6027 | LR: 2.46e-04 | 3995.8ms/step | 15.0 samples/s
799
+ Step 25850/50462 | Loss: 2.5720 | LR: 2.46e-04 | 4006.7ms/step | 15.0 samples/s
800
+ Step 25860/50462 | Loss: 2.5917 | LR: 2.46e-04 | 4005.4ms/step | 15.0 samples/s
801
+ Step 25870/50462 | Loss: 2.5764 | LR: 2.46e-04 | 4006.7ms/step | 15.0 samples/s
802
+ Step 25880/50462 | Loss: 2.5948 | LR: 2.46e-04 | 4014.5ms/step | 14.9 samples/s
803
+ Step 25890/50462 | Loss: 2.5971 | LR: 2.46e-04 | 4001.3ms/step | 15.0 samples/s
804
+ Step 25900/50462 | Loss: 2.5545 | LR: 2.45e-04 | 4007.0ms/step | 15.0 samples/s
805
+ Step 25910/50462 | Loss: 2.5409 | LR: 2.45e-04 | 4009.3ms/step | 15.0 samples/s
806
+ Step 25920/50462 | Loss: 2.5949 | LR: 2.45e-04 | 4010.9ms/step | 15.0 samples/s
807
+ Step 25930/50462 | Loss: 2.5714 | LR: 2.45e-04 | 4005.5ms/step | 15.0 samples/s
808
+ Step 25940/50462 | Loss: 2.6134 | LR: 2.45e-04 | 4006.4ms/step | 15.0 samples/s
809
+ Step 25950/50462 | Loss: 2.5804 | LR: 2.45e-04 | 4007.5ms/step | 15.0 samples/s
810
+ Step 25960/50462 | Loss: 2.5824 | LR: 2.44e-04 | 3990.6ms/step | 15.0 samples/s
811
+ Step 25970/50462 | Loss: 2.5731 | LR: 2.44e-04 | 4012.8ms/step | 15.0 samples/s
812
+ Step 25980/50462 | Loss: 2.5856 | LR: 2.44e-04 | 3998.0ms/step | 15.0 samples/s
813
+ Step 25990/50462 | Loss: 2.5979 | LR: 2.44e-04 | 4015.9ms/step | 14.9 samples/s
814
+ Step 26000/50462 | Loss: 2.5886 | LR: 2.44e-04 | 4005.6ms/step | 15.0 samples/s
815
+ Validation at step 26000: Loss = 2.5702, Perplexity = 13.07
816
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-26000
817
+ Step 26010/50462 | Loss: 2.5853 | LR: 2.44e-04 | 5922.2ms/step | 10.1 samples/s
818
+ Step 26020/50462 | Loss: 2.5679 | LR: 2.43e-04 | 4004.6ms/step | 15.0 samples/s
819
+ Step 26030/50462 | Loss: 2.5918 | LR: 2.43e-04 | 4009.0ms/step | 15.0 samples/s
820
+ Step 26040/50462 | Loss: 2.5904 | LR: 2.43e-04 | 4004.4ms/step | 15.0 samples/s
821
+ Step 26050/50462 | Loss: 2.5795 | LR: 2.43e-04 | 4010.4ms/step | 15.0 samples/s
822
+ Step 26060/50462 | Loss: 2.5457 | LR: 2.43e-04 | 4002.3ms/step | 15.0 samples/s
823
+ Step 26070/50462 | Loss: 2.5660 | LR: 2.43e-04 | 3998.9ms/step | 15.0 samples/s
824
+ Step 26080/50462 | Loss: 2.5865 | LR: 2.43e-04 | 4008.5ms/step | 15.0 samples/s
825
+ Step 26090/50462 | Loss: 2.6020 | LR: 2.42e-04 | 3991.1ms/step | 15.0 samples/s
826
+ Step 26100/50462 | Loss: 2.5812 | LR: 2.42e-04 | 3997.7ms/step | 15.0 samples/s
827
+ Step 26110/50462 | Loss: 2.5513 | LR: 2.42e-04 | 3995.2ms/step | 15.0 samples/s
828
+ Step 26120/50462 | Loss: 2.5998 | LR: 2.42e-04 | 3999.8ms/step | 15.0 samples/s
829
+ Step 26130/50462 | Loss: 2.5710 | LR: 2.42e-04 | 3997.3ms/step | 15.0 samples/s
830
+ Step 26140/50462 | Loss: 2.5609 | LR: 2.42e-04 | 4008.1ms/step | 15.0 samples/s
831
+ Step 26150/50462 | Loss: 2.5603 | LR: 2.41e-04 | 4002.5ms/step | 15.0 samples/s
832
+ Step 26160/50462 | Loss: 2.5625 | LR: 2.41e-04 | 3998.7ms/step | 15.0 samples/s
833
+ Step 26170/50462 | Loss: 2.5948 | LR: 2.41e-04 | 4009.5ms/step | 15.0 samples/s
834
+ Step 26180/50462 | Loss: 2.5684 | LR: 2.41e-04 | 4007.3ms/step | 15.0 samples/s
835
+ Step 26190/50462 | Loss: 2.5878 | LR: 2.41e-04 | 4010.0ms/step | 15.0 samples/s
836
+ Step 26200/50462 | Loss: 2.5831 | LR: 2.41e-04 | 4002.6ms/step | 15.0 samples/s
837
+ Step 26210/50462 | Loss: 2.5930 | LR: 2.40e-04 | 3999.1ms/step | 15.0 samples/s
838
+ Step 26220/50462 | Loss: 2.5730 | LR: 2.40e-04 | 4012.2ms/step | 15.0 samples/s
839
+ Step 26230/50462 | Loss: 2.5269 | LR: 2.40e-04 | 3996.1ms/step | 15.0 samples/s
840
+ Step 26240/50462 | Loss: 2.6148 | LR: 2.40e-04 | 4005.3ms/step | 15.0 samples/s
841
+ Step 26250/50462 | Loss: 2.5738 | LR: 2.40e-04 | 4010.5ms/step | 15.0 samples/s
842
+ Validation at step 26250: Loss = 2.5687, Perplexity = 13.05
843
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-26250
844
+ Step 26260/50462 | Loss: 2.5979 | LR: 2.40e-04 | 5924.6ms/step | 10.1 samples/s
845
+ Step 26270/50462 | Loss: 2.5975 | LR: 2.40e-04 | 4004.3ms/step | 15.0 samples/s
846
+ Step 26280/50462 | Loss: 2.5604 | LR: 2.39e-04 | 3998.4ms/step | 15.0 samples/s
847
+ Step 26290/50462 | Loss: 2.6018 | LR: 2.39e-04 | 4006.7ms/step | 15.0 samples/s
848
+ Step 26300/50462 | Loss: 2.5809 | LR: 2.39e-04 | 4010.7ms/step | 15.0 samples/s
849
+ Step 26310/50462 | Loss: 2.6289 | LR: 2.39e-04 | 3992.1ms/step | 15.0 samples/s
850
+ Step 26320/50462 | Loss: 2.5666 | LR: 2.39e-04 | 3999.6ms/step | 15.0 samples/s
851
+ Step 26330/50462 | Loss: 2.5243 | LR: 2.39e-04 | 3997.9ms/step | 15.0 samples/s
852
+ Step 26340/50462 | Loss: 2.5885 | LR: 2.38e-04 | 3986.7ms/step | 15.1 samples/s
853
+ Step 26350/50462 | Loss: 2.5683 | LR: 2.38e-04 | 4001.0ms/step | 15.0 samples/s
854
+ Step 26360/50462 | Loss: 2.5817 | LR: 2.38e-04 | 4003.0ms/step | 15.0 samples/s
855
+ Step 26370/50462 | Loss: 2.6078 | LR: 2.38e-04 | 4008.8ms/step | 15.0 samples/s
856
+ Step 26380/50462 | Loss: 2.5984 | LR: 2.38e-04 | 3992.2ms/step | 15.0 samples/s
857
+ Step 26390/50462 | Loss: 2.5754 | LR: 2.38e-04 | 4009.1ms/step | 15.0 samples/s
858
+ Step 26400/50462 | Loss: 2.6076 | LR: 2.37e-04 | 4000.4ms/step | 15.0 samples/s
859
+ Step 26410/50462 | Loss: 2.5982 | LR: 2.37e-04 | 4005.5ms/step | 15.0 samples/s
860
+ Step 26420/50462 | Loss: 2.5926 | LR: 2.37e-04 | 4004.3ms/step | 15.0 samples/s
861
+ Step 26430/50462 | Loss: 2.5709 | LR: 2.37e-04 | 4011.7ms/step | 15.0 samples/s
862
+ Step 26440/50462 | Loss: 2.5866 | LR: 2.37e-04 | 4002.9ms/step | 15.0 samples/s
863
+ Step 26450/50462 | Loss: 2.5543 | LR: 2.37e-04 | 4009.6ms/step | 15.0 samples/s
864
+ Step 26460/50462 | Loss: 2.5711 | LR: 2.37e-04 | 4011.7ms/step | 15.0 samples/s
865
+ Step 26470/50462 | Loss: 2.5593 | LR: 2.36e-04 | 4003.9ms/step | 15.0 samples/s
866
+ Step 26480/50462 | Loss: 2.5828 | LR: 2.36e-04 | 4004.8ms/step | 15.0 samples/s
867
+ Step 26490/50462 | Loss: 2.5785 | LR: 2.36e-04 | 4010.1ms/step | 15.0 samples/s
868
+ Step 26500/50462 | Loss: 2.5800 | LR: 2.36e-04 | 4014.9ms/step | 14.9 samples/s
869
+ Validation at step 26500: Loss = 2.5651, Perplexity = 13.00
870
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-26500
871
+ Step 26510/50462 | Loss: 2.5561 | LR: 2.36e-04 | 5908.7ms/step | 10.2 samples/s
872
+ Step 26520/50462 | Loss: 2.5245 | LR: 2.36e-04 | 3997.4ms/step | 15.0 samples/s
873
+ Step 26530/50462 | Loss: 2.6269 | LR: 2.35e-04 | 4001.1ms/step | 15.0 samples/s
874
+ Step 26540/50462 | Loss: 2.5941 | LR: 2.35e-04 | 4003.7ms/step | 15.0 samples/s
875
+ Step 26550/50462 | Loss: 2.5485 | LR: 2.35e-04 | 4006.8ms/step | 15.0 samples/s
876
+ Step 26560/50462 | Loss: 2.5359 | LR: 2.35e-04 | 4005.7ms/step | 15.0 samples/s
877
+ Step 26570/50462 | Loss: 2.6018 | LR: 2.35e-04 | 3996.2ms/step | 15.0 samples/s
878
+ Step 26580/50462 | Loss: 2.5900 | LR: 2.35e-04 | 4012.3ms/step | 15.0 samples/s
879
+ Step 26590/50462 | Loss: 2.5881 | LR: 2.34e-04 | 4009.3ms/step | 15.0 samples/s
880
+ Step 26600/50462 | Loss: 2.5993 | LR: 2.34e-04 | 4002.1ms/step | 15.0 samples/s
881
+ Step 26610/50462 | Loss: 2.5763 | LR: 2.34e-04 | 4007.5ms/step | 15.0 samples/s
882
+ Step 26620/50462 | Loss: 2.5859 | LR: 2.34e-04 | 4006.2ms/step | 15.0 samples/s
883
+ Step 26630/50462 | Loss: 2.6454 | LR: 2.34e-04 | 4003.0ms/step | 15.0 samples/s
884
+ Step 26640/50462 | Loss: 2.5829 | LR: 2.34e-04 | 4003.5ms/step | 15.0 samples/s
885
+ Step 26650/50462 | Loss: 2.5554 | LR: 2.34e-04 | 4001.6ms/step | 15.0 samples/s
886
+ Step 26660/50462 | Loss: 2.5537 | LR: 2.33e-04 | 4000.5ms/step | 15.0 samples/s
887
+ Step 26670/50462 | Loss: 2.5727 | LR: 2.33e-04 | 4005.3ms/step | 15.0 samples/s
888
+ Step 26680/50462 | Loss: 2.5786 | LR: 2.33e-04 | 3992.7ms/step | 15.0 samples/s
889
+ Step 26690/50462 | Loss: 2.5954 | LR: 2.33e-04 | 4014.3ms/step | 14.9 samples/s
890
+ Step 26700/50462 | Loss: 2.5679 | LR: 2.33e-04 | 3987.4ms/step | 15.0 samples/s
891
+ Step 26710/50462 | Loss: 2.5489 | LR: 2.33e-04 | 4008.5ms/step | 15.0 samples/s
892
+ Step 26720/50462 | Loss: 2.5618 | LR: 2.32e-04 | 3999.9ms/step | 15.0 samples/s
893
+ Step 26730/50462 | Loss: 2.5700 | LR: 2.32e-04 | 3998.1ms/step | 15.0 samples/s
894
+ Step 26740/50462 | Loss: 2.5840 | LR: 2.32e-04 | 4009.4ms/step | 15.0 samples/s
895
+ Step 26750/50462 | Loss: 2.5573 | LR: 2.32e-04 | 4009.6ms/step | 15.0 samples/s
896
+ Validation at step 26750: Loss = 2.5634, Perplexity = 12.98
897
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-26750
898
+ Step 26760/50462 | Loss: 2.5768 | LR: 2.32e-04 | 5916.8ms/step | 10.1 samples/s
899
+ Step 26770/50462 | Loss: 2.5515 | LR: 2.32e-04 | 3998.0ms/step | 15.0 samples/s
900
+ Step 26780/50462 | Loss: 2.5740 | LR: 2.31e-04 | 3999.6ms/step | 15.0 samples/s
901
+ Step 26790/50462 | Loss: 2.5752 | LR: 2.31e-04 | 3992.9ms/step | 15.0 samples/s
902
+ Step 26800/50462 | Loss: 2.5459 | LR: 2.31e-04 | 3996.4ms/step | 15.0 samples/s
903
+ Step 26810/50462 | Loss: 2.5643 | LR: 2.31e-04 | 4006.3ms/step | 15.0 samples/s
904
+ Step 26820/50462 | Loss: 2.5504 | LR: 2.31e-04 | 4003.4ms/step | 15.0 samples/s
905
+ Step 26830/50462 | Loss: 2.5364 | LR: 2.31e-04 | 4006.3ms/step | 15.0 samples/s
906
+ Step 26840/50462 | Loss: 2.5849 | LR: 2.31e-04 | 4002.1ms/step | 15.0 samples/s
907
+ Step 26850/50462 | Loss: 2.5504 | LR: 2.30e-04 | 4002.1ms/step | 15.0 samples/s
908
+ Step 26860/50462 | Loss: 2.5915 | LR: 2.30e-04 | 4002.4ms/step | 15.0 samples/s
909
+ Step 26870/50462 | Loss: 2.5809 | LR: 2.30e-04 | 3997.9ms/step | 15.0 samples/s
910
+ Step 26880/50462 | Loss: 2.5727 | LR: 2.30e-04 | 4001.1ms/step | 15.0 samples/s
911
+ Step 26890/50462 | Loss: 2.5514 | LR: 2.30e-04 | 4007.6ms/step | 15.0 samples/s
912
+ Step 26900/50462 | Loss: 2.6068 | LR: 2.30e-04 | 4000.2ms/step | 15.0 samples/s
913
+ Step 26910/50462 | Loss: 2.6170 | LR: 2.29e-04 | 4002.7ms/step | 15.0 samples/s
914
+ Step 26920/50462 | Loss: 2.5687 | LR: 2.29e-04 | 4005.0ms/step | 15.0 samples/s
915
+ Step 26930/50462 | Loss: 2.5355 | LR: 2.29e-04 | 4003.7ms/step | 15.0 samples/s
916
+ Step 26940/50462 | Loss: 2.6090 | LR: 2.29e-04 | 4004.5ms/step | 15.0 samples/s
917
+ Step 26950/50462 | Loss: 2.5946 | LR: 2.29e-04 | 4003.0ms/step | 15.0 samples/s
918
+ Step 26960/50462 | Loss: 2.5522 | LR: 2.29e-04 | 3999.0ms/step | 15.0 samples/s
919
+ Step 26970/50462 | Loss: 2.5767 | LR: 2.28e-04 | 4005.5ms/step | 15.0 samples/s
920
+ Step 26980/50462 | Loss: 2.5696 | LR: 2.28e-04 | 4004.9ms/step | 15.0 samples/s
921
+ Step 26990/50462 | Loss: 2.5683 | LR: 2.28e-04 | 4003.3ms/step | 15.0 samples/s
922
+ Step 27000/50462 | Loss: 2.5912 | LR: 2.28e-04 | 4011.5ms/step | 15.0 samples/s
923
+ Validation at step 27000: Loss = 2.5625, Perplexity = 12.97
924
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-27000
925
+ Step 27010/50462 | Loss: 2.5989 | LR: 2.28e-04 | 5917.4ms/step | 10.1 samples/s
926
+ Step 27020/50462 | Loss: 2.6072 | LR: 2.28e-04 | 3988.8ms/step | 15.0 samples/s
927
+ Step 27030/50462 | Loss: 2.6004 | LR: 2.28e-04 | 4004.1ms/step | 15.0 samples/s
928
+ Step 27040/50462 | Loss: 2.5242 | LR: 2.27e-04 | 4009.3ms/step | 15.0 samples/s
929
+ Step 27050/50462 | Loss: 2.5660 | LR: 2.27e-04 | 4002.4ms/step | 15.0 samples/s
930
+ Step 27060/50462 | Loss: 2.5669 | LR: 2.27e-04 | 3998.0ms/step | 15.0 samples/s
931
+ Step 27070/50462 | Loss: 2.5967 | LR: 2.27e-04 | 4001.3ms/step | 15.0 samples/s
932
+ Step 27080/50462 | Loss: 2.5527 | LR: 2.27e-04 | 4016.1ms/step | 14.9 samples/s
933
+ Step 27090/50462 | Loss: 2.5592 | LR: 2.27e-04 | 4009.7ms/step | 15.0 samples/s
934
+ Step 27100/50462 | Loss: 2.5861 | LR: 2.26e-04 | 4021.6ms/step | 14.9 samples/s
935
+ Step 27110/50462 | Loss: 2.5831 | LR: 2.26e-04 | 4004.4ms/step | 15.0 samples/s
936
+ Step 27120/50462 | Loss: 2.5577 | LR: 2.26e-04 | 4000.3ms/step | 15.0 samples/s
937
+ Step 27130/50462 | Loss: 2.5772 | LR: 2.26e-04 | 4001.1ms/step | 15.0 samples/s
938
+ Step 27140/50462 | Loss: 2.5848 | LR: 2.26e-04 | 3993.8ms/step | 15.0 samples/s
939
+ Step 27150/50462 | Loss: 2.5795 | LR: 2.26e-04 | 4011.1ms/step | 15.0 samples/s
940
+ Step 27160/50462 | Loss: 2.5487 | LR: 2.25e-04 | 4001.2ms/step | 15.0 samples/s
941
+ Step 27170/50462 | Loss: 2.5490 | LR: 2.25e-04 | 3997.4ms/step | 15.0 samples/s
942
+ Step 27180/50462 | Loss: 2.5896 | LR: 2.25e-04 | 4008.5ms/step | 15.0 samples/s
943
+ Step 27190/50462 | Loss: 2.5625 | LR: 2.25e-04 | 4007.4ms/step | 15.0 samples/s
944
+ Step 27200/50462 | Loss: 2.6169 | LR: 2.25e-04 | 4017.6ms/step | 14.9 samples/s
945
+ Step 27210/50462 | Loss: 2.5609 | LR: 2.25e-04 | 4007.0ms/step | 15.0 samples/s
946
+ Step 27220/50462 | Loss: 2.5491 | LR: 2.25e-04 | 3999.7ms/step | 15.0 samples/s
947
+ Step 27230/50462 | Loss: 2.5629 | LR: 2.24e-04 | 4007.2ms/step | 15.0 samples/s
948
+ Step 27240/50462 | Loss: 2.5905 | LR: 2.24e-04 | 4011.5ms/step | 15.0 samples/s
949
+ Step 27250/50462 | Loss: 2.5995 | LR: 2.24e-04 | 4011.7ms/step | 15.0 samples/s
950
+ Validation at step 27250: Loss = 2.5606, Perplexity = 12.94
951
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-27250
952
+ Step 27260/50462 | Loss: 2.5741 | LR: 2.24e-04 | 5919.7ms/step | 10.1 samples/s
953
+ Step 27270/50462 | Loss: 2.5848 | LR: 2.24e-04 | 3998.3ms/step | 15.0 samples/s
954
+ Step 27280/50462 | Loss: 2.5819 | LR: 2.24e-04 | 3991.1ms/step | 15.0 samples/s
955
+ Step 27290/50462 | Loss: 2.5490 | LR: 2.23e-04 | 3994.7ms/step | 15.0 samples/s
956
+ Step 27300/50462 | Loss: 2.5861 | LR: 2.23e-04 | 4005.5ms/step | 15.0 samples/s
957
+ Step 27310/50462 | Loss: 2.5679 | LR: 2.23e-04 | 3988.4ms/step | 15.0 samples/s
958
+ Step 27320/50462 | Loss: 2.5816 | LR: 2.23e-04 | 4004.6ms/step | 15.0 samples/s
959
+ Step 27330/50462 | Loss: 2.5525 | LR: 2.23e-04 | 3996.1ms/step | 15.0 samples/s
960
+ Step 27340/50462 | Loss: 2.5757 | LR: 2.23e-04 | 3999.3ms/step | 15.0 samples/s
961
+ Step 27350/50462 | Loss: 2.5835 | LR: 2.23e-04 | 3998.4ms/step | 15.0 samples/s
962
+ Step 27360/50462 | Loss: 2.5503 | LR: 2.22e-04 | 4000.9ms/step | 15.0 samples/s
963
+ Step 27370/50462 | Loss: 2.5657 | LR: 2.22e-04 | 3987.2ms/step | 15.0 samples/s
964
+ Step 27380/50462 | Loss: 2.5320 | LR: 2.22e-04 | 3997.4ms/step | 15.0 samples/s
965
+ Step 27390/50462 | Loss: 2.5833 | LR: 2.22e-04 | 4008.2ms/step | 15.0 samples/s
966
+ Step 27400/50462 | Loss: 2.5290 | LR: 2.22e-04 | 3998.1ms/step | 15.0 samples/s
967
+ Step 27410/50462 | Loss: 2.5743 | LR: 2.22e-04 | 4014.9ms/step | 14.9 samples/s
968
+ Step 27420/50462 | Loss: 2.5371 | LR: 2.21e-04 | 4008.8ms/step | 15.0 samples/s
969
+ Step 27430/50462 | Loss: 2.5836 | LR: 2.21e-04 | 3997.4ms/step | 15.0 samples/s
970
+ Step 27440/50462 | Loss: 2.5785 | LR: 2.21e-04 | 4004.1ms/step | 15.0 samples/s
971
+ Step 27450/50462 | Loss: 2.5777 | LR: 2.21e-04 | 3999.8ms/step | 15.0 samples/s
972
+ Step 27460/50462 | Loss: 2.5614 | LR: 2.21e-04 | 3998.6ms/step | 15.0 samples/s
973
+ Step 27470/50462 | Loss: 2.5634 | LR: 2.21e-04 | 4012.4ms/step | 15.0 samples/s
974
+ Step 27480/50462 | Loss: 2.5870 | LR: 2.20e-04 | 4016.5ms/step | 14.9 samples/s
975
+ Step 27490/50462 | Loss: 2.5347 | LR: 2.20e-04 | 3991.1ms/step | 15.0 samples/s
976
+ Step 27500/50462 | Loss: 2.5773 | LR: 2.20e-04 | 4000.9ms/step | 15.0 samples/s
977
+ Validation at step 27500: Loss = 2.5579, Perplexity = 12.91
978
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-27500
979
+ Step 27510/50462 | Loss: 2.5588 | LR: 2.20e-04 | 5913.3ms/step | 10.1 samples/s
980
+ Step 27520/50462 | Loss: 2.5526 | LR: 2.20e-04 | 4009.4ms/step | 15.0 samples/s
981
+ Step 27530/50462 | Loss: 2.5877 | LR: 2.20e-04 | 3992.9ms/step | 15.0 samples/s
982
+ Step 27540/50462 | Loss: 2.5789 | LR: 2.20e-04 | 4016.2ms/step | 14.9 samples/s
983
+ Step 27550/50462 | Loss: 2.5632 | LR: 2.19e-04 | 4006.0ms/step | 15.0 samples/s
984
+ Step 27560/50462 | Loss: 2.5664 | LR: 2.19e-04 | 3995.9ms/step | 15.0 samples/s
985
+ Step 27570/50462 | Loss: 2.5612 | LR: 2.19e-04 | 4002.9ms/step | 15.0 samples/s
986
+ Step 27580/50462 | Loss: 2.5892 | LR: 2.19e-04 | 3992.5ms/step | 15.0 samples/s
987
+ Step 27590/50462 | Loss: 2.5953 | LR: 2.19e-04 | 4002.1ms/step | 15.0 samples/s
988
+ Step 27600/50462 | Loss: 2.5947 | LR: 2.19e-04 | 4003.3ms/step | 15.0 samples/s
989
+ Step 27610/50462 | Loss: 2.5757 | LR: 2.18e-04 | 3996.7ms/step | 15.0 samples/s
990
+ Step 27620/50462 | Loss: 2.5892 | LR: 2.18e-04 | 3991.6ms/step | 15.0 samples/s
991
+ Step 27630/50462 | Loss: 2.5567 | LR: 2.18e-04 | 4006.6ms/step | 15.0 samples/s
992
+ Step 27640/50462 | Loss: 2.5297 | LR: 2.18e-04 | 4001.2ms/step | 15.0 samples/s
993
+ Step 27650/50462 | Loss: 2.5818 | LR: 2.18e-04 | 4003.5ms/step | 15.0 samples/s
994
+ Step 27660/50462 | Loss: 2.5621 | LR: 2.18e-04 | 4003.4ms/step | 15.0 samples/s
995
+ Step 27670/50462 | Loss: 2.5713 | LR: 2.17e-04 | 3992.5ms/step | 15.0 samples/s
996
+ Step 27680/50462 | Loss: 2.5435 | LR: 2.17e-04 | 4006.7ms/step | 15.0 samples/s
997
+ Step 27690/50462 | Loss: 2.5638 | LR: 2.17e-04 | 4010.3ms/step | 15.0 samples/s
998
+ Step 27700/50462 | Loss: 2.5336 | LR: 2.17e-04 | 4010.1ms/step | 15.0 samples/s
999
+ Step 27710/50462 | Loss: 2.6022 | LR: 2.17e-04 | 3995.9ms/step | 15.0 samples/s
1000
+ Step 27720/50462 | Loss: 2.5675 | LR: 2.17e-04 | 4005.7ms/step | 15.0 samples/s
1001
+ Step 27730/50462 | Loss: 2.5633 | LR: 2.17e-04 | 4005.5ms/step | 15.0 samples/s
1002
+ Step 27740/50462 | Loss: 2.5804 | LR: 2.16e-04 | 4008.7ms/step | 15.0 samples/s
1003
+ Step 27750/50462 | Loss: 2.5693 | LR: 2.16e-04 | 3995.8ms/step | 15.0 samples/s
1004
+ Validation at step 27750: Loss = 2.5557, Perplexity = 12.88
1005
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-27750
1006
+ Step 27760/50462 | Loss: 2.5820 | LR: 2.16e-04 | 5909.8ms/step | 10.2 samples/s
1007
+ Step 27770/50462 | Loss: 2.5693 | LR: 2.16e-04 | 3995.7ms/step | 15.0 samples/s
1008
+ Step 27780/50462 | Loss: 2.6004 | LR: 2.16e-04 | 3995.5ms/step | 15.0 samples/s
1009
+ Step 27790/50462 | Loss: 2.5483 | LR: 2.16e-04 | 4003.7ms/step | 15.0 samples/s
1010
+ Step 27800/50462 | Loss: 2.5653 | LR: 2.15e-04 | 4009.8ms/step | 15.0 samples/s
1011
+ Step 27810/50462 | Loss: 2.5357 | LR: 2.15e-04 | 4012.7ms/step | 15.0 samples/s
1012
+ Step 27820/50462 | Loss: 2.5600 | LR: 2.15e-04 | 4002.0ms/step | 15.0 samples/s
1013
+ Step 27830/50462 | Loss: 2.5827 | LR: 2.15e-04 | 4000.5ms/step | 15.0 samples/s
1014
+ Step 27840/50462 | Loss: 2.5719 | LR: 2.15e-04 | 3985.6ms/step | 15.1 samples/s
1015
+ Step 27850/50462 | Loss: 2.6025 | LR: 2.15e-04 | 3996.1ms/step | 15.0 samples/s
1016
+ Step 27860/50462 | Loss: 2.5799 | LR: 2.15e-04 | 4005.4ms/step | 15.0 samples/s
1017
+ Step 27870/50462 | Loss: 2.5761 | LR: 2.14e-04 | 4002.9ms/step | 15.0 samples/s
1018
+ Step 27880/50462 | Loss: 2.5400 | LR: 2.14e-04 | 4005.6ms/step | 15.0 samples/s
1019
+ Step 27890/50462 | Loss: 2.6089 | LR: 2.14e-04 | 4014.6ms/step | 14.9 samples/s
1020
+ Step 27900/50462 | Loss: 2.5541 | LR: 2.14e-04 | 3998.2ms/step | 15.0 samples/s
1021
+ Step 27910/50462 | Loss: 2.5587 | LR: 2.14e-04 | 3994.0ms/step | 15.0 samples/s
1022
+ Step 27920/50462 | Loss: 2.5748 | LR: 2.14e-04 | 3993.8ms/step | 15.0 samples/s
1023
+ Step 27930/50462 | Loss: 2.5611 | LR: 2.13e-04 | 4004.0ms/step | 15.0 samples/s
1024
+ Step 27940/50462 | Loss: 2.5887 | LR: 2.13e-04 | 4010.3ms/step | 15.0 samples/s
1025
+ Step 27950/50462 | Loss: 2.5430 | LR: 2.13e-04 | 4003.4ms/step | 15.0 samples/s
1026
+ Step 27960/50462 | Loss: 2.5855 | LR: 2.13e-04 | 3989.3ms/step | 15.0 samples/s
1027
+ Step 27970/50462 | Loss: 2.5676 | LR: 2.13e-04 | 3997.4ms/step | 15.0 samples/s
1028
+ Step 27980/50462 | Loss: 2.5574 | LR: 2.13e-04 | 3996.9ms/step | 15.0 samples/s
1029
+ Step 27990/50462 | Loss: 2.5700 | LR: 2.12e-04 | 4004.3ms/step | 15.0 samples/s
1030
+ Step 28000/50462 | Loss: 2.5788 | LR: 2.12e-04 | 3997.9ms/step | 15.0 samples/s
1031
+ Validation at step 28000: Loss = 2.5552, Perplexity = 12.87
1032
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28000
1033
+ Step 28010/50462 | Loss: 2.5542 | LR: 2.12e-04 | 5897.3ms/step | 10.2 samples/s
1034
+ Step 28020/50462 | Loss: 2.5689 | LR: 2.12e-04 | 4008.9ms/step | 15.0 samples/s
1035
+ Step 28030/50462 | Loss: 2.5306 | LR: 2.12e-04 | 4013.1ms/step | 15.0 samples/s
1036
+ Step 28040/50462 | Loss: 2.5590 | LR: 2.12e-04 | 4002.4ms/step | 15.0 samples/s
1037
+ Step 28050/50462 | Loss: 2.5491 | LR: 2.12e-04 | 3999.3ms/step | 15.0 samples/s
1038
+ Step 28060/50462 | Loss: 2.5716 | LR: 2.11e-04 | 4004.9ms/step | 15.0 samples/s
1039
+ Step 28070/50462 | Loss: 2.5797 | LR: 2.11e-04 | 4003.7ms/step | 15.0 samples/s
1040
+ Step 28080/50462 | Loss: 2.5786 | LR: 2.11e-04 | 3997.4ms/step | 15.0 samples/s
1041
+ Step 28090/50462 | Loss: 2.5528 | LR: 2.11e-04 | 4009.1ms/step | 15.0 samples/s
1042
+ Step 28100/50462 | Loss: 2.5619 | LR: 2.11e-04 | 4007.3ms/step | 15.0 samples/s
1043
+ Step 28110/50462 | Loss: 2.5442 | LR: 2.11e-04 | 4004.1ms/step | 15.0 samples/s
1044
+ Step 28120/50462 | Loss: 2.5819 | LR: 2.10e-04 | 4008.3ms/step | 15.0 samples/s
1045
+ Step 28130/50462 | Loss: 2.5483 | LR: 2.10e-04 | 3996.0ms/step | 15.0 samples/s
1046
+ Step 28140/50462 | Loss: 2.5941 | LR: 2.10e-04 | 3998.9ms/step | 15.0 samples/s
1047
+ Step 28150/50462 | Loss: 2.5835 | LR: 2.10e-04 | 4005.1ms/step | 15.0 samples/s
1048
+ Step 28160/50462 | Loss: 2.5544 | LR: 2.10e-04 | 4007.1ms/step | 15.0 samples/s
1049
+ Step 28170/50462 | Loss: 2.5295 | LR: 2.10e-04 | 4007.2ms/step | 15.0 samples/s
1050
+ Step 28180/50462 | Loss: 2.5733 | LR: 2.10e-04 | 4005.9ms/step | 15.0 samples/s
1051
+ Step 28190/50462 | Loss: 2.5658 | LR: 2.09e-04 | 4006.9ms/step | 15.0 samples/s
1052
+ Step 28200/50462 | Loss: 2.5727 | LR: 2.09e-04 | 4002.0ms/step | 15.0 samples/s
1053
+ Step 28210/50462 | Loss: 2.5753 | LR: 2.09e-04 | 3993.9ms/step | 15.0 samples/s
1054
+ Step 28220/50462 | Loss: 2.5715 | LR: 2.09e-04 | 3998.8ms/step | 15.0 samples/s
1055
+ Step 28230/50462 | Loss: 2.5193 | LR: 2.09e-04 | 4002.6ms/step | 15.0 samples/s
1056
+ Step 28240/50462 | Loss: 2.5965 | LR: 2.09e-04 | 3997.4ms/step | 15.0 samples/s
1057
+ Step 28250/50462 | Loss: 2.5152 | LR: 2.08e-04 | 4003.6ms/step | 15.0 samples/s
1058
+ Validation at step 28250: Loss = 2.5527, Perplexity = 12.84
1059
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28250
1060
+ Step 28260/50462 | Loss: 2.5543 | LR: 2.08e-04 | 5905.1ms/step | 10.2 samples/s
1061
+ Step 28270/50462 | Loss: 2.5699 | LR: 2.08e-04 | 4003.7ms/step | 15.0 samples/s
1062
+ Step 28280/50462 | Loss: 2.5745 | LR: 2.08e-04 | 4008.5ms/step | 15.0 samples/s
1063
+ Step 28290/50462 | Loss: 2.5631 | LR: 2.08e-04 | 4005.1ms/step | 15.0 samples/s
1064
+ Step 28300/50462 | Loss: 2.5898 | LR: 2.08e-04 | 4002.3ms/step | 15.0 samples/s
1065
+ Step 28310/50462 | Loss: 2.5663 | LR: 2.07e-04 | 4007.0ms/step | 15.0 samples/s
1066
+ Step 28320/50462 | Loss: 2.6217 | LR: 2.07e-04 | 4005.8ms/step | 15.0 samples/s
1067
+ Step 28330/50462 | Loss: 2.5402 | LR: 2.07e-04 | 4007.7ms/step | 15.0 samples/s
1068
+ Step 28340/50462 | Loss: 2.5357 | LR: 2.07e-04 | 4011.7ms/step | 15.0 samples/s
1069
+ Step 28350/50462 | Loss: 2.5634 | LR: 2.07e-04 | 4001.2ms/step | 15.0 samples/s
1070
+ Step 28360/50462 | Loss: 2.5303 | LR: 2.07e-04 | 3999.0ms/step | 15.0 samples/s
1071
+ Step 28370/50462 | Loss: 2.5356 | LR: 2.07e-04 | 3993.6ms/step | 15.0 samples/s
1072
+ Step 28380/50462 | Loss: 2.5667 | LR: 2.06e-04 | 3998.0ms/step | 15.0 samples/s
1073
+ Step 28390/50462 | Loss: 2.5117 | LR: 2.06e-04 | 4006.8ms/step | 15.0 samples/s
1074
+ Step 28400/50462 | Loss: 2.5875 | LR: 2.06e-04 | 4004.2ms/step | 15.0 samples/s
1075
+ Step 28410/50462 | Loss: 2.5496 | LR: 2.06e-04 | 4001.5ms/step | 15.0 samples/s
1076
+ Step 28420/50462 | Loss: 2.5435 | LR: 2.06e-04 | 4006.9ms/step | 15.0 samples/s
1077
+ Step 28430/50462 | Loss: 2.5572 | LR: 2.06e-04 | 4002.5ms/step | 15.0 samples/s
1078
+ Step 28440/50462 | Loss: 2.5398 | LR: 2.05e-04 | 3991.7ms/step | 15.0 samples/s
1079
+ Step 28450/50462 | Loss: 2.5629 | LR: 2.05e-04 | 4001.7ms/step | 15.0 samples/s
1080
+ Step 28460/50462 | Loss: 2.5405 | LR: 2.05e-04 | 4015.4ms/step | 14.9 samples/s
1081
+ Step 28470/50462 | Loss: 2.5613 | LR: 2.05e-04 | 3996.2ms/step | 15.0 samples/s
1082
+ Step 28480/50462 | Loss: 2.5636 | LR: 2.05e-04 | 4004.3ms/step | 15.0 samples/s
1083
+ Step 28490/50462 | Loss: 2.5680 | LR: 2.05e-04 | 3999.6ms/step | 15.0 samples/s
1084
+ Step 28500/50462 | Loss: 2.5813 | LR: 2.05e-04 | 4002.6ms/step | 15.0 samples/s
1085
+ Validation at step 28500: Loss = 2.5508, Perplexity = 12.82
1086
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28500
1087
+ Step 28510/50462 | Loss: 2.5457 | LR: 2.04e-04 | 5909.1ms/step | 10.2 samples/s
1088
+ Step 28520/50462 | Loss: 2.5713 | LR: 2.04e-04 | 4005.6ms/step | 15.0 samples/s
1089
+ Step 28530/50462 | Loss: 2.5483 | LR: 2.04e-04 | 4008.9ms/step | 15.0 samples/s
1090
+ Step 28540/50462 | Loss: 2.5646 | LR: 2.04e-04 | 3996.1ms/step | 15.0 samples/s
1091
+ Step 28550/50462 | Loss: 2.5907 | LR: 2.04e-04 | 3996.3ms/step | 15.0 samples/s
1092
+ Step 28560/50462 | Loss: 2.5594 | LR: 2.04e-04 | 4005.0ms/step | 15.0 samples/s
1093
+ Step 28570/50462 | Loss: 2.5596 | LR: 2.03e-04 | 4009.2ms/step | 15.0 samples/s
1094
+ Step 28580/50462 | Loss: 2.5632 | LR: 2.03e-04 | 4001.0ms/step | 15.0 samples/s
1095
+ Step 28590/50462 | Loss: 2.5861 | LR: 2.03e-04 | 4014.6ms/step | 14.9 samples/s
1096
+ Step 28600/50462 | Loss: 2.5440 | LR: 2.03e-04 | 4008.6ms/step | 15.0 samples/s
1097
+ Step 28610/50462 | Loss: 2.5932 | LR: 2.03e-04 | 4008.2ms/step | 15.0 samples/s
1098
+ Step 28620/50462 | Loss: 2.5402 | LR: 2.03e-04 | 4001.1ms/step | 15.0 samples/s
1099
+ Step 28630/50462 | Loss: 2.5786 | LR: 2.03e-04 | 4002.9ms/step | 15.0 samples/s
1100
+ Step 28640/50462 | Loss: 2.5801 | LR: 2.02e-04 | 4006.7ms/step | 15.0 samples/s
1101
+ Step 28650/50462 | Loss: 2.5695 | LR: 2.02e-04 | 4008.5ms/step | 15.0 samples/s
1102
+ Step 28660/50462 | Loss: 2.5655 | LR: 2.02e-04 | 4004.5ms/step | 15.0 samples/s
1103
+ Step 28670/50462 | Loss: 2.5694 | LR: 2.02e-04 | 4001.9ms/step | 15.0 samples/s
1104
+ Step 28680/50462 | Loss: 2.5620 | LR: 2.02e-04 | 4000.2ms/step | 15.0 samples/s
1105
+ Step 28690/50462 | Loss: 2.5732 | LR: 2.02e-04 | 3997.7ms/step | 15.0 samples/s
1106
+ Step 28700/50462 | Loss: 2.5999 | LR: 2.01e-04 | 4000.5ms/step | 15.0 samples/s
1107
+ Step 28710/50462 | Loss: 2.5713 | LR: 2.01e-04 | 4012.9ms/step | 15.0 samples/s
1108
+ Step 28720/50462 | Loss: 2.5216 | LR: 2.01e-04 | 3997.0ms/step | 15.0 samples/s
1109
+ Step 28730/50462 | Loss: 2.5630 | LR: 2.01e-04 | 4008.6ms/step | 15.0 samples/s
epfl_final/SparseAtt/wandb/run-20260827_163926-8ynn1we6/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260827_163926-8ynn1we6/run-8ynn1we6.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c05e2341274fe7dc6f9c7eab7ae7029910b62d3d4861da58f010e0a222909c9a
3
+ size 14221312
epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/files/output.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 8250: Loss = 2.8611, Perplexity = 17.48
3
+ wandb: WARNING Tried to log to step 8250 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 8251: sparse_stochastic (configured sparse attention)
5
+ Step 8260/50462 | Loss: 2.7394 | LR: 4.72e-04 | 27071.2ms/step | 2.2 samples/s
6
+ wandb: WARNING Tried to log to step 8260 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 8270/50462 | Loss: 2.7790 | LR: 4.72e-04 | 4034.1ms/step | 14.9 samples/s
8
+ wandb: WARNING Tried to log to step 8270 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 8280/50462 | Loss: 2.7315 | LR: 4.72e-04 | 4058.6ms/step | 14.8 samples/s
10
+ wandb: WARNING Tried to log to step 8280 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 8290/50462 | Loss: 2.7680 | LR: 4.72e-04 | 4040.9ms/step | 14.8 samples/s
12
+ wandb: WARNING Tried to log to step 8290 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 8300/50462 | Loss: 2.7817 | LR: 4.72e-04 | 4038.2ms/step | 14.9 samples/s
14
+ wandb: WARNING Tried to log to step 8300 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 8310/50462 | Loss: 2.7250 | LR: 4.72e-04 | 4035.8ms/step | 14.9 samples/s
16
+ wandb: WARNING Tried to log to step 8310 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 8320/50462 | Loss: 2.7829 | LR: 4.72e-04 | 4041.2ms/step | 14.8 samples/s
18
+ wandb: WARNING Tried to log to step 8320 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
19
+ Step 8330/50462 | Loss: 2.7627 | LR: 4.72e-04 | 4033.5ms/step | 14.9 samples/s
20
+ wandb: WARNING Tried to log to step 8330 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/files/wandb-metadata.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-27T16:39:45.292741Z",
5
+ "args": [
6
+ "--config",
7
+ "configs/p2_1p2b_mixed_seed72_4gpu_resume8250.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/SparseAtt.git",
14
+ "commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-train-4-gpu-0-21",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 4,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "6455472693248"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-a2122da0-1020-9f78-7e6f-a1d5011f694b"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-4c1d91e9-bf01-f8fd-1e8b-79aa8e0831df"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-086f9f0b-a6d8-6263-83e6-c2b66721f47f"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7"
61
+ }
62
+ ],
63
+ "cudaVersion": "13.0",
64
+ "writerId": "fnnegfltmnvbdyxntczzdhllarsub1fy"
65
+ }
epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/logs/debug-internal.log ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-08-27T16:39:45.885292324Z","level":"INFO","msg":"wandb-core"}
2
+ {"time":"2026-08-27T16:39:45.892434682Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
3
+ {"time":"2026-08-27T16:39:46.3194807Z","level":"INFO","msg":"stream: created new stream","id":"z1i60278"}
4
+ {"time":"2026-08-27T16:39:46.319576974Z","level":"INFO","msg":"handler: started"}
5
+ {"time":"2026-08-27T16:39:46.323191502Z","level":"INFO","msg":"stream: started"}
6
+ {"time":"2026-08-27T16:39:46.324179388Z","level":"INFO","msg":"sender: started"}
7
+ {"time":"2026-08-27T16:39:46.32418714Z","level":"INFO","msg":"writer: started","stream_id":"z1i60278"}
8
+ {"time":"2026-08-27T16:39:46.947009096Z","level":"INFO","msg":"filestream: sending request","total_files":1}
9
+ {"time":"2026-08-27T16:39:47.30556333Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
10
+ {"time":"2026-08-27T16:40:01.947408048Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1,"uploaded_len":2}
11
+ {"time":"2026-08-27T16:40:02.162576891Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
12
+ {"time":"2026-08-27T16:40:16.946783612Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
13
+ {"time":"2026-08-27T16:40:17.173208525Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
14
+ {"time":"2026-08-27T16:40:31.674002519Z","level":"WARN","msg":"handler: ignoring partial history record","step":8250,"current":8481}
15
+ {"time":"2026-08-27T16:40:31.947411796Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":2,"events_lines":2,"console_offset":1,"console_lines":1}
16
+ {"time":"2026-08-27T16:40:32.174725757Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
17
+ {"time":"2026-08-27T16:40:46.94715627Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":2,"console_lines":1}
18
+ {"time":"2026-08-27T16:40:47.185922526Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
19
+ {"time":"2026-08-27T16:41:01.947456385Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
20
+ {"time":"2026-08-27T16:41:02.216494483Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
21
+ {"time":"2026-08-27T16:41:16.946816655Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
22
+ {"time":"2026-08-27T16:41:17.160866727Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
23
+ {"time":"2026-08-27T16:41:31.947463755Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
24
+ {"time":"2026-08-27T16:41:32.163521041Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
25
+ {"time":"2026-08-27T16:41:46.947684316Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
26
+ {"time":"2026-08-27T16:41:47.164287348Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
27
+ {"time":"2026-08-27T16:42:01.947401238Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
28
+ {"time":"2026-08-27T16:42:02.126991569Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
29
+ {"time":"2026-08-27T16:42:16.947619498Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
30
+ {"time":"2026-08-27T16:42:17.16577628Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
31
+ {"time":"2026-08-27T16:42:31.947778789Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
32
+ {"time":"2026-08-27T16:42:32.156848071Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
33
+ {"time":"2026-08-27T16:42:46.947771392Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":20,"events_lines":2,"console_offset":3,"console_lines":1}
34
+ {"time":"2026-08-27T16:42:47.178977651Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
35
+ {"time":"2026-08-27T16:43:01.947701071Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
36
+ {"time":"2026-08-27T16:43:02.144723744Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
37
+ {"time":"2026-08-27T16:43:16.947034068Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
38
+ {"time":"2026-08-27T16:43:17.155590785Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
39
+ {"time":"2026-08-27T16:43:31.947678625Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
40
+ {"time":"2026-08-27T16:43:32.189987513Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
41
+ {"time":"2026-08-27T16:43:46.947689233Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
42
+ {"time":"2026-08-27T16:43:47.189449943Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
43
+ {"time":"2026-08-27T16:44:01.947746703Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
44
+ {"time":"2026-08-27T16:44:02.165275998Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
45
+ {"time":"2026-08-27T16:44:16.94736038Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
46
+ {"time":"2026-08-27T16:44:17.160341523Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
47
+ {"time":"2026-08-27T16:44:17.927881005Z","level":"WARN","msg":"handler: ignoring partial history record","step":8260,"current":8481}
48
+ {"time":"2026-08-27T16:44:31.947710667Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":34,"events_lines":2,"console_offset":4,"console_lines":2}
49
+ {"time":"2026-08-27T16:44:32.150283758Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
50
+ {"time":"2026-08-27T16:44:46.94754313Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
51
+ {"time":"2026-08-27T16:44:47.182021215Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
52
+ {"time":"2026-08-27T16:44:58.267816412Z","level":"WARN","msg":"handler: ignoring partial history record","step":8270,"current":8481}
53
+ {"time":"2026-08-27T16:45:01.947101194Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":38,"events_lines":2,"console_offset":6,"console_lines":1}
54
+ {"time":"2026-08-27T16:45:02.135359539Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
55
+ {"time":"2026-08-27T16:45:16.947800993Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":40,"events_lines":2,"console_offset":7,"console_lines":1}
56
+ {"time":"2026-08-27T16:45:17.165628834Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
57
+ {"time":"2026-08-27T16:45:31.947580498Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
58
+ {"time":"2026-08-27T16:45:32.157745796Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
59
+ {"time":"2026-08-27T16:45:38.852825515Z","level":"WARN","msg":"handler: ignoring partial history record","step":8280,"current":8481}
60
+ {"time":"2026-08-27T16:45:46.946869732Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":44,"events_lines":2,"console_offset":8,"console_lines":1}
61
+ {"time":"2026-08-27T16:45:47.148429662Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
62
+ {"time":"2026-08-27T16:46:01.947788015Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":46,"events_lines":2,"console_offset":9,"console_lines":1}
63
+ {"time":"2026-08-27T16:46:02.160739595Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
64
+ {"time":"2026-08-27T16:46:16.947331095Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
65
+ {"time":"2026-08-27T16:46:17.141261562Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
66
+ {"time":"2026-08-27T16:46:19.262271387Z","level":"WARN","msg":"handler: ignoring partial history record","step":8290,"current":8481}
67
+ {"time":"2026-08-27T16:46:31.947361733Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":50,"events_lines":2,"console_offset":10,"console_lines":2}
68
+ {"time":"2026-08-27T16:46:32.136178935Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
69
+ {"time":"2026-08-27T16:46:46.947192271Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":52,"events_lines":2}
70
+ {"time":"2026-08-27T16:46:47.160063771Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
71
+ {"time":"2026-08-27T16:46:59.64488824Z","level":"WARN","msg":"handler: ignoring partial history record","step":8300,"current":8481}
72
+ {"time":"2026-08-27T16:47:01.94770572Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":54,"events_lines":2,"console_offset":12,"console_lines":1}
73
+ {"time":"2026-08-27T16:47:02.157840023Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
74
+ {"time":"2026-08-27T16:47:16.947437653Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":56,"events_lines":2,"console_offset":13,"console_lines":1}
75
+ {"time":"2026-08-27T16:47:17.16699082Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
76
+ {"time":"2026-08-27T16:47:31.947329063Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
77
+ {"time":"2026-08-27T16:47:32.151531979Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
78
+ {"time":"2026-08-27T16:47:40.003430465Z","level":"WARN","msg":"handler: ignoring partial history record","step":8310,"current":8481}
79
+ {"time":"2026-08-27T16:47:46.947198741Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":60,"events_lines":2,"console_offset":14,"console_lines":1}
80
+ {"time":"2026-08-27T16:47:47.140599176Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
81
+ {"time":"2026-08-27T16:48:01.947605179Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":62,"events_lines":2,"console_offset":15,"console_lines":1}
82
+ {"time":"2026-08-27T16:48:02.148414258Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
83
+ {"time":"2026-08-27T16:48:16.947047539Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":64,"events_lines":2}
84
+ {"time":"2026-08-27T16:48:17.131806181Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
85
+ {"time":"2026-08-27T16:48:20.414970917Z","level":"WARN","msg":"handler: ignoring partial history record","step":8320,"current":8481}
86
+ {"time":"2026-08-27T16:48:31.947237124Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":66,"events_lines":2,"console_offset":16,"console_lines":2}
87
+ {"time":"2026-08-27T16:48:32.131230483Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
88
+ {"time":"2026-08-27T16:48:46.947786907Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
89
+ {"time":"2026-08-27T16:48:47.155931619Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
90
+ {"time":"2026-08-27T16:49:00.749868045Z","level":"WARN","msg":"handler: ignoring partial history record","step":8330,"current":8481}
91
+ {"time":"2026-08-27T16:49:01.946885113Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":70,"events_lines":2,"console_offset":18,"console_lines":1}
92
+ {"time":"2026-08-27T16:49:02.136252446Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
93
+ {"time":"2026-08-27T16:49:16.946825766Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":72,"events_lines":2,"console_offset":19,"console_lines":1}
94
+ {"time":"2026-08-27T16:49:17.147778469Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-27 16:39:45,313 INFO MainThread:221 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-27 16:39:45,313 INFO MainThread:221 [wandb_setup.py:_flush():81] Configure stats pid to 221
3
+ 2026-08-27 16:39:45,313 INFO MainThread:221 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-27 16:39:45,313 INFO MainThread:221 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_163945-z1i60278/logs/debug.log
5
+ 2026-08-27 16:39:45,313 INFO MainThread:221 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_163945-z1i60278/logs/debug-internal.log
6
+ 2026-08-27 16:39:45,313 INFO MainThread:221 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-27 16:39:45,313 INFO MainThread:221 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 72, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed72 6gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed72'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 72, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-27 16:39:45,313 INFO MainThread:221 [wandb_init.py:init():898] starting backend
10
+ 2026-08-27 16:39:45,866 INFO MainThread:221 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-27 16:39:46,324 INFO MainThread:221 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-27 16:39:46,328 INFO MainThread:221 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-27 16:39:46,368 INFO MainThread:221 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-27 16:39:46,942 INFO MainThread:221 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-27 16:39:46,945 INFO MainThread:221 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-27 16:39:47,200 INFO MainThread:221 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-27 16:39:47,200 INFO MainThread:221 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-27 16:39:47,201 INFO MainThread:221 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-27 16:39:47,201 INFO MainThread:221 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-27 16:39:47,210 INFO MainThread:221 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260827_163945-z1i60278/run-z1i60278.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5bada03eaa26fb78605e8f3706522dd15305965a1c6c796ae5812e062c7357f
3
+ size 163840
epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/files/output.log ADDED
@@ -0,0 +1,530 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 8250: Loss = 2.8611, Perplexity = 17.48
3
+ wandb: WARNING Tried to log to step 8250 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 8251: sparse_stochastic (configured sparse attention)
5
+ Step 8260/50462 | Loss: 2.7394 | LR: 4.72e-04 | 25210.2ms/step | 2.4 samples/s
6
+ wandb: WARNING Tried to log to step 8260 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 8270/50462 | Loss: 2.7790 | LR: 4.72e-04 | 3991.4ms/step | 15.0 samples/s
8
+ wandb: WARNING Tried to log to step 8270 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 8280/50462 | Loss: 2.7315 | LR: 4.72e-04 | 4020.8ms/step | 14.9 samples/s
10
+ wandb: WARNING Tried to log to step 8280 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 8290/50462 | Loss: 2.7680 | LR: 4.72e-04 | 4015.8ms/step | 14.9 samples/s
12
+ wandb: WARNING Tried to log to step 8290 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 8300/50462 | Loss: 2.7817 | LR: 4.72e-04 | 4016.7ms/step | 14.9 samples/s
14
+ wandb: WARNING Tried to log to step 8300 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 8310/50462 | Loss: 2.7250 | LR: 4.72e-04 | 4009.1ms/step | 15.0 samples/s
16
+ wandb: WARNING Tried to log to step 8310 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 8320/50462 | Loss: 2.7829 | LR: 4.72e-04 | 4016.9ms/step | 14.9 samples/s
18
+ wandb: WARNING Tried to log to step 8320 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
19
+ Step 8330/50462 | Loss: 2.7627 | LR: 4.72e-04 | 4009.6ms/step | 15.0 samples/s
20
+ wandb: WARNING Tried to log to step 8330 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
21
+ Step 8340/50462 | Loss: 2.7749 | LR: 4.72e-04 | 4038.2ms/step | 14.9 samples/s
22
+ wandb: WARNING Tried to log to step 8340 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
23
+ Step 8350/50462 | Loss: 2.7762 | LR: 4.72e-04 | 4017.8ms/step | 14.9 samples/s
24
+ wandb: WARNING Tried to log to step 8350 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
25
+ Step 8360/50462 | Loss: 2.7303 | LR: 4.72e-04 | 4024.9ms/step | 14.9 samples/s
26
+ wandb: WARNING Tried to log to step 8360 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
27
+ Step 8370/50462 | Loss: 2.7964 | LR: 4.72e-04 | 4015.3ms/step | 14.9 samples/s
28
+ wandb: WARNING Tried to log to step 8370 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
29
+ Step 8380/50462 | Loss: 2.7636 | LR: 4.71e-04 | 4016.7ms/step | 14.9 samples/s
30
+ wandb: WARNING Tried to log to step 8380 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
31
+ Step 8390/50462 | Loss: 2.7618 | LR: 4.71e-04 | 4010.3ms/step | 15.0 samples/s
32
+ wandb: WARNING Tried to log to step 8390 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
33
+ Step 8400/50462 | Loss: 2.7211 | LR: 4.71e-04 | 4020.6ms/step | 14.9 samples/s
34
+ wandb: WARNING Tried to log to step 8400 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
35
+ Step 8410/50462 | Loss: 2.7478 | LR: 4.71e-04 | 4009.5ms/step | 15.0 samples/s
36
+ wandb: WARNING Tried to log to step 8410 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
37
+ Step 8420/50462 | Loss: 2.7156 | LR: 4.71e-04 | 4014.7ms/step | 14.9 samples/s
38
+ wandb: WARNING Tried to log to step 8420 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
39
+ Step 8430/50462 | Loss: 2.7761 | LR: 4.71e-04 | 4013.9ms/step | 14.9 samples/s
40
+ wandb: WARNING Tried to log to step 8430 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
41
+ Step 8440/50462 | Loss: 2.7260 | LR: 4.71e-04 | 4007.1ms/step | 15.0 samples/s
42
+ wandb: WARNING Tried to log to step 8440 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
43
+ Step 8450/50462 | Loss: 2.7464 | LR: 4.71e-04 | 4007.6ms/step | 15.0 samples/s
44
+ wandb: WARNING Tried to log to step 8450 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
45
+ Step 8460/50462 | Loss: 2.7657 | LR: 4.71e-04 | 4015.8ms/step | 14.9 samples/s
46
+ wandb: WARNING Tried to log to step 8460 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
47
+ Step 8470/50462 | Loss: 2.7412 | LR: 4.71e-04 | 4017.0ms/step | 14.9 samples/s
48
+ wandb: WARNING Tried to log to step 8470 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
49
+ Step 8480/50462 | Loss: 2.7585 | LR: 4.71e-04 | 4021.3ms/step | 14.9 samples/s
50
+ wandb: WARNING Tried to log to step 8480 that is less than the current step 8481. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
51
+ Step 8490/50462 | Loss: 2.7542 | LR: 4.71e-04 | 3999.6ms/step | 15.0 samples/s
52
+ Step 8500/50462 | Loss: 2.7541 | LR: 4.71e-04 | 4006.5ms/step | 15.0 samples/s
53
+ Validation at step 8500: Loss = 2.8565, Perplexity = 17.40
54
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8500
55
+ Step 8510/50462 | Loss: 2.7523 | LR: 4.71e-04 | 5919.3ms/step | 10.1 samples/s
56
+ Step 8520/50462 | Loss: 2.7336 | LR: 4.70e-04 | 4008.2ms/step | 15.0 samples/s
57
+ Step 8530/50462 | Loss: 2.7116 | LR: 4.70e-04 | 4008.3ms/step | 15.0 samples/s
58
+ Step 8540/50462 | Loss: 2.7526 | LR: 4.70e-04 | 4029.1ms/step | 14.9 samples/s
59
+ Step 8550/50462 | Loss: 2.7305 | LR: 4.70e-04 | 4015.4ms/step | 14.9 samples/s
60
+ Step 8560/50462 | Loss: 2.7267 | LR: 4.70e-04 | 4018.8ms/step | 14.9 samples/s
61
+ Step 8570/50462 | Loss: 2.7103 | LR: 4.70e-04 | 4002.2ms/step | 15.0 samples/s
62
+ Step 8580/50462 | Loss: 2.7249 | LR: 4.70e-04 | 3995.4ms/step | 15.0 samples/s
63
+ Step 8590/50462 | Loss: 2.7633 | LR: 4.70e-04 | 4011.9ms/step | 15.0 samples/s
64
+ Step 8600/50462 | Loss: 2.7553 | LR: 4.70e-04 | 4015.2ms/step | 14.9 samples/s
65
+ Step 8610/50462 | Loss: 2.7673 | LR: 4.70e-04 | 4012.2ms/step | 15.0 samples/s
66
+ Step 8620/50462 | Loss: 2.7474 | LR: 4.70e-04 | 4012.7ms/step | 15.0 samples/s
67
+ Step 8630/50462 | Loss: 2.7741 | LR: 4.70e-04 | 4013.2ms/step | 15.0 samples/s
68
+ Step 8640/50462 | Loss: 2.7566 | LR: 4.70e-04 | 4021.0ms/step | 14.9 samples/s
69
+ Step 8650/50462 | Loss: 2.7237 | LR: 4.69e-04 | 4015.9ms/step | 14.9 samples/s
70
+ Step 8660/50462 | Loss: 2.7136 | LR: 4.69e-04 | 4014.4ms/step | 14.9 samples/s
71
+ Step 8670/50462 | Loss: 2.8036 | LR: 4.69e-04 | 4024.7ms/step | 14.9 samples/s
72
+ Step 8680/50462 | Loss: 2.8222 | LR: 4.69e-04 | 4029.9ms/step | 14.9 samples/s
73
+ Step 8690/50462 | Loss: 2.8280 | LR: 4.69e-04 | 4020.4ms/step | 14.9 samples/s
74
+ Step 8700/50462 | Loss: 2.8465 | LR: 4.69e-04 | 4024.2ms/step | 14.9 samples/s
75
+ Step 8710/50462 | Loss: 2.8272 | LR: 4.69e-04 | 4015.9ms/step | 14.9 samples/s
76
+ Step 8720/50462 | Loss: 2.8590 | LR: 4.69e-04 | 4022.5ms/step | 14.9 samples/s
77
+ Step 8730/50462 | Loss: 2.8163 | LR: 4.69e-04 | 4018.4ms/step | 14.9 samples/s
78
+ Step 8740/50462 | Loss: 2.8061 | LR: 4.69e-04 | 4008.2ms/step | 15.0 samples/s
79
+ Step 8750/50462 | Loss: 2.8217 | LR: 4.69e-04 | 4016.9ms/step | 14.9 samples/s
80
+ Validation at step 8750: Loss = 2.8455, Perplexity = 17.21
81
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8750
82
+ Step 8760/50462 | Loss: 2.8314 | LR: 4.69e-04 | 5884.7ms/step | 10.2 samples/s
83
+ Step 8770/50462 | Loss: 2.8724 | LR: 4.69e-04 | 4015.4ms/step | 14.9 samples/s
84
+ Step 8780/50462 | Loss: 2.8087 | LR: 4.68e-04 | 4021.4ms/step | 14.9 samples/s
85
+ Step 8790/50462 | Loss: 2.7953 | LR: 4.68e-04 | 4009.5ms/step | 15.0 samples/s
86
+ Step 8800/50462 | Loss: 2.8772 | LR: 4.68e-04 | 4021.5ms/step | 14.9 samples/s
87
+ Step 8810/50462 | Loss: 2.8346 | LR: 4.68e-04 | 4009.5ms/step | 15.0 samples/s
88
+ Step 8820/50462 | Loss: 2.8667 | LR: 4.68e-04 | 4011.9ms/step | 15.0 samples/s
89
+ Step 8830/50462 | Loss: 2.8352 | LR: 4.68e-04 | 4018.1ms/step | 14.9 samples/s
90
+ Step 8840/50462 | Loss: 2.8236 | LR: 4.68e-04 | 4006.4ms/step | 15.0 samples/s
91
+ Step 8850/50462 | Loss: 2.8225 | LR: 4.68e-04 | 4020.6ms/step | 14.9 samples/s
92
+ Step 8860/50462 | Loss: 2.8112 | LR: 4.68e-04 | 4011.9ms/step | 15.0 samples/s
93
+ Step 8870/50462 | Loss: 2.8148 | LR: 4.68e-04 | 4022.2ms/step | 14.9 samples/s
94
+ Step 8880/50462 | Loss: 2.7695 | LR: 4.68e-04 | 4017.1ms/step | 14.9 samples/s
95
+ Step 8890/50462 | Loss: 2.8272 | LR: 4.68e-04 | 4013.3ms/step | 15.0 samples/s
96
+ Step 8900/50462 | Loss: 2.8745 | LR: 4.68e-04 | 4007.8ms/step | 15.0 samples/s
97
+ Step 8910/50462 | Loss: 2.8406 | LR: 4.67e-04 | 4009.2ms/step | 15.0 samples/s
98
+ Step 8920/50462 | Loss: 2.7976 | LR: 4.67e-04 | 4011.9ms/step | 15.0 samples/s
99
+ Step 8930/50462 | Loss: 2.7960 | LR: 4.67e-04 | 4014.9ms/step | 14.9 samples/s
100
+ Step 8940/50462 | Loss: 2.8124 | LR: 4.67e-04 | 4015.2ms/step | 14.9 samples/s
101
+ Step 8950/50462 | Loss: 2.8485 | LR: 4.67e-04 | 4019.0ms/step | 14.9 samples/s
102
+ Step 8960/50462 | Loss: 2.8009 | LR: 4.67e-04 | 4017.8ms/step | 14.9 samples/s
103
+ Step 8970/50462 | Loss: 2.8190 | LR: 4.67e-04 | 4016.7ms/step | 14.9 samples/s
104
+ Step 8980/50462 | Loss: 2.8171 | LR: 4.67e-04 | 4007.4ms/step | 15.0 samples/s
105
+ Step 8990/50462 | Loss: 2.8261 | LR: 4.67e-04 | 4018.3ms/step | 14.9 samples/s
106
+ Step 9000/50462 | Loss: 2.8473 | LR: 4.67e-04 | 4007.5ms/step | 15.0 samples/s
107
+ Validation at step 9000: Loss = 2.8357, Perplexity = 17.04
108
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-9000
109
+ Step 9010/50462 | Loss: 2.8311 | LR: 4.67e-04 | 5880.5ms/step | 10.2 samples/s
110
+ Step 9020/50462 | Loss: 2.8378 | LR: 4.67e-04 | 4011.7ms/step | 15.0 samples/s
111
+ Step 9030/50462 | Loss: 2.8148 | LR: 4.67e-04 | 4005.1ms/step | 15.0 samples/s
112
+ Step 9040/50462 | Loss: 2.7948 | LR: 4.66e-04 | 4015.1ms/step | 14.9 samples/s
113
+ Step 9050/50462 | Loss: 2.8323 | LR: 4.66e-04 | 4024.6ms/step | 14.9 samples/s
114
+ Step 9060/50462 | Loss: 2.8213 | LR: 4.66e-04 | 4019.5ms/step | 14.9 samples/s
115
+ Step 9070/50462 | Loss: 2.8456 | LR: 4.66e-04 | 4018.8ms/step | 14.9 samples/s
116
+ Step 9080/50462 | Loss: 2.8299 | LR: 4.66e-04 | 4008.9ms/step | 15.0 samples/s
117
+ Step 9090/50462 | Loss: 2.7960 | LR: 4.66e-04 | 4004.9ms/step | 15.0 samples/s
118
+ Step 9100/50462 | Loss: 2.7983 | LR: 4.66e-04 | 4015.7ms/step | 14.9 samples/s
119
+ Step 9110/50462 | Loss: 2.8215 | LR: 4.66e-04 | 4012.0ms/step | 15.0 samples/s
120
+ Step 9120/50462 | Loss: 2.7971 | LR: 4.66e-04 | 4013.7ms/step | 14.9 samples/s
121
+ Step 9130/50462 | Loss: 2.7712 | LR: 4.66e-04 | 4028.7ms/step | 14.9 samples/s
122
+ Step 9140/50462 | Loss: 2.8209 | LR: 4.66e-04 | 4012.0ms/step | 15.0 samples/s
123
+ Step 9150/50462 | Loss: 2.8109 | LR: 4.66e-04 | 4009.6ms/step | 15.0 samples/s
124
+ Step 9160/50462 | Loss: 2.8104 | LR: 4.66e-04 | 4021.8ms/step | 14.9 samples/s
125
+ Step 9170/50462 | Loss: 2.7905 | LR: 4.65e-04 | 4007.8ms/step | 15.0 samples/s
126
+ Step 9180/50462 | Loss: 2.8504 | LR: 4.65e-04 | 4033.6ms/step | 14.9 samples/s
127
+ Step 9190/50462 | Loss: 2.8212 | LR: 4.65e-04 | 4004.8ms/step | 15.0 samples/s
128
+ Step 9200/50462 | Loss: 2.8102 | LR: 4.65e-04 | 4023.9ms/step | 14.9 samples/s
129
+ Step 9210/50462 | Loss: 2.8253 | LR: 4.65e-04 | 4020.1ms/step | 14.9 samples/s
130
+ Step 9220/50462 | Loss: 2.8055 | LR: 4.65e-04 | 4013.8ms/step | 14.9 samples/s
131
+ Step 9230/50462 | Loss: 2.7812 | LR: 4.65e-04 | 4015.6ms/step | 14.9 samples/s
132
+ Step 9240/50462 | Loss: 2.8088 | LR: 4.65e-04 | 4023.0ms/step | 14.9 samples/s
133
+ Step 9250/50462 | Loss: 2.8111 | LR: 4.65e-04 | 4014.6ms/step | 14.9 samples/s
134
+ Validation at step 9250: Loss = 2.8286, Perplexity = 16.92
135
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-9250
136
+ Step 9260/50462 | Loss: 2.8137 | LR: 4.65e-04 | 5883.7ms/step | 10.2 samples/s
137
+ Step 9270/50462 | Loss: 2.7981 | LR: 4.65e-04 | 4009.0ms/step | 15.0 samples/s
138
+ Step 9280/50462 | Loss: 2.7995 | LR: 4.65e-04 | 3999.5ms/step | 15.0 samples/s
139
+ Step 9290/50462 | Loss: 2.7876 | LR: 4.64e-04 | 4025.9ms/step | 14.9 samples/s
140
+ Step 9300/50462 | Loss: 2.8177 | LR: 4.64e-04 | 4021.1ms/step | 14.9 samples/s
141
+ Step 9310/50462 | Loss: 2.8394 | LR: 4.64e-04 | 4004.6ms/step | 15.0 samples/s
142
+ Step 9320/50462 | Loss: 2.8334 | LR: 4.64e-04 | 4027.7ms/step | 14.9 samples/s
143
+ Step 9330/50462 | Loss: 2.8372 | LR: 4.64e-04 | 4022.2ms/step | 14.9 samples/s
144
+ Step 9340/50462 | Loss: 2.7949 | LR: 4.64e-04 | 4023.2ms/step | 14.9 samples/s
145
+ Step 9350/50462 | Loss: 2.7918 | LR: 4.64e-04 | 4008.0ms/step | 15.0 samples/s
146
+ Step 9360/50462 | Loss: 2.8171 | LR: 4.64e-04 | 4012.9ms/step | 15.0 samples/s
147
+ Step 9370/50462 | Loss: 2.8048 | LR: 4.64e-04 | 4011.2ms/step | 15.0 samples/s
148
+ Step 9380/50462 | Loss: 2.8022 | LR: 4.64e-04 | 4005.3ms/step | 15.0 samples/s
149
+ Step 9390/50462 | Loss: 2.7919 | LR: 4.64e-04 | 4014.2ms/step | 14.9 samples/s
150
+ Step 9400/50462 | Loss: 2.7971 | LR: 4.64e-04 | 4009.3ms/step | 15.0 samples/s
151
+ Step 9410/50462 | Loss: 2.8152 | LR: 4.63e-04 | 4019.5ms/step | 14.9 samples/s
152
+ Step 9420/50462 | Loss: 2.8324 | LR: 4.63e-04 | 4004.6ms/step | 15.0 samples/s
153
+ Step 9430/50462 | Loss: 2.8018 | LR: 4.63e-04 | 4020.3ms/step | 14.9 samples/s
154
+ Step 9440/50462 | Loss: 2.8072 | LR: 4.63e-04 | 4019.4ms/step | 14.9 samples/s
155
+ Step 9450/50462 | Loss: 2.8193 | LR: 4.63e-04 | 4015.0ms/step | 14.9 samples/s
156
+ Step 9460/50462 | Loss: 2.8045 | LR: 4.63e-04 | 4022.0ms/step | 14.9 samples/s
157
+ Step 9470/50462 | Loss: 2.8386 | LR: 4.63e-04 | 4018.2ms/step | 14.9 samples/s
158
+ Step 9480/50462 | Loss: 2.8338 | LR: 4.63e-04 | 4019.8ms/step | 14.9 samples/s
159
+ Step 9490/50462 | Loss: 2.7984 | LR: 4.63e-04 | 4011.7ms/step | 15.0 samples/s
160
+ Step 9500/50462 | Loss: 2.8064 | LR: 4.63e-04 | 4014.3ms/step | 14.9 samples/s
161
+ Validation at step 9500: Loss = 2.8192, Perplexity = 16.76
162
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-9500
163
+ Step 9510/50462 | Loss: 2.8204 | LR: 4.63e-04 | 5898.9ms/step | 10.2 samples/s
164
+ Step 9520/50462 | Loss: 2.8014 | LR: 4.63e-04 | 4012.8ms/step | 15.0 samples/s
165
+ Step 9530/50462 | Loss: 2.7959 | LR: 4.62e-04 | 4010.5ms/step | 15.0 samples/s
166
+ Step 9540/50462 | Loss: 2.8346 | LR: 4.62e-04 | 4005.9ms/step | 15.0 samples/s
167
+ Step 9550/50462 | Loss: 2.7763 | LR: 4.62e-04 | 4011.1ms/step | 15.0 samples/s
168
+ Step 9560/50462 | Loss: 2.7748 | LR: 4.62e-04 | 4019.2ms/step | 14.9 samples/s
169
+ Step 9570/50462 | Loss: 2.8238 | LR: 4.62e-04 | 4013.3ms/step | 15.0 samples/s
170
+ Step 9580/50462 | Loss: 2.7983 | LR: 4.62e-04 | 4021.7ms/step | 14.9 samples/s
171
+ Step 9590/50462 | Loss: 2.8049 | LR: 4.62e-04 | 4018.6ms/step | 14.9 samples/s
172
+ Step 9600/50462 | Loss: 2.7710 | LR: 4.62e-04 | 4009.2ms/step | 15.0 samples/s
173
+ Step 9610/50462 | Loss: 2.7836 | LR: 4.62e-04 | 4015.4ms/step | 14.9 samples/s
174
+ Step 9620/50462 | Loss: 2.7863 | LR: 4.62e-04 | 4017.3ms/step | 14.9 samples/s
175
+ Step 9630/50462 | Loss: 2.8027 | LR: 4.62e-04 | 4017.6ms/step | 14.9 samples/s
176
+ Step 9640/50462 | Loss: 2.7988 | LR: 4.62e-04 | 4010.6ms/step | 15.0 samples/s
177
+ Step 9650/50462 | Loss: 2.8145 | LR: 4.61e-04 | 4020.0ms/step | 14.9 samples/s
178
+ Step 9660/50462 | Loss: 2.8111 | LR: 4.61e-04 | 4013.2ms/step | 15.0 samples/s
179
+ Step 9670/50462 | Loss: 2.8097 | LR: 4.61e-04 | 4017.9ms/step | 14.9 samples/s
180
+ Step 9680/50462 | Loss: 2.7806 | LR: 4.61e-04 | 4017.8ms/step | 14.9 samples/s
181
+ Step 9690/50462 | Loss: 2.8079 | LR: 4.61e-04 | 4012.1ms/step | 15.0 samples/s
182
+ Step 9700/50462 | Loss: 2.8260 | LR: 4.61e-04 | 4023.9ms/step | 14.9 samples/s
183
+ Step 9710/50462 | Loss: 2.7815 | LR: 4.61e-04 | 4016.6ms/step | 14.9 samples/s
184
+ Step 9720/50462 | Loss: 2.8069 | LR: 4.61e-04 | 4017.2ms/step | 14.9 samples/s
185
+ Step 9730/50462 | Loss: 2.8046 | LR: 4.61e-04 | 4016.4ms/step | 14.9 samples/s
186
+ Step 9740/50462 | Loss: 2.7809 | LR: 4.61e-04 | 4022.8ms/step | 14.9 samples/s
187
+ Step 9750/50462 | Loss: 2.7793 | LR: 4.61e-04 | 4003.0ms/step | 15.0 samples/s
188
+ Validation at step 9750: Loss = 2.8154, Perplexity = 16.70
189
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-9750
190
+ Step 9760/50462 | Loss: 2.8017 | LR: 4.61e-04 | 5874.9ms/step | 10.2 samples/s
191
+ Step 9770/50462 | Loss: 2.7777 | LR: 4.60e-04 | 4014.6ms/step | 14.9 samples/s
192
+ Step 9780/50462 | Loss: 2.7877 | LR: 4.60e-04 | 4011.5ms/step | 15.0 samples/s
193
+ Step 9790/50462 | Loss: 2.7952 | LR: 4.60e-04 | 4003.3ms/step | 15.0 samples/s
194
+ Step 9800/50462 | Loss: 2.7946 | LR: 4.60e-04 | 4023.7ms/step | 14.9 samples/s
195
+ Step 9810/50462 | Loss: 2.7983 | LR: 4.60e-04 | 4010.8ms/step | 15.0 samples/s
196
+ Step 9820/50462 | Loss: 2.7709 | LR: 4.60e-04 | 4000.0ms/step | 15.0 samples/s
197
+ Step 9830/50462 | Loss: 2.7854 | LR: 4.60e-04 | 4016.0ms/step | 14.9 samples/s
198
+ Step 9840/50462 | Loss: 2.7656 | LR: 4.60e-04 | 4010.2ms/step | 15.0 samples/s
199
+ Step 9850/50462 | Loss: 2.8045 | LR: 4.60e-04 | 3999.0ms/step | 15.0 samples/s
200
+ Step 9860/50462 | Loss: 2.7647 | LR: 4.60e-04 | 4006.5ms/step | 15.0 samples/s
201
+ Step 9870/50462 | Loss: 2.7849 | LR: 4.60e-04 | 4002.6ms/step | 15.0 samples/s
202
+ Step 9880/50462 | Loss: 2.8040 | LR: 4.60e-04 | 4011.2ms/step | 15.0 samples/s
203
+ Step 9890/50462 | Loss: 2.7795 | LR: 4.59e-04 | 4005.4ms/step | 15.0 samples/s
204
+ Step 9900/50462 | Loss: 2.7902 | LR: 4.59e-04 | 4005.3ms/step | 15.0 samples/s
205
+ Step 9910/50462 | Loss: 2.8048 | LR: 4.59e-04 | 4003.8ms/step | 15.0 samples/s
206
+ Step 9920/50462 | Loss: 2.8062 | LR: 4.59e-04 | 4017.1ms/step | 14.9 samples/s
207
+ Step 9930/50462 | Loss: 2.7986 | LR: 4.59e-04 | 4018.9ms/step | 14.9 samples/s
208
+ Step 9940/50462 | Loss: 2.7825 | LR: 4.59e-04 | 4012.5ms/step | 15.0 samples/s
209
+ Step 9950/50462 | Loss: 2.7723 | LR: 4.59e-04 | 4013.2ms/step | 15.0 samples/s
210
+ Step 9960/50462 | Loss: 2.7699 | LR: 4.59e-04 | 4020.7ms/step | 14.9 samples/s
211
+ Step 9970/50462 | Loss: 2.7890 | LR: 4.59e-04 | 4020.8ms/step | 14.9 samples/s
212
+ Step 9980/50462 | Loss: 2.7719 | LR: 4.59e-04 | 4005.5ms/step | 15.0 samples/s
213
+ Step 9990/50462 | Loss: 2.8018 | LR: 4.59e-04 | 4009.5ms/step | 15.0 samples/s
214
+ Step 10000/50462 | Loss: 2.8052 | LR: 4.58e-04 | 4000.1ms/step | 15.0 samples/s
215
+ Validation at step 10000: Loss = 2.8076, Perplexity = 16.57
216
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-10000
217
+ Step 10010/50462 | Loss: 2.7814 | LR: 4.58e-04 | 5878.6ms/step | 10.2 samples/s
218
+ Step 10020/50462 | Loss: 2.8112 | LR: 4.58e-04 | 4012.9ms/step | 15.0 samples/s
219
+ Step 10030/50462 | Loss: 2.7894 | LR: 4.58e-04 | 4002.4ms/step | 15.0 samples/s
220
+ Step 10040/50462 | Loss: 2.8066 | LR: 4.58e-04 | 4009.1ms/step | 15.0 samples/s
221
+ Step 10050/50462 | Loss: 2.8003 | LR: 4.58e-04 | 4011.7ms/step | 15.0 samples/s
222
+ Step 10060/50462 | Loss: 2.7596 | LR: 4.58e-04 | 4001.0ms/step | 15.0 samples/s
223
+ Step 10070/50462 | Loss: 2.7947 | LR: 4.58e-04 | 4009.9ms/step | 15.0 samples/s
224
+ Step 10080/50462 | Loss: 2.7670 | LR: 4.58e-04 | 4011.4ms/step | 15.0 samples/s
225
+ Step 10090/50462 | Loss: 2.7960 | LR: 4.58e-04 | 4004.5ms/step | 15.0 samples/s
226
+ Step 10100/50462 | Loss: 2.7781 | LR: 4.58e-04 | 4015.6ms/step | 14.9 samples/s
227
+ Step 10110/50462 | Loss: 2.8010 | LR: 4.58e-04 | 4004.4ms/step | 15.0 samples/s
228
+ Step 10120/50462 | Loss: 2.7916 | LR: 4.57e-04 | 4008.9ms/step | 15.0 samples/s
229
+ Step 10130/50462 | Loss: 2.7793 | LR: 4.57e-04 | 4002.4ms/step | 15.0 samples/s
230
+ Step 10140/50462 | Loss: 2.7984 | LR: 4.57e-04 | 3999.9ms/step | 15.0 samples/s
231
+ Step 10150/50462 | Loss: 2.7354 | LR: 4.57e-04 | 4003.4ms/step | 15.0 samples/s
232
+ Step 10160/50462 | Loss: 2.7629 | LR: 4.57e-04 | 4004.8ms/step | 15.0 samples/s
233
+ Step 10170/50462 | Loss: 2.7799 | LR: 4.57e-04 | 4004.7ms/step | 15.0 samples/s
234
+ Step 10180/50462 | Loss: 2.7932 | LR: 4.57e-04 | 4012.8ms/step | 15.0 samples/s
235
+ Step 10190/50462 | Loss: 2.7606 | LR: 4.57e-04 | 4014.2ms/step | 14.9 samples/s
236
+ Step 10200/50462 | Loss: 2.7596 | LR: 4.57e-04 | 4013.4ms/step | 14.9 samples/s
237
+ Step 10210/50462 | Loss: 2.8070 | LR: 4.57e-04 | 4017.8ms/step | 14.9 samples/s
238
+ Step 10220/50462 | Loss: 2.7870 | LR: 4.57e-04 | 4020.3ms/step | 14.9 samples/s
239
+ Step 10230/50462 | Loss: 2.8149 | LR: 4.56e-04 | 4018.6ms/step | 14.9 samples/s
240
+ Step 10240/50462 | Loss: 2.8051 | LR: 4.56e-04 | 4009.7ms/step | 15.0 samples/s
241
+ Step 10250/50462 | Loss: 2.7936 | LR: 4.56e-04 | 4005.4ms/step | 15.0 samples/s
242
+ Validation at step 10250: Loss = 2.8017, Perplexity = 16.47
243
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-10250
244
+ Step 10260/50462 | Loss: 2.7959 | LR: 4.56e-04 | 5905.0ms/step | 10.2 samples/s
245
+ Step 10270/50462 | Loss: 2.7910 | LR: 4.56e-04 | 4003.2ms/step | 15.0 samples/s
246
+ Step 10280/50462 | Loss: 2.7763 | LR: 4.56e-04 | 4013.8ms/step | 14.9 samples/s
247
+ Step 10290/50462 | Loss: 2.7785 | LR: 4.56e-04 | 4014.9ms/step | 14.9 samples/s
248
+ Step 10300/50462 | Loss: 2.7828 | LR: 4.56e-04 | 4007.6ms/step | 15.0 samples/s
249
+ Step 10310/50462 | Loss: 2.7958 | LR: 4.56e-04 | 4011.4ms/step | 15.0 samples/s
250
+ Step 10320/50462 | Loss: 2.7848 | LR: 4.56e-04 | 4008.6ms/step | 15.0 samples/s
251
+ Step 10330/50462 | Loss: 2.7910 | LR: 4.56e-04 | 4014.5ms/step | 14.9 samples/s
252
+ Step 10340/50462 | Loss: 2.7757 | LR: 4.55e-04 | 4010.6ms/step | 15.0 samples/s
253
+ Step 10350/50462 | Loss: 2.7573 | LR: 4.55e-04 | 4007.0ms/step | 15.0 samples/s
254
+ Step 10360/50462 | Loss: 2.7826 | LR: 4.55e-04 | 4005.7ms/step | 15.0 samples/s
255
+ Step 10370/50462 | Loss: 2.7906 | LR: 4.55e-04 | 4013.8ms/step | 14.9 samples/s
256
+ Step 10380/50462 | Loss: 2.8095 | LR: 4.55e-04 | 4008.1ms/step | 15.0 samples/s
257
+ Step 10390/50462 | Loss: 2.7638 | LR: 4.55e-04 | 4017.4ms/step | 14.9 samples/s
258
+ Step 10400/50462 | Loss: 2.7800 | LR: 4.55e-04 | 4016.9ms/step | 14.9 samples/s
259
+ Step 10410/50462 | Loss: 2.7536 | LR: 4.55e-04 | 4005.4ms/step | 15.0 samples/s
260
+ Step 10420/50462 | Loss: 2.7792 | LR: 4.55e-04 | 4007.4ms/step | 15.0 samples/s
261
+ Step 10430/50462 | Loss: 2.7685 | LR: 4.55e-04 | 3999.4ms/step | 15.0 samples/s
262
+ Step 10440/50462 | Loss: 2.7627 | LR: 4.55e-04 | 4006.5ms/step | 15.0 samples/s
263
+ Step 10450/50462 | Loss: 2.8025 | LR: 4.54e-04 | 4011.8ms/step | 15.0 samples/s
264
+ Step 10460/50462 | Loss: 2.7610 | LR: 4.54e-04 | 3999.2ms/step | 15.0 samples/s
265
+ Step 10470/50462 | Loss: 2.7901 | LR: 4.54e-04 | 4004.7ms/step | 15.0 samples/s
266
+ Step 10480/50462 | Loss: 2.7906 | LR: 4.54e-04 | 4018.9ms/step | 14.9 samples/s
267
+ Step 10490/50462 | Loss: 2.7912 | LR: 4.54e-04 | 4011.5ms/step | 15.0 samples/s
268
+ Step 10500/50462 | Loss: 2.7794 | LR: 4.54e-04 | 3998.6ms/step | 15.0 samples/s
269
+ Validation at step 10500: Loss = 2.7957, Perplexity = 16.37
270
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-10500
271
+ Step 10510/50462 | Loss: 2.7576 | LR: 4.54e-04 | 5884.5ms/step | 10.2 samples/s
272
+ Step 10520/50462 | Loss: 2.7676 | LR: 4.54e-04 | 4014.0ms/step | 14.9 samples/s
273
+ Step 10530/50462 | Loss: 2.7977 | LR: 4.54e-04 | 4012.0ms/step | 15.0 samples/s
274
+ Step 10540/50462 | Loss: 2.7865 | LR: 4.54e-04 | 4020.6ms/step | 14.9 samples/s
275
+ Step 10550/50462 | Loss: 2.8009 | LR: 4.54e-04 | 4022.8ms/step | 14.9 samples/s
276
+ Step 10560/50462 | Loss: 2.7620 | LR: 4.53e-04 | 4016.7ms/step | 14.9 samples/s
277
+ Step 10570/50462 | Loss: 2.7682 | LR: 4.53e-04 | 4022.1ms/step | 14.9 samples/s
278
+ Step 10580/50462 | Loss: 2.7385 | LR: 4.53e-04 | 4009.0ms/step | 15.0 samples/s
279
+ Step 10590/50462 | Loss: 2.7896 | LR: 4.53e-04 | 4009.3ms/step | 15.0 samples/s
280
+ Step 10600/50462 | Loss: 2.7033 | LR: 4.53e-04 | 4010.1ms/step | 15.0 samples/s
281
+ Step 10610/50462 | Loss: 2.7826 | LR: 4.53e-04 | 4001.9ms/step | 15.0 samples/s
282
+ Step 10620/50462 | Loss: 2.7800 | LR: 4.53e-04 | 4010.0ms/step | 15.0 samples/s
283
+ Step 10630/50462 | Loss: 2.7960 | LR: 4.53e-04 | 4014.3ms/step | 14.9 samples/s
284
+ Step 10640/50462 | Loss: 2.8079 | LR: 4.53e-04 | 4016.8ms/step | 14.9 samples/s
285
+ Step 10650/50462 | Loss: 2.8131 | LR: 4.53e-04 | 4008.5ms/step | 15.0 samples/s
286
+ Step 10660/50462 | Loss: 2.7649 | LR: 4.53e-04 | 4016.3ms/step | 14.9 samples/s
287
+ Step 10670/50462 | Loss: 2.7893 | LR: 4.52e-04 | 4015.3ms/step | 14.9 samples/s
288
+ Step 10680/50462 | Loss: 2.7893 | LR: 4.52e-04 | 4006.5ms/step | 15.0 samples/s
289
+ Step 10690/50462 | Loss: 2.7935 | LR: 4.52e-04 | 4012.1ms/step | 15.0 samples/s
290
+ Step 10700/50462 | Loss: 2.7568 | LR: 4.52e-04 | 4009.1ms/step | 15.0 samples/s
291
+ Step 10710/50462 | Loss: 2.7938 | LR: 4.52e-04 | 4006.9ms/step | 15.0 samples/s
292
+ Step 10720/50462 | Loss: 2.7717 | LR: 4.52e-04 | 4011.7ms/step | 15.0 samples/s
293
+ Step 10730/50462 | Loss: 2.7429 | LR: 4.52e-04 | 4008.9ms/step | 15.0 samples/s
294
+ Step 10740/50462 | Loss: 2.8088 | LR: 4.52e-04 | 4009.2ms/step | 15.0 samples/s
295
+ Step 10750/50462 | Loss: 2.7259 | LR: 4.52e-04 | 4014.8ms/step | 14.9 samples/s
296
+ Validation at step 10750: Loss = 2.7898, Perplexity = 16.28
297
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-10750
298
+ Step 10760/50462 | Loss: 2.7912 | LR: 4.52e-04 | 5872.7ms/step | 10.2 samples/s
299
+ Step 10770/50462 | Loss: 2.7486 | LR: 4.52e-04 | 4014.0ms/step | 14.9 samples/s
300
+ Step 10780/50462 | Loss: 2.7776 | LR: 4.51e-04 | 4009.5ms/step | 15.0 samples/s
301
+ Step 10790/50462 | Loss: 2.7759 | LR: 4.51e-04 | 4012.7ms/step | 15.0 samples/s
302
+ Step 10800/50462 | Loss: 2.7516 | LR: 4.51e-04 | 4012.6ms/step | 15.0 samples/s
303
+ Step 10810/50462 | Loss: 2.7922 | LR: 4.51e-04 | 4009.0ms/step | 15.0 samples/s
304
+ Step 10820/50462 | Loss: 2.7786 | LR: 4.51e-04 | 4005.9ms/step | 15.0 samples/s
305
+ Step 10830/50462 | Loss: 2.7668 | LR: 4.51e-04 | 4004.2ms/step | 15.0 samples/s
306
+ Step 10840/50462 | Loss: 2.7570 | LR: 4.51e-04 | 4001.8ms/step | 15.0 samples/s
307
+ Step 10850/50462 | Loss: 2.7805 | LR: 4.51e-04 | 4005.3ms/step | 15.0 samples/s
308
+ Step 10860/50462 | Loss: 2.7984 | LR: 4.51e-04 | 4007.3ms/step | 15.0 samples/s
309
+ Step 10870/50462 | Loss: 2.7621 | LR: 4.51e-04 | 4013.3ms/step | 15.0 samples/s
310
+ Step 10880/50462 | Loss: 2.7772 | LR: 4.50e-04 | 4003.8ms/step | 15.0 samples/s
311
+ Step 10890/50462 | Loss: 2.7853 | LR: 4.50e-04 | 4014.3ms/step | 14.9 samples/s
312
+ Step 10900/50462 | Loss: 2.7896 | LR: 4.50e-04 | 4015.2ms/step | 14.9 samples/s
313
+ Step 10910/50462 | Loss: 2.7881 | LR: 4.50e-04 | 4021.2ms/step | 14.9 samples/s
314
+ Step 10920/50462 | Loss: 2.7565 | LR: 4.50e-04 | 4003.5ms/step | 15.0 samples/s
315
+ Step 10930/50462 | Loss: 2.7736 | LR: 4.50e-04 | 4013.0ms/step | 15.0 samples/s
316
+ Step 10940/50462 | Loss: 2.7647 | LR: 4.50e-04 | 4014.8ms/step | 14.9 samples/s
317
+ Step 10950/50462 | Loss: 2.7889 | LR: 4.50e-04 | 4014.8ms/step | 14.9 samples/s
318
+ Step 10960/50462 | Loss: 2.7584 | LR: 4.50e-04 | 3999.7ms/step | 15.0 samples/s
319
+ Step 10970/50462 | Loss: 2.7345 | LR: 4.50e-04 | 4005.9ms/step | 15.0 samples/s
320
+ Step 10980/50462 | Loss: 2.7710 | LR: 4.50e-04 | 4012.8ms/step | 15.0 samples/s
321
+ Step 10990/50462 | Loss: 2.7515 | LR: 4.49e-04 | 4005.4ms/step | 15.0 samples/s
322
+ Step 11000/50462 | Loss: 2.7825 | LR: 4.49e-04 | 4008.6ms/step | 15.0 samples/s
323
+ Validation at step 11000: Loss = 2.7824, Perplexity = 16.16
324
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-11000
325
+ Step 11010/50462 | Loss: 2.7783 | LR: 4.49e-04 | 5915.7ms/step | 10.1 samples/s
326
+ Step 11020/50462 | Loss: 2.7471 | LR: 4.49e-04 | 4017.0ms/step | 14.9 samples/s
327
+ Step 11030/50462 | Loss: 2.7573 | LR: 4.49e-04 | 4017.0ms/step | 14.9 samples/s
328
+ Step 11040/50462 | Loss: 2.7911 | LR: 4.49e-04 | 4014.4ms/step | 14.9 samples/s
329
+ Step 11050/50462 | Loss: 2.7354 | LR: 4.49e-04 | 4003.5ms/step | 15.0 samples/s
330
+ Step 11060/50462 | Loss: 2.7843 | LR: 4.49e-04 | 4008.4ms/step | 15.0 samples/s
331
+ Step 11070/50462 | Loss: 2.7618 | LR: 4.49e-04 | 4006.2ms/step | 15.0 samples/s
332
+ Step 11080/50462 | Loss: 2.7698 | LR: 4.49e-04 | 4009.6ms/step | 15.0 samples/s
333
+ Step 11090/50462 | Loss: 2.7671 | LR: 4.48e-04 | 4008.7ms/step | 15.0 samples/s
334
+ Step 11100/50462 | Loss: 2.7409 | LR: 4.48e-04 | 4022.7ms/step | 14.9 samples/s
335
+ Step 11110/50462 | Loss: 2.7672 | LR: 4.48e-04 | 4009.6ms/step | 15.0 samples/s
336
+ Step 11120/50462 | Loss: 2.7942 | LR: 4.48e-04 | 4008.8ms/step | 15.0 samples/s
337
+ Step 11130/50462 | Loss: 2.7714 | LR: 4.48e-04 | 4017.4ms/step | 14.9 samples/s
338
+ Step 11140/50462 | Loss: 2.7960 | LR: 4.48e-04 | 4004.7ms/step | 15.0 samples/s
339
+ Step 11150/50462 | Loss: 2.7485 | LR: 4.48e-04 | 4013.4ms/step | 15.0 samples/s
340
+ Step 11160/50462 | Loss: 2.7854 | LR: 4.48e-04 | 4003.0ms/step | 15.0 samples/s
341
+ Step 11170/50462 | Loss: 2.7743 | LR: 4.48e-04 | 4000.9ms/step | 15.0 samples/s
342
+ Step 11180/50462 | Loss: 2.7590 | LR: 4.48e-04 | 4011.2ms/step | 15.0 samples/s
343
+ Step 11190/50462 | Loss: 2.8113 | LR: 4.48e-04 | 4004.9ms/step | 15.0 samples/s
344
+ Step 11200/50462 | Loss: 2.7258 | LR: 4.47e-04 | 4022.2ms/step | 14.9 samples/s
345
+ Step 11210/50462 | Loss: 2.7655 | LR: 4.47e-04 | 4009.7ms/step | 15.0 samples/s
346
+ Step 11220/50462 | Loss: 2.7546 | LR: 4.47e-04 | 4005.6ms/step | 15.0 samples/s
347
+ Step 11230/50462 | Loss: 2.7765 | LR: 4.47e-04 | 4020.3ms/step | 14.9 samples/s
348
+ Step 11240/50462 | Loss: 2.7836 | LR: 4.47e-04 | 4007.3ms/step | 15.0 samples/s
349
+ Step 11250/50462 | Loss: 2.7469 | LR: 4.47e-04 | 4009.8ms/step | 15.0 samples/s
350
+ Validation at step 11250: Loss = 2.7799, Perplexity = 16.12
351
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-11250
352
+ Step 11260/50462 | Loss: 2.7781 | LR: 4.47e-04 | 5936.5ms/step | 10.1 samples/s
353
+ Step 11270/50462 | Loss: 2.7304 | LR: 4.47e-04 | 4002.8ms/step | 15.0 samples/s
354
+ Step 11280/50462 | Loss: 2.7732 | LR: 4.47e-04 | 4005.5ms/step | 15.0 samples/s
355
+ Step 11290/50462 | Loss: 2.7801 | LR: 4.47e-04 | 4001.9ms/step | 15.0 samples/s
356
+ Step 11300/50462 | Loss: 2.7644 | LR: 4.46e-04 | 4007.1ms/step | 15.0 samples/s
357
+ Step 11310/50462 | Loss: 2.7284 | LR: 4.46e-04 | 4019.2ms/step | 14.9 samples/s
358
+ Step 11320/50462 | Loss: 2.7261 | LR: 4.46e-04 | 4016.7ms/step | 14.9 samples/s
359
+ Step 11330/50462 | Loss: 2.7186 | LR: 4.46e-04 | 4009.4ms/step | 15.0 samples/s
360
+ Step 11340/50462 | Loss: 2.7897 | LR: 4.46e-04 | 4008.4ms/step | 15.0 samples/s
361
+ Step 11350/50462 | Loss: 2.7893 | LR: 4.46e-04 | 4010.3ms/step | 15.0 samples/s
362
+ Step 11360/50462 | Loss: 2.7743 | LR: 4.46e-04 | 4017.9ms/step | 14.9 samples/s
363
+ Step 11370/50462 | Loss: 2.7765 | LR: 4.46e-04 | 4017.3ms/step | 14.9 samples/s
364
+ Step 11380/50462 | Loss: 2.7437 | LR: 4.46e-04 | 4011.3ms/step | 15.0 samples/s
365
+ Step 11390/50462 | Loss: 2.7268 | LR: 4.46e-04 | 4011.8ms/step | 15.0 samples/s
366
+ Step 11400/50462 | Loss: 2.7342 | LR: 4.45e-04 | 4001.2ms/step | 15.0 samples/s
367
+ Step 11410/50462 | Loss: 2.7648 | LR: 4.45e-04 | 4018.2ms/step | 14.9 samples/s
368
+ Step 11420/50462 | Loss: 2.7680 | LR: 4.45e-04 | 4004.9ms/step | 15.0 samples/s
369
+ Step 11430/50462 | Loss: 2.7492 | LR: 4.45e-04 | 4011.5ms/step | 15.0 samples/s
370
+ Step 11440/50462 | Loss: 2.7593 | LR: 4.45e-04 | 4006.3ms/step | 15.0 samples/s
371
+ Step 11450/50462 | Loss: 2.7530 | LR: 4.45e-04 | 4007.6ms/step | 15.0 samples/s
372
+ Step 11460/50462 | Loss: 2.7743 | LR: 4.45e-04 | 4009.2ms/step | 15.0 samples/s
373
+ Step 11470/50462 | Loss: 2.7438 | LR: 4.45e-04 | 4005.1ms/step | 15.0 samples/s
374
+ Step 11480/50462 | Loss: 2.7526 | LR: 4.45e-04 | 4001.5ms/step | 15.0 samples/s
375
+ Step 11490/50462 | Loss: 2.7598 | LR: 4.45e-04 | 4003.0ms/step | 15.0 samples/s
376
+ Step 11500/50462 | Loss: 2.7705 | LR: 4.44e-04 | 4011.6ms/step | 15.0 samples/s
377
+ Validation at step 11500: Loss = 2.7733, Perplexity = 16.01
378
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-11500
379
+ Step 11510/50462 | Loss: 2.7679 | LR: 4.44e-04 | 5882.1ms/step | 10.2 samples/s
380
+ Step 11520/50462 | Loss: 2.7397 | LR: 4.44e-04 | 4008.5ms/step | 15.0 samples/s
381
+ Step 11530/50462 | Loss: 2.7976 | LR: 4.44e-04 | 4012.6ms/step | 15.0 samples/s
382
+ Step 11540/50462 | Loss: 2.7662 | LR: 4.44e-04 | 4017.5ms/step | 14.9 samples/s
383
+ Step 11550/50462 | Loss: 2.7960 | LR: 4.44e-04 | 4026.0ms/step | 14.9 samples/s
384
+ Step 11560/50462 | Loss: 2.7346 | LR: 4.44e-04 | 4011.0ms/step | 15.0 samples/s
385
+ Step 11570/50462 | Loss: 2.7898 | LR: 4.44e-04 | 4020.8ms/step | 14.9 samples/s
386
+ Step 11580/50462 | Loss: 2.7288 | LR: 4.44e-04 | 4021.6ms/step | 14.9 samples/s
387
+ Step 11590/50462 | Loss: 2.7489 | LR: 4.44e-04 | 4009.9ms/step | 15.0 samples/s
388
+ Step 11600/50462 | Loss: 2.7626 | LR: 4.43e-04 | 4023.6ms/step | 14.9 samples/s
389
+ Step 11610/50462 | Loss: 2.7244 | LR: 4.43e-04 | 4008.5ms/step | 15.0 samples/s
390
+ Step 11620/50462 | Loss: 2.7654 | LR: 4.43e-04 | 4014.3ms/step | 14.9 samples/s
391
+ Step 11630/50462 | Loss: 2.7503 | LR: 4.43e-04 | 4012.2ms/step | 15.0 samples/s
392
+ Step 11640/50462 | Loss: 2.7536 | LR: 4.43e-04 | 4008.1ms/step | 15.0 samples/s
393
+ Step 11650/50462 | Loss: 2.7874 | LR: 4.43e-04 | 4009.7ms/step | 15.0 samples/s
394
+ Step 11660/50462 | Loss: 2.7643 | LR: 4.43e-04 | 4029.2ms/step | 14.9 samples/s
395
+ Step 11670/50462 | Loss: 2.7569 | LR: 4.43e-04 | 4006.3ms/step | 15.0 samples/s
396
+ Step 11680/50462 | Loss: 2.7619 | LR: 4.43e-04 | 4009.1ms/step | 15.0 samples/s
397
+ Step 11690/50462 | Loss: 2.7696 | LR: 4.43e-04 | 4024.1ms/step | 14.9 samples/s
398
+ Step 11700/50462 | Loss: 2.7473 | LR: 4.42e-04 | 4008.6ms/step | 15.0 samples/s
399
+ Step 11710/50462 | Loss: 2.7309 | LR: 4.42e-04 | 4015.9ms/step | 14.9 samples/s
400
+ Step 11720/50462 | Loss: 2.7448 | LR: 4.42e-04 | 4017.9ms/step | 14.9 samples/s
401
+ Step 11730/50462 | Loss: 2.7703 | LR: 4.42e-04 | 4018.0ms/step | 14.9 samples/s
402
+ Step 11740/50462 | Loss: 2.7744 | LR: 4.42e-04 | 4025.2ms/step | 14.9 samples/s
403
+ Step 11750/50462 | Loss: 2.7255 | LR: 4.42e-04 | 4016.0ms/step | 14.9 samples/s
404
+ Validation at step 11750: Loss = 2.7701, Perplexity = 15.96
405
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-11750
406
+ Step 11760/50462 | Loss: 2.7440 | LR: 4.42e-04 | 5875.5ms/step | 10.2 samples/s
407
+ Step 11770/50462 | Loss: 2.7667 | LR: 4.42e-04 | 4002.7ms/step | 15.0 samples/s
408
+ Step 11780/50462 | Loss: 2.7306 | LR: 4.42e-04 | 4006.8ms/step | 15.0 samples/s
409
+ Step 11790/50462 | Loss: 2.7286 | LR: 4.42e-04 | 4010.0ms/step | 15.0 samples/s
410
+ Step 11800/50462 | Loss: 2.7841 | LR: 4.41e-04 | 4003.7ms/step | 15.0 samples/s
411
+ Step 11810/50462 | Loss: 2.7286 | LR: 4.41e-04 | 4017.4ms/step | 14.9 samples/s
412
+ Step 11820/50462 | Loss: 2.7874 | LR: 4.41e-04 | 4009.7ms/step | 15.0 samples/s
413
+ Step 11830/50462 | Loss: 2.7410 | LR: 4.41e-04 | 4012.0ms/step | 15.0 samples/s
414
+ Step 11840/50462 | Loss: 2.7584 | LR: 4.41e-04 | 4011.7ms/step | 15.0 samples/s
415
+ Step 11850/50462 | Loss: 2.7384 | LR: 4.41e-04 | 4011.2ms/step | 15.0 samples/s
416
+ Step 11860/50462 | Loss: 2.7941 | LR: 4.41e-04 | 4021.8ms/step | 14.9 samples/s
417
+ Step 11870/50462 | Loss: 2.7258 | LR: 4.41e-04 | 4046.0ms/step | 14.8 samples/s
418
+ Step 11880/50462 | Loss: 2.7724 | LR: 4.41e-04 | 4039.2ms/step | 14.9 samples/s
419
+ Step 11890/50462 | Loss: 2.7354 | LR: 4.41e-04 | 4046.5ms/step | 14.8 samples/s
420
+ Step 11900/50462 | Loss: 2.7815 | LR: 4.40e-04 | 4027.8ms/step | 14.9 samples/s
421
+ Step 11910/50462 | Loss: 2.7686 | LR: 4.40e-04 | 4036.6ms/step | 14.9 samples/s
422
+ Step 11920/50462 | Loss: 2.7426 | LR: 4.40e-04 | 4013.7ms/step | 14.9 samples/s
423
+ Step 11930/50462 | Loss: 2.7456 | LR: 4.40e-04 | 4025.7ms/step | 14.9 samples/s
424
+ Step 11940/50462 | Loss: 2.7818 | LR: 4.40e-04 | 4016.7ms/step | 14.9 samples/s
425
+ Step 11950/50462 | Loss: 2.7521 | LR: 4.40e-04 | 4022.3ms/step | 14.9 samples/s
426
+ Step 11960/50462 | Loss: 2.7350 | LR: 4.40e-04 | 4022.2ms/step | 14.9 samples/s
427
+ Step 11970/50462 | Loss: 2.7267 | LR: 4.40e-04 | 4025.4ms/step | 14.9 samples/s
428
+ Step 11980/50462 | Loss: 2.7184 | LR: 4.40e-04 | 4018.3ms/step | 14.9 samples/s
429
+ Step 11990/50462 | Loss: 2.7589 | LR: 4.40e-04 | 4011.1ms/step | 15.0 samples/s
430
+ Step 12000/50462 | Loss: 2.7454 | LR: 4.39e-04 | 4028.9ms/step | 14.9 samples/s
431
+ Validation at step 12000: Loss = 2.7675, Perplexity = 15.92
432
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-12000
433
+ Step 12010/50462 | Loss: 2.7620 | LR: 4.39e-04 | 6115.2ms/step | 9.8 samples/s
434
+ Step 12020/50462 | Loss: 2.7640 | LR: 4.39e-04 | 4025.0ms/step | 14.9 samples/s
435
+ Step 12030/50462 | Loss: 2.7462 | LR: 4.39e-04 | 4022.9ms/step | 14.9 samples/s
436
+ Step 12040/50462 | Loss: 2.7621 | LR: 4.39e-04 | 4025.6ms/step | 14.9 samples/s
437
+ Step 12050/50462 | Loss: 2.7131 | LR: 4.39e-04 | 4021.7ms/step | 14.9 samples/s
438
+ Step 12060/50462 | Loss: 2.7423 | LR: 4.39e-04 | 4003.1ms/step | 15.0 samples/s
439
+ Step 12070/50462 | Loss: 2.7533 | LR: 4.39e-04 | 4021.1ms/step | 14.9 samples/s
440
+ Step 12080/50462 | Loss: 2.7407 | LR: 4.39e-04 | 4027.2ms/step | 14.9 samples/s
441
+ Step 12090/50462 | Loss: 2.7486 | LR: 4.39e-04 | 4021.3ms/step | 14.9 samples/s
442
+ Step 12100/50462 | Loss: 2.7575 | LR: 4.38e-04 | 4024.1ms/step | 14.9 samples/s
443
+ Step 12110/50462 | Loss: 2.7310 | LR: 4.38e-04 | 4019.5ms/step | 14.9 samples/s
444
+ Step 12120/50462 | Loss: 2.7506 | LR: 4.38e-04 | 4019.5ms/step | 14.9 samples/s
445
+ Step 12130/50462 | Loss: 2.7616 | LR: 4.38e-04 | 4018.2ms/step | 14.9 samples/s
446
+ Step 12140/50462 | Loss: 2.7201 | LR: 4.38e-04 | 4019.9ms/step | 14.9 samples/s
447
+ Step 12150/50462 | Loss: 2.7378 | LR: 4.38e-04 | 4012.1ms/step | 15.0 samples/s
448
+ Step 12160/50462 | Loss: 2.7408 | LR: 4.38e-04 | 4019.2ms/step | 14.9 samples/s
449
+ Step 12170/50462 | Loss: 2.7371 | LR: 4.38e-04 | 4013.5ms/step | 14.9 samples/s
450
+ Step 12180/50462 | Loss: 2.7587 | LR: 4.38e-04 | 4012.9ms/step | 15.0 samples/s
451
+ Step 12190/50462 | Loss: 2.7977 | LR: 4.37e-04 | 4016.0ms/step | 14.9 samples/s
452
+ Step 12200/50462 | Loss: 2.7372 | LR: 4.37e-04 | 4019.4ms/step | 14.9 samples/s
453
+ Step 12210/50462 | Loss: 2.7489 | LR: 4.37e-04 | 4020.0ms/step | 14.9 samples/s
454
+ Step 12220/50462 | Loss: 2.7113 | LR: 4.37e-04 | 4016.4ms/step | 14.9 samples/s
455
+ Step 12230/50462 | Loss: 2.7808 | LR: 4.37e-04 | 4009.6ms/step | 15.0 samples/s
456
+ Step 12240/50462 | Loss: 2.7325 | LR: 4.37e-04 | 4009.6ms/step | 15.0 samples/s
457
+ Step 12250/50462 | Loss: 2.7323 | LR: 4.37e-04 | 4012.3ms/step | 15.0 samples/s
458
+ Validation at step 12250: Loss = 2.7616, Perplexity = 15.83
459
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-12250
460
+ Step 12260/50462 | Loss: 2.7367 | LR: 4.37e-04 | 5988.3ms/step | 10.0 samples/s
461
+ Step 12270/50462 | Loss: 2.7291 | LR: 4.37e-04 | 4025.9ms/step | 14.9 samples/s
462
+ Step 12280/50462 | Loss: 2.7079 | LR: 4.37e-04 | 4008.2ms/step | 15.0 samples/s
463
+ Step 12290/50462 | Loss: 2.7046 | LR: 4.36e-04 | 4017.8ms/step | 14.9 samples/s
464
+ Step 12300/50462 | Loss: 2.7455 | LR: 4.36e-04 | 4019.8ms/step | 14.9 samples/s
465
+ Step 12310/50462 | Loss: 2.7560 | LR: 4.36e-04 | 4021.6ms/step | 14.9 samples/s
466
+ Step 12320/50462 | Loss: 2.7670 | LR: 4.36e-04 | 4024.2ms/step | 14.9 samples/s
467
+ Step 12330/50462 | Loss: 2.7517 | LR: 4.36e-04 | 4010.5ms/step | 15.0 samples/s
468
+ Step 12340/50462 | Loss: 2.7417 | LR: 4.36e-04 | 4010.9ms/step | 15.0 samples/s
469
+ Step 12350/50462 | Loss: 2.7360 | LR: 4.36e-04 | 4005.8ms/step | 15.0 samples/s
470
+ Step 12360/50462 | Loss: 2.7193 | LR: 4.36e-04 | 4028.1ms/step | 14.9 samples/s
471
+ Step 12370/50462 | Loss: 2.7488 | LR: 4.36e-04 | 4019.4ms/step | 14.9 samples/s
472
+ Step 12380/50462 | Loss: 2.7572 | LR: 4.35e-04 | 4028.7ms/step | 14.9 samples/s
473
+ Step 12390/50462 | Loss: 2.7249 | LR: 4.35e-04 | 4001.8ms/step | 15.0 samples/s
474
+ Step 12400/50462 | Loss: 2.7383 | LR: 4.35e-04 | 4008.5ms/step | 15.0 samples/s
475
+ Step 12410/50462 | Loss: 2.7476 | LR: 4.35e-04 | 4020.7ms/step | 14.9 samples/s
476
+ Step 12420/50462 | Loss: 2.7198 | LR: 4.35e-04 | 4016.1ms/step | 14.9 samples/s
477
+ Step 12430/50462 | Loss: 2.7702 | LR: 4.35e-04 | 4019.3ms/step | 14.9 samples/s
478
+ Step 12440/50462 | Loss: 2.7654 | LR: 4.35e-04 | 4017.4ms/step | 14.9 samples/s
479
+ Step 12450/50462 | Loss: 2.7430 | LR: 4.35e-04 | 4016.9ms/step | 14.9 samples/s
480
+ Step 12460/50462 | Loss: 2.7761 | LR: 4.35e-04 | 4009.3ms/step | 15.0 samples/s
481
+ Step 12470/50462 | Loss: 2.7799 | LR: 4.35e-04 | 4006.7ms/step | 15.0 samples/s
482
+ Step 12480/50462 | Loss: 2.7625 | LR: 4.34e-04 | 4010.9ms/step | 15.0 samples/s
483
+ Step 12490/50462 | Loss: 2.7638 | LR: 4.34e-04 | 4016.7ms/step | 14.9 samples/s
484
+ Step 12500/50462 | Loss: 2.7333 | LR: 4.34e-04 | 4008.0ms/step | 15.0 samples/s
485
+ Validation at step 12500: Loss = 2.7564, Perplexity = 15.74
486
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-12500
487
+ Step 12510/50462 | Loss: 2.7882 | LR: 4.34e-04 | 5945.0ms/step | 10.1 samples/s
488
+ Step 12520/50462 | Loss: 2.7345 | LR: 4.34e-04 | 4008.0ms/step | 15.0 samples/s
489
+ Step 12530/50462 | Loss: 2.7241 | LR: 4.34e-04 | 4004.9ms/step | 15.0 samples/s
490
+ Step 12540/50462 | Loss: 2.7476 | LR: 4.34e-04 | 4017.5ms/step | 14.9 samples/s
491
+ Step 12550/50462 | Loss: 2.7224 | LR: 4.34e-04 | 4017.3ms/step | 14.9 samples/s
492
+ Step 12560/50462 | Loss: 2.7463 | LR: 4.34e-04 | 4023.3ms/step | 14.9 samples/s
493
+ Step 12570/50462 | Loss: 2.7292 | LR: 4.33e-04 | 4005.4ms/step | 15.0 samples/s
494
+ Step 12580/50462 | Loss: 2.7384 | LR: 4.33e-04 | 4008.9ms/step | 15.0 samples/s
495
+ Step 12590/50462 | Loss: 2.7390 | LR: 4.33e-04 | 4007.1ms/step | 15.0 samples/s
496
+ Step 12600/50462 | Loss: 2.7657 | LR: 4.33e-04 | 4000.8ms/step | 15.0 samples/s
497
+ Step 12610/50462 | Loss: 2.7621 | LR: 4.33e-04 | 4007.8ms/step | 15.0 samples/s
498
+ Step 12620/50462 | Loss: 2.7417 | LR: 4.33e-04 | 4013.8ms/step | 14.9 samples/s
499
+ Step 12630/50462 | Loss: 2.7345 | LR: 4.33e-04 | 4004.5ms/step | 15.0 samples/s
500
+ Step 12640/50462 | Loss: 2.7272 | LR: 4.33e-04 | 4014.0ms/step | 14.9 samples/s
501
+ Step 12650/50462 | Loss: 2.7412 | LR: 4.33e-04 | 4009.2ms/step | 15.0 samples/s
502
+ Step 12660/50462 | Loss: 2.7278 | LR: 4.32e-04 | 4001.3ms/step | 15.0 samples/s
503
+ Step 12670/50462 | Loss: 2.7532 | LR: 4.32e-04 | 3996.7ms/step | 15.0 samples/s
504
+ Step 12680/50462 | Loss: 2.7499 | LR: 4.32e-04 | 4004.9ms/step | 15.0 samples/s
505
+ Step 12690/50462 | Loss: 2.7222 | LR: 4.32e-04 | 4005.5ms/step | 15.0 samples/s
506
+ Step 12700/50462 | Loss: 2.7320 | LR: 4.32e-04 | 3997.9ms/step | 15.0 samples/s
507
+ Step 12710/50462 | Loss: 2.7021 | LR: 4.32e-04 | 4014.1ms/step | 14.9 samples/s
508
+ Step 12720/50462 | Loss: 2.7415 | LR: 4.32e-04 | 4011.7ms/step | 15.0 samples/s
509
+ Step 12730/50462 | Loss: 2.7394 | LR: 4.32e-04 | 4013.9ms/step | 14.9 samples/s
510
+ Step 12740/50462 | Loss: 2.7133 | LR: 4.32e-04 | 4013.8ms/step | 14.9 samples/s
511
+ Step 12750/50462 | Loss: 2.7106 | LR: 4.31e-04 | 4009.0ms/step | 15.0 samples/s
512
+ Validation at step 12750: Loss = 2.7530, Perplexity = 15.69
513
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-12750
514
+ Step 12760/50462 | Loss: 2.7268 | LR: 4.31e-04 | 5897.8ms/step | 10.2 samples/s
515
+ Step 12770/50462 | Loss: 2.7491 | LR: 4.31e-04 | 4000.2ms/step | 15.0 samples/s
516
+ Step 12780/50462 | Loss: 2.7323 | LR: 4.31e-04 | 4006.7ms/step | 15.0 samples/s
517
+ Step 12790/50462 | Loss: 2.7085 | LR: 4.31e-04 | 4009.7ms/step | 15.0 samples/s
518
+ Step 12800/50462 | Loss: 2.7448 | LR: 4.31e-04 | 4005.1ms/step | 15.0 samples/s
519
+ Step 12810/50462 | Loss: 2.7395 | LR: 4.31e-04 | 4010.7ms/step | 15.0 samples/s
520
+ Step 12820/50462 | Loss: 2.7227 | LR: 4.31e-04 | 4011.5ms/step | 15.0 samples/s
521
+ Step 12830/50462 | Loss: 2.7101 | LR: 4.31e-04 | 4009.5ms/step | 15.0 samples/s
522
+ Step 12840/50462 | Loss: 2.7227 | LR: 4.31e-04 | 4010.5ms/step | 15.0 samples/s
523
+ Step 12850/50462 | Loss: 2.7403 | LR: 4.30e-04 | 4019.2ms/step | 14.9 samples/s
524
+ Step 12860/50462 | Loss: 2.7390 | LR: 4.30e-04 | 4008.9ms/step | 15.0 samples/s
525
+ Step 12870/50462 | Loss: 2.7295 | LR: 4.30e-04 | 4016.7ms/step | 14.9 samples/s
526
+ Step 12880/50462 | Loss: 2.7335 | LR: 4.30e-04 | 4013.1ms/step | 15.0 samples/s
527
+ Step 12890/50462 | Loss: 2.7365 | LR: 4.30e-04 | 4016.3ms/step | 14.9 samples/s
528
+ Step 12900/50462 | Loss: 2.7064 | LR: 4.30e-04 | 4003.5ms/step | 15.0 samples/s
529
+ Step 12910/50462 | Loss: 2.7282 | LR: 4.30e-04 | 4007.5ms/step | 15.0 samples/s
530
+ Step 12920/50462 | Loss: 2.7631 | LR: 4.30e-04 | 4014.1ms/step | 14.9 samples/s
epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/files/wandb-metadata.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-27T16:58:38.882060Z",
5
+ "args": [
6
+ "--config",
7
+ "configs/p2_1p2b_mixed_seed72_4gpu_autoresume_current.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/SparseAtt.git",
14
+ "commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-train-4-gpu-0-22",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 4,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "6455624826880"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-a2122da0-1020-9f78-7e6f-a1d5011f694b"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-4c1d91e9-bf01-f8fd-1e8b-79aa8e0831df"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-086f9f0b-a6d8-6263-83e6-c2b66721f47f"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-bdb5e23e-0700-20a0-0b21-1ad0901f08b7"
61
+ }
62
+ ],
63
+ "cudaVersion": "13.0",
64
+ "writerId": "jfquojzg36v88qdn5rmtnn2me97o8kgu"
65
+ }
epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-27 16:58:38,905 INFO MainThread:219 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-27 16:58:38,905 INFO MainThread:219 [wandb_setup.py:_flush():81] Configure stats pid to 219
3
+ 2026-08-27 16:58:38,905 INFO MainThread:219 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-27 16:58:38,905 INFO MainThread:219 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_165838-z1i60278/logs/debug.log
5
+ 2026-08-27 16:58:38,905 INFO MainThread:219 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260827_165838-z1i60278/logs/debug-internal.log
6
+ 2026-08-27 16:58:38,905 INFO MainThread:219 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-27 16:58:38,905 INFO MainThread:219 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-8250', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 72, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed72 6gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed72'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 72, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-27 16:58:38,905 INFO MainThread:219 [wandb_init.py:init():898] starting backend
10
+ 2026-08-27 16:58:39,196 INFO MainThread:219 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-27 16:58:39,526 INFO MainThread:219 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-27 16:58:39,529 INFO MainThread:219 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-27 16:58:39,552 INFO MainThread:219 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-27 16:58:40,075 INFO MainThread:219 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-27 16:58:40,077 INFO MainThread:219 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-27 16:58:40,406 INFO MainThread:219 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-27 16:58:40,406 INFO MainThread:219 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-27 16:58:40,406 INFO MainThread:219 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-27 16:58:40,406 INFO MainThread:219 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-27 16:58:40,421 INFO MainThread:219 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260827_165838-z1i60278/run-z1i60278.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e25b89e8895eee96d50c18e071f552661e4c4d0cb741fd40c232bd55245ed60c
3
+ size 6422528
epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/files/output.log ADDED
@@ -0,0 +1,1170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 12750: Loss = 2.7528, Perplexity = 15.69
3
+ wandb: WARNING Tried to log to step 12750 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 12751: sparse_stochastic (configured sparse attention)
5
+ Step 12760/50462 | Loss: 2.7272 | LR: 4.31e-04 | 37445.3ms/step | 1.6 samples/s
6
+ wandb: WARNING Tried to log to step 12760 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 12770/50462 | Loss: 2.7485 | LR: 4.31e-04 | 3992.3ms/step | 15.0 samples/s
8
+ wandb: WARNING Tried to log to step 12770 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 12780/50462 | Loss: 2.7336 | LR: 4.31e-04 | 4005.8ms/step | 15.0 samples/s
10
+ wandb: WARNING Tried to log to step 12780 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 12790/50462 | Loss: 2.7068 | LR: 4.31e-04 | 4014.1ms/step | 14.9 samples/s
12
+ wandb: WARNING Tried to log to step 12790 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 12800/50462 | Loss: 2.7451 | LR: 4.31e-04 | 4014.0ms/step | 14.9 samples/s
14
+ wandb: WARNING Tried to log to step 12800 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 12810/50462 | Loss: 2.7403 | LR: 4.31e-04 | 4011.7ms/step | 15.0 samples/s
16
+ wandb: WARNING Tried to log to step 12810 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 12820/50462 | Loss: 2.7213 | LR: 4.31e-04 | 4015.7ms/step | 14.9 samples/s
18
+ wandb: WARNING Tried to log to step 12820 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
19
+ Step 12830/50462 | Loss: 2.7120 | LR: 4.31e-04 | 4021.7ms/step | 14.9 samples/s
20
+ wandb: WARNING Tried to log to step 12830 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
21
+ Step 12840/50462 | Loss: 2.7233 | LR: 4.31e-04 | 4017.3ms/step | 14.9 samples/s
22
+ wandb: WARNING Tried to log to step 12840 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
23
+ Step 12850/50462 | Loss: 2.7387 | LR: 4.30e-04 | 4023.2ms/step | 14.9 samples/s
24
+ wandb: WARNING Tried to log to step 12850 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
25
+ Step 12860/50462 | Loss: 2.7376 | LR: 4.30e-04 | 4030.8ms/step | 14.9 samples/s
26
+ wandb: WARNING Tried to log to step 12860 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
27
+ Step 12870/50462 | Loss: 2.7300 | LR: 4.30e-04 | 4040.6ms/step | 14.8 samples/s
28
+ wandb: WARNING Tried to log to step 12870 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
29
+ Step 12880/50462 | Loss: 2.7331 | LR: 4.30e-04 | 4031.5ms/step | 14.9 samples/s
30
+ wandb: WARNING Tried to log to step 12880 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
31
+ Step 12890/50462 | Loss: 2.7371 | LR: 4.30e-04 | 4026.0ms/step | 14.9 samples/s
32
+ wandb: WARNING Tried to log to step 12890 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
33
+ Step 12900/50462 | Loss: 2.7054 | LR: 4.30e-04 | 4027.4ms/step | 14.9 samples/s
34
+ wandb: WARNING Tried to log to step 12900 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
35
+ Step 12910/50462 | Loss: 2.7295 | LR: 4.30e-04 | 4018.2ms/step | 14.9 samples/s
36
+ wandb: WARNING Tried to log to step 12910 that is less than the current step 12911. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
37
+ Step 12920/50462 | Loss: 2.7623 | LR: 4.30e-04 | 4015.4ms/step | 14.9 samples/s
38
+ Step 12930/50462 | Loss: 2.7110 | LR: 4.30e-04 | 4032.1ms/step | 14.9 samples/s
39
+ Step 12940/50462 | Loss: 2.7219 | LR: 4.29e-04 | 4014.5ms/step | 14.9 samples/s
40
+ Step 12950/50462 | Loss: 2.7198 | LR: 4.29e-04 | 4022.3ms/step | 14.9 samples/s
41
+ Step 12960/50462 | Loss: 2.7215 | LR: 4.29e-04 | 4029.8ms/step | 14.9 samples/s
42
+ Step 12970/50462 | Loss: 2.7443 | LR: 4.29e-04 | 4031.1ms/step | 14.9 samples/s
43
+ Step 12980/50462 | Loss: 2.7060 | LR: 4.29e-04 | 4027.8ms/step | 14.9 samples/s
44
+ Step 12990/50462 | Loss: 2.7517 | LR: 4.29e-04 | 4008.9ms/step | 15.0 samples/s
45
+ Step 13000/50462 | Loss: 2.7729 | LR: 4.29e-04 | 4026.3ms/step | 14.9 samples/s
46
+ Validation at step 13000: Loss = 2.7482, Perplexity = 15.61
47
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-13000
48
+ Step 13010/50462 | Loss: 2.7626 | LR: 4.29e-04 | 5990.5ms/step | 10.0 samples/s
49
+ Step 13020/50462 | Loss: 2.7621 | LR: 4.29e-04 | 4032.4ms/step | 14.9 samples/s
50
+ Step 13030/50462 | Loss: 2.7093 | LR: 4.28e-04 | 4022.0ms/step | 14.9 samples/s
51
+ Step 13040/50462 | Loss: 2.7207 | LR: 4.28e-04 | 4020.9ms/step | 14.9 samples/s
52
+ Step 13050/50462 | Loss: 2.7129 | LR: 4.28e-04 | 4032.5ms/step | 14.9 samples/s
53
+ Step 13060/50462 | Loss: 2.7306 | LR: 4.28e-04 | 4030.0ms/step | 14.9 samples/s
54
+ Step 13070/50462 | Loss: 2.7359 | LR: 4.28e-04 | 4036.5ms/step | 14.9 samples/s
55
+ Step 13080/50462 | Loss: 2.7467 | LR: 4.28e-04 | 4021.2ms/step | 14.9 samples/s
56
+ Step 13090/50462 | Loss: 2.7329 | LR: 4.28e-04 | 4026.0ms/step | 14.9 samples/s
57
+ Step 13100/50462 | Loss: 2.7498 | LR: 4.28e-04 | 4026.6ms/step | 14.9 samples/s
58
+ Step 13110/50462 | Loss: 2.7517 | LR: 4.28e-04 | 4024.5ms/step | 14.9 samples/s
59
+ Step 13120/50462 | Loss: 2.7235 | LR: 4.27e-04 | 4034.2ms/step | 14.9 samples/s
60
+ Step 13130/50462 | Loss: 2.7172 | LR: 4.27e-04 | 4031.9ms/step | 14.9 samples/s
61
+ Step 13140/50462 | Loss: 2.7450 | LR: 4.27e-04 | 4018.9ms/step | 14.9 samples/s
62
+ Step 13150/50462 | Loss: 2.7064 | LR: 4.27e-04 | 4021.8ms/step | 14.9 samples/s
63
+ Step 13160/50462 | Loss: 2.7428 | LR: 4.27e-04 | 4036.7ms/step | 14.9 samples/s
64
+ Step 13170/50462 | Loss: 2.7194 | LR: 4.27e-04 | 4026.0ms/step | 14.9 samples/s
65
+ Step 13180/50462 | Loss: 2.7133 | LR: 4.27e-04 | 4014.0ms/step | 14.9 samples/s
66
+ Step 13190/50462 | Loss: 2.7582 | LR: 4.27e-04 | 4028.0ms/step | 14.9 samples/s
67
+ Step 13200/50462 | Loss: 2.7196 | LR: 4.27e-04 | 4032.6ms/step | 14.9 samples/s
68
+ Step 13210/50462 | Loss: 2.7525 | LR: 4.26e-04 | 4023.3ms/step | 14.9 samples/s
69
+ Step 13220/50462 | Loss: 2.7583 | LR: 4.26e-04 | 4041.2ms/step | 14.8 samples/s
70
+ Step 13230/50462 | Loss: 2.7293 | LR: 4.26e-04 | 4023.1ms/step | 14.9 samples/s
71
+ Step 13240/50462 | Loss: 2.7653 | LR: 4.26e-04 | 4030.3ms/step | 14.9 samples/s
72
+ Step 13250/50462 | Loss: 2.7523 | LR: 4.26e-04 | 4023.5ms/step | 14.9 samples/s
73
+ Validation at step 13250: Loss = 2.7446, Perplexity = 15.56
74
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-13250
75
+ Step 13260/50462 | Loss: 2.7150 | LR: 4.26e-04 | 6018.1ms/step | 10.0 samples/s
76
+ Step 13270/50462 | Loss: 2.7166 | LR: 4.26e-04 | 4027.2ms/step | 14.9 samples/s
77
+ Step 13280/50462 | Loss: 2.7396 | LR: 4.26e-04 | 4037.5ms/step | 14.9 samples/s
78
+ Step 13290/50462 | Loss: 2.7304 | LR: 4.26e-04 | 4026.7ms/step | 14.9 samples/s
79
+ Step 13300/50462 | Loss: 2.7397 | LR: 4.25e-04 | 4033.7ms/step | 14.9 samples/s
80
+ Step 13310/50462 | Loss: 2.7085 | LR: 4.25e-04 | 4040.6ms/step | 14.8 samples/s
81
+ Step 13320/50462 | Loss: 2.7246 | LR: 4.25e-04 | 4022.5ms/step | 14.9 samples/s
82
+ Step 13330/50462 | Loss: 2.7206 | LR: 4.25e-04 | 4033.2ms/step | 14.9 samples/s
83
+ Step 13340/50462 | Loss: 2.7331 | LR: 4.25e-04 | 4036.6ms/step | 14.9 samples/s
84
+ Step 13350/50462 | Loss: 2.7568 | LR: 4.25e-04 | 4035.1ms/step | 14.9 samples/s
85
+ Step 13360/50462 | Loss: 2.7484 | LR: 4.25e-04 | 4020.3ms/step | 14.9 samples/s
86
+ Step 13370/50462 | Loss: 2.6744 | LR: 4.25e-04 | 4032.1ms/step | 14.9 samples/s
87
+ Step 13380/50462 | Loss: 2.7364 | LR: 4.25e-04 | 4021.9ms/step | 14.9 samples/s
88
+ Step 13390/50462 | Loss: 2.7041 | LR: 4.24e-04 | 4026.0ms/step | 14.9 samples/s
89
+ Step 13400/50462 | Loss: 2.6890 | LR: 4.24e-04 | 4030.8ms/step | 14.9 samples/s
90
+ Step 13410/50462 | Loss: 2.7052 | LR: 4.24e-04 | 4014.6ms/step | 14.9 samples/s
91
+ Step 13420/50462 | Loss: 2.7168 | LR: 4.24e-04 | 4025.7ms/step | 14.9 samples/s
92
+ Step 13430/50462 | Loss: 2.6992 | LR: 4.24e-04 | 4021.9ms/step | 14.9 samples/s
93
+ Step 13440/50462 | Loss: 2.7224 | LR: 4.24e-04 | 4030.7ms/step | 14.9 samples/s
94
+ Step 13450/50462 | Loss: 2.6793 | LR: 4.24e-04 | 4021.3ms/step | 14.9 samples/s
95
+ Step 13460/50462 | Loss: 2.7297 | LR: 4.24e-04 | 4022.6ms/step | 14.9 samples/s
96
+ Step 13470/50462 | Loss: 2.7132 | LR: 4.23e-04 | 4021.1ms/step | 14.9 samples/s
97
+ Step 13480/50462 | Loss: 2.7301 | LR: 4.23e-04 | 4028.9ms/step | 14.9 samples/s
98
+ Step 13490/50462 | Loss: 2.7160 | LR: 4.23e-04 | 4017.1ms/step | 14.9 samples/s
99
+ Step 13500/50462 | Loss: 2.7228 | LR: 4.23e-04 | 4035.3ms/step | 14.9 samples/s
100
+ Validation at step 13500: Loss = 2.7405, Perplexity = 15.50
101
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-13500
102
+ Step 13510/50462 | Loss: 2.7112 | LR: 4.23e-04 | 5983.8ms/step | 10.0 samples/s
103
+ Step 13520/50462 | Loss: 2.7143 | LR: 4.23e-04 | 4011.9ms/step | 15.0 samples/s
104
+ Step 13530/50462 | Loss: 2.7548 | LR: 4.23e-04 | 4023.5ms/step | 14.9 samples/s
105
+ Step 13540/50462 | Loss: 2.7319 | LR: 4.23e-04 | 4029.0ms/step | 14.9 samples/s
106
+ Step 13550/50462 | Loss: 2.7199 | LR: 4.23e-04 | 4019.7ms/step | 14.9 samples/s
107
+ Step 13560/50462 | Loss: 2.7225 | LR: 4.22e-04 | 4026.2ms/step | 14.9 samples/s
108
+ Step 13570/50462 | Loss: 2.7104 | LR: 4.22e-04 | 4025.7ms/step | 14.9 samples/s
109
+ Step 13580/50462 | Loss: 2.7036 | LR: 4.22e-04 | 4017.9ms/step | 14.9 samples/s
110
+ Step 13590/50462 | Loss: 2.7133 | LR: 4.22e-04 | 4012.2ms/step | 15.0 samples/s
111
+ Step 13600/50462 | Loss: 2.7144 | LR: 4.22e-04 | 4014.7ms/step | 14.9 samples/s
112
+ Step 13610/50462 | Loss: 2.7112 | LR: 4.22e-04 | 4032.1ms/step | 14.9 samples/s
113
+ Step 13620/50462 | Loss: 2.7299 | LR: 4.22e-04 | 4010.0ms/step | 15.0 samples/s
114
+ Step 13630/50462 | Loss: 2.7320 | LR: 4.22e-04 | 4023.7ms/step | 14.9 samples/s
115
+ Step 13640/50462 | Loss: 2.7175 | LR: 4.22e-04 | 4021.9ms/step | 14.9 samples/s
116
+ Step 13650/50462 | Loss: 2.7268 | LR: 4.21e-04 | 4021.4ms/step | 14.9 samples/s
117
+ Step 13660/50462 | Loss: 2.7187 | LR: 4.21e-04 | 4018.8ms/step | 14.9 samples/s
118
+ Step 13670/50462 | Loss: 2.7244 | LR: 4.21e-04 | 4021.9ms/step | 14.9 samples/s
119
+ Step 13680/50462 | Loss: 2.7251 | LR: 4.21e-04 | 4023.6ms/step | 14.9 samples/s
120
+ Step 13690/50462 | Loss: 2.7180 | LR: 4.21e-04 | 4011.3ms/step | 15.0 samples/s
121
+ Step 13700/50462 | Loss: 2.7018 | LR: 4.21e-04 | 4018.2ms/step | 14.9 samples/s
122
+ Step 13710/50462 | Loss: 2.6982 | LR: 4.21e-04 | 4009.5ms/step | 15.0 samples/s
123
+ Step 13720/50462 | Loss: 2.6923 | LR: 4.21e-04 | 4014.4ms/step | 14.9 samples/s
124
+ Step 13730/50462 | Loss: 2.6893 | LR: 4.21e-04 | 4025.5ms/step | 14.9 samples/s
125
+ Step 13740/50462 | Loss: 2.7230 | LR: 4.20e-04 | 4020.2ms/step | 14.9 samples/s
126
+ Step 13750/50462 | Loss: 2.7012 | LR: 4.20e-04 | 4014.9ms/step | 14.9 samples/s
127
+ Validation at step 13750: Loss = 2.7390, Perplexity = 15.47
128
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-13750
129
+ Step 13760/50462 | Loss: 2.7607 | LR: 4.20e-04 | 5968.9ms/step | 10.1 samples/s
130
+ Step 13770/50462 | Loss: 2.7173 | LR: 4.20e-04 | 4009.8ms/step | 15.0 samples/s
131
+ Step 13780/50462 | Loss: 2.7164 | LR: 4.20e-04 | 4024.8ms/step | 14.9 samples/s
132
+ Step 13790/50462 | Loss: 2.7417 | LR: 4.20e-04 | 4027.9ms/step | 14.9 samples/s
133
+ Step 13800/50462 | Loss: 2.7713 | LR: 4.20e-04 | 4020.0ms/step | 14.9 samples/s
134
+ Step 13810/50462 | Loss: 2.7233 | LR: 4.20e-04 | 4012.5ms/step | 15.0 samples/s
135
+ Step 13820/50462 | Loss: 2.7503 | LR: 4.19e-04 | 4012.4ms/step | 15.0 samples/s
136
+ Step 13830/50462 | Loss: 2.7242 | LR: 4.19e-04 | 4030.8ms/step | 14.9 samples/s
137
+ Step 13840/50462 | Loss: 2.7048 | LR: 4.19e-04 | 4024.6ms/step | 14.9 samples/s
138
+ Step 13850/50462 | Loss: 2.7077 | LR: 4.19e-04 | 4027.0ms/step | 14.9 samples/s
139
+ Step 13860/50462 | Loss: 2.7235 | LR: 4.19e-04 | 4017.0ms/step | 14.9 samples/s
140
+ Step 13870/50462 | Loss: 2.7201 | LR: 4.19e-04 | 4007.8ms/step | 15.0 samples/s
141
+ Step 13880/50462 | Loss: 2.7191 | LR: 4.19e-04 | 4004.5ms/step | 15.0 samples/s
142
+ Step 13890/50462 | Loss: 2.7156 | LR: 4.19e-04 | 4009.4ms/step | 15.0 samples/s
143
+ Step 13900/50462 | Loss: 2.7480 | LR: 4.19e-04 | 4014.4ms/step | 14.9 samples/s
144
+ Step 13910/50462 | Loss: 2.6811 | LR: 4.18e-04 | 4018.7ms/step | 14.9 samples/s
145
+ Step 13920/50462 | Loss: 2.7350 | LR: 4.18e-04 | 4012.5ms/step | 15.0 samples/s
146
+ Step 13930/50462 | Loss: 2.7440 | LR: 4.18e-04 | 4023.3ms/step | 14.9 samples/s
147
+ Step 13940/50462 | Loss: 2.7106 | LR: 4.18e-04 | 4014.5ms/step | 14.9 samples/s
148
+ Step 13950/50462 | Loss: 2.7478 | LR: 4.18e-04 | 4014.2ms/step | 14.9 samples/s
149
+ Step 13960/50462 | Loss: 2.7089 | LR: 4.18e-04 | 4015.1ms/step | 14.9 samples/s
150
+ Step 13970/50462 | Loss: 2.7536 | LR: 4.18e-04 | 4020.5ms/step | 14.9 samples/s
151
+ Step 13980/50462 | Loss: 2.7055 | LR: 4.18e-04 | 4018.2ms/step | 14.9 samples/s
152
+ Step 13990/50462 | Loss: 2.7216 | LR: 4.17e-04 | 4017.5ms/step | 14.9 samples/s
153
+ Step 14000/50462 | Loss: 2.7400 | LR: 4.17e-04 | 4019.4ms/step | 14.9 samples/s
154
+ Validation at step 14000: Loss = 2.7337, Perplexity = 15.39
155
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-14000
156
+ Step 14010/50462 | Loss: 2.6458 | LR: 4.17e-04 | 5979.7ms/step | 10.0 samples/s
157
+ Step 14020/50462 | Loss: 2.7110 | LR: 4.17e-04 | 4005.3ms/step | 15.0 samples/s
158
+ Step 14030/50462 | Loss: 2.7184 | LR: 4.17e-04 | 4029.4ms/step | 14.9 samples/s
159
+ Step 14040/50462 | Loss: 2.7420 | LR: 4.17e-04 | 4021.2ms/step | 14.9 samples/s
160
+ Step 14050/50462 | Loss: 2.7159 | LR: 4.17e-04 | 4016.8ms/step | 14.9 samples/s
161
+ Step 14060/50462 | Loss: 2.7528 | LR: 4.17e-04 | 4013.7ms/step | 14.9 samples/s
162
+ Step 14070/50462 | Loss: 2.6917 | LR: 4.17e-04 | 4007.7ms/step | 15.0 samples/s
163
+ Step 14080/50462 | Loss: 2.7285 | LR: 4.16e-04 | 4026.4ms/step | 14.9 samples/s
164
+ Step 14090/50462 | Loss: 2.7174 | LR: 4.16e-04 | 4016.5ms/step | 14.9 samples/s
165
+ Step 14100/50462 | Loss: 2.7091 | LR: 4.16e-04 | 4023.6ms/step | 14.9 samples/s
166
+ Step 14110/50462 | Loss: 2.6750 | LR: 4.16e-04 | 4011.8ms/step | 15.0 samples/s
167
+ Step 14120/50462 | Loss: 2.7185 | LR: 4.16e-04 | 4015.9ms/step | 14.9 samples/s
168
+ Step 14130/50462 | Loss: 2.7267 | LR: 4.16e-04 | 4008.0ms/step | 15.0 samples/s
169
+ Step 14140/50462 | Loss: 2.7302 | LR: 4.16e-04 | 4013.8ms/step | 14.9 samples/s
170
+ Step 14150/50462 | Loss: 2.6959 | LR: 4.16e-04 | 4011.2ms/step | 15.0 samples/s
171
+ Step 14160/50462 | Loss: 2.7191 | LR: 4.15e-04 | 4012.5ms/step | 15.0 samples/s
172
+ Step 14170/50462 | Loss: 2.7313 | LR: 4.15e-04 | 4008.6ms/step | 15.0 samples/s
173
+ Step 14180/50462 | Loss: 2.7025 | LR: 4.15e-04 | 4019.0ms/step | 14.9 samples/s
174
+ Step 14190/50462 | Loss: 2.7579 | LR: 4.15e-04 | 4019.4ms/step | 14.9 samples/s
175
+ Step 14200/50462 | Loss: 2.7058 | LR: 4.15e-04 | 4020.2ms/step | 14.9 samples/s
176
+ Step 14210/50462 | Loss: 2.7364 | LR: 4.15e-04 | 4018.9ms/step | 14.9 samples/s
177
+ Step 14220/50462 | Loss: 2.7248 | LR: 4.15e-04 | 4018.7ms/step | 14.9 samples/s
178
+ Step 14230/50462 | Loss: 2.7052 | LR: 4.15e-04 | 4004.0ms/step | 15.0 samples/s
179
+ Step 14240/50462 | Loss: 2.7063 | LR: 4.15e-04 | 4025.1ms/step | 14.9 samples/s
180
+ Step 14250/50462 | Loss: 2.7241 | LR: 4.14e-04 | 4019.7ms/step | 14.9 samples/s
181
+ Validation at step 14250: Loss = 2.7331, Perplexity = 15.38
182
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-14250
183
+ Step 14260/50462 | Loss: 2.6867 | LR: 4.14e-04 | 5962.3ms/step | 10.1 samples/s
184
+ Step 14270/50462 | Loss: 2.7149 | LR: 4.14e-04 | 4015.3ms/step | 14.9 samples/s
185
+ Step 14280/50462 | Loss: 2.6948 | LR: 4.14e-04 | 4003.0ms/step | 15.0 samples/s
186
+ Step 14290/50462 | Loss: 2.7311 | LR: 4.14e-04 | 4005.5ms/step | 15.0 samples/s
187
+ Step 14300/50462 | Loss: 2.7273 | LR: 4.14e-04 | 4010.9ms/step | 15.0 samples/s
188
+ Step 14310/50462 | Loss: 2.6862 | LR: 4.14e-04 | 4011.9ms/step | 15.0 samples/s
189
+ Step 14320/50462 | Loss: 2.7129 | LR: 4.14e-04 | 4012.6ms/step | 15.0 samples/s
190
+ Step 14330/50462 | Loss: 2.6832 | LR: 4.13e-04 | 4011.4ms/step | 15.0 samples/s
191
+ Step 14340/50462 | Loss: 2.7302 | LR: 4.13e-04 | 4016.7ms/step | 14.9 samples/s
192
+ Step 14350/50462 | Loss: 2.7500 | LR: 4.13e-04 | 4014.8ms/step | 14.9 samples/s
193
+ Step 14360/50462 | Loss: 2.7155 | LR: 4.13e-04 | 3993.3ms/step | 15.0 samples/s
194
+ Step 14370/50462 | Loss: 2.7010 | LR: 4.13e-04 | 4011.3ms/step | 15.0 samples/s
195
+ Step 14380/50462 | Loss: 2.7362 | LR: 4.13e-04 | 4006.9ms/step | 15.0 samples/s
196
+ Step 14390/50462 | Loss: 2.6854 | LR: 4.13e-04 | 4014.9ms/step | 14.9 samples/s
197
+ Step 14400/50462 | Loss: 2.7238 | LR: 4.13e-04 | 3994.4ms/step | 15.0 samples/s
198
+ Step 14410/50462 | Loss: 2.6898 | LR: 4.12e-04 | 4012.3ms/step | 15.0 samples/s
199
+ Step 14420/50462 | Loss: 2.7323 | LR: 4.12e-04 | 4011.3ms/step | 15.0 samples/s
200
+ Step 14430/50462 | Loss: 2.6979 | LR: 4.12e-04 | 4001.8ms/step | 15.0 samples/s
201
+ Step 14440/50462 | Loss: 2.7344 | LR: 4.12e-04 | 4009.2ms/step | 15.0 samples/s
202
+ Step 14450/50462 | Loss: 2.7132 | LR: 4.12e-04 | 4001.2ms/step | 15.0 samples/s
203
+ Step 14460/50462 | Loss: 2.6866 | LR: 4.12e-04 | 4003.0ms/step | 15.0 samples/s
204
+ Step 14470/50462 | Loss: 2.7019 | LR: 4.12e-04 | 4012.6ms/step | 15.0 samples/s
205
+ Step 14480/50462 | Loss: 2.7302 | LR: 4.12e-04 | 4005.8ms/step | 15.0 samples/s
206
+ Step 14490/50462 | Loss: 2.7361 | LR: 4.12e-04 | 4011.3ms/step | 15.0 samples/s
207
+ Step 14500/50462 | Loss: 2.6802 | LR: 4.11e-04 | 4007.3ms/step | 15.0 samples/s
208
+ Validation at step 14500: Loss = 2.7292, Perplexity = 15.32
209
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-14500
210
+ Step 14510/50462 | Loss: 2.7137 | LR: 4.11e-04 | 5968.5ms/step | 10.1 samples/s
211
+ Step 14520/50462 | Loss: 2.6919 | LR: 4.11e-04 | 4000.2ms/step | 15.0 samples/s
212
+ Step 14530/50462 | Loss: 2.6990 | LR: 4.11e-04 | 4012.9ms/step | 15.0 samples/s
213
+ Step 14540/50462 | Loss: 2.6883 | LR: 4.11e-04 | 4017.9ms/step | 14.9 samples/s
214
+ Step 14550/50462 | Loss: 2.7259 | LR: 4.11e-04 | 4000.7ms/step | 15.0 samples/s
215
+ Step 14560/50462 | Loss: 2.7091 | LR: 4.11e-04 | 4007.2ms/step | 15.0 samples/s
216
+ Step 14570/50462 | Loss: 2.7241 | LR: 4.11e-04 | 4003.6ms/step | 15.0 samples/s
217
+ Step 14580/50462 | Loss: 2.7179 | LR: 4.10e-04 | 4005.2ms/step | 15.0 samples/s
218
+ Step 14590/50462 | Loss: 2.7130 | LR: 4.10e-04 | 4006.7ms/step | 15.0 samples/s
219
+ Step 14600/50462 | Loss: 2.7097 | LR: 4.10e-04 | 4001.8ms/step | 15.0 samples/s
220
+ Step 14610/50462 | Loss: 2.6809 | LR: 4.10e-04 | 4012.1ms/step | 15.0 samples/s
221
+ Step 14620/50462 | Loss: 2.7082 | LR: 4.10e-04 | 4008.0ms/step | 15.0 samples/s
222
+ Step 14630/50462 | Loss: 2.7073 | LR: 4.10e-04 | 4008.1ms/step | 15.0 samples/s
223
+ Step 14640/50462 | Loss: 2.7164 | LR: 4.10e-04 | 3999.2ms/step | 15.0 samples/s
224
+ Step 14650/50462 | Loss: 2.7148 | LR: 4.10e-04 | 4006.4ms/step | 15.0 samples/s
225
+ Step 14660/50462 | Loss: 2.7379 | LR: 4.09e-04 | 4001.0ms/step | 15.0 samples/s
226
+ Step 14670/50462 | Loss: 2.7071 | LR: 4.09e-04 | 4002.3ms/step | 15.0 samples/s
227
+ Step 14680/50462 | Loss: 2.6978 | LR: 4.09e-04 | 4012.1ms/step | 15.0 samples/s
228
+ Step 14690/50462 | Loss: 2.6772 | LR: 4.09e-04 | 4010.3ms/step | 15.0 samples/s
229
+ Step 14700/50462 | Loss: 2.6842 | LR: 4.09e-04 | 4004.0ms/step | 15.0 samples/s
230
+ Step 14710/50462 | Loss: 2.7023 | LR: 4.09e-04 | 4001.6ms/step | 15.0 samples/s
231
+ Step 14720/50462 | Loss: 2.6839 | LR: 4.09e-04 | 4010.4ms/step | 15.0 samples/s
232
+ Step 14730/50462 | Loss: 2.6864 | LR: 4.09e-04 | 3999.0ms/step | 15.0 samples/s
233
+ Step 14740/50462 | Loss: 2.6844 | LR: 4.08e-04 | 4012.0ms/step | 15.0 samples/s
234
+ Step 14750/50462 | Loss: 2.6957 | LR: 4.08e-04 | 4011.6ms/step | 15.0 samples/s
235
+ Validation at step 14750: Loss = 2.7238, Perplexity = 15.24
236
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-14750
237
+ Step 14760/50462 | Loss: 2.7194 | LR: 4.08e-04 | 5985.5ms/step | 10.0 samples/s
238
+ Step 14770/50462 | Loss: 2.6879 | LR: 4.08e-04 | 4010.5ms/step | 15.0 samples/s
239
+ Step 14780/50462 | Loss: 2.7001 | LR: 4.08e-04 | 4001.7ms/step | 15.0 samples/s
240
+ Step 14790/50462 | Loss: 2.7353 | LR: 4.08e-04 | 3997.7ms/step | 15.0 samples/s
241
+ Step 14800/50462 | Loss: 2.7046 | LR: 4.08e-04 | 3999.9ms/step | 15.0 samples/s
242
+ Step 14810/50462 | Loss: 2.7194 | LR: 4.08e-04 | 4014.8ms/step | 14.9 samples/s
243
+ Step 14820/50462 | Loss: 2.6871 | LR: 4.08e-04 | 4006.4ms/step | 15.0 samples/s
244
+ Step 14830/50462 | Loss: 2.7302 | LR: 4.07e-04 | 4005.8ms/step | 15.0 samples/s
245
+ Step 14840/50462 | Loss: 2.6759 | LR: 4.07e-04 | 4012.4ms/step | 15.0 samples/s
246
+ Step 14850/50462 | Loss: 2.7142 | LR: 4.07e-04 | 4015.0ms/step | 14.9 samples/s
247
+ Step 14860/50462 | Loss: 2.7172 | LR: 4.07e-04 | 4012.0ms/step | 15.0 samples/s
248
+ Step 14870/50462 | Loss: 2.6863 | LR: 4.07e-04 | 4008.0ms/step | 15.0 samples/s
249
+ Step 14880/50462 | Loss: 2.7308 | LR: 4.07e-04 | 4018.5ms/step | 14.9 samples/s
250
+ Step 14890/50462 | Loss: 2.6787 | LR: 4.07e-04 | 4018.9ms/step | 14.9 samples/s
251
+ Step 14900/50462 | Loss: 2.6897 | LR: 4.07e-04 | 4013.5ms/step | 14.9 samples/s
252
+ Step 14910/50462 | Loss: 2.6841 | LR: 4.06e-04 | 4007.2ms/step | 15.0 samples/s
253
+ Step 14920/50462 | Loss: 2.6731 | LR: 4.06e-04 | 4006.6ms/step | 15.0 samples/s
254
+ Step 14930/50462 | Loss: 2.7251 | LR: 4.06e-04 | 4009.6ms/step | 15.0 samples/s
255
+ Step 14940/50462 | Loss: 2.7061 | LR: 4.06e-04 | 3998.0ms/step | 15.0 samples/s
256
+ Step 14950/50462 | Loss: 2.6881 | LR: 4.06e-04 | 4003.9ms/step | 15.0 samples/s
257
+ Step 14960/50462 | Loss: 2.7167 | LR: 4.06e-04 | 4004.0ms/step | 15.0 samples/s
258
+ Step 14970/50462 | Loss: 2.6950 | LR: 4.06e-04 | 4011.0ms/step | 15.0 samples/s
259
+ Step 14980/50462 | Loss: 2.6984 | LR: 4.06e-04 | 4012.5ms/step | 15.0 samples/s
260
+ Step 14990/50462 | Loss: 2.6928 | LR: 4.05e-04 | 3998.3ms/step | 15.0 samples/s
261
+ Step 15000/50462 | Loss: 2.7452 | LR: 4.05e-04 | 4006.4ms/step | 15.0 samples/s
262
+ Validation at step 15000: Loss = 2.7208, Perplexity = 15.19
263
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-15000
264
+ Step 15010/50462 | Loss: 2.7034 | LR: 4.05e-04 | 5970.4ms/step | 10.0 samples/s
265
+ Step 15020/50462 | Loss: 2.6949 | LR: 4.05e-04 | 4011.3ms/step | 15.0 samples/s
266
+ Step 15030/50462 | Loss: 2.6840 | LR: 4.05e-04 | 3999.4ms/step | 15.0 samples/s
267
+ Step 15040/50462 | Loss: 2.6799 | LR: 4.05e-04 | 4008.1ms/step | 15.0 samples/s
268
+ Step 15050/50462 | Loss: 2.7005 | LR: 4.05e-04 | 4017.9ms/step | 14.9 samples/s
269
+ Step 15060/50462 | Loss: 2.7248 | LR: 4.05e-04 | 4000.3ms/step | 15.0 samples/s
270
+ Step 15070/50462 | Loss: 2.7047 | LR: 4.04e-04 | 4004.3ms/step | 15.0 samples/s
271
+ Step 15080/50462 | Loss: 2.7254 | LR: 4.04e-04 | 4008.1ms/step | 15.0 samples/s
272
+ Step 15090/50462 | Loss: 2.6936 | LR: 4.04e-04 | 4005.8ms/step | 15.0 samples/s
273
+ Step 15100/50462 | Loss: 2.6797 | LR: 4.04e-04 | 4014.0ms/step | 14.9 samples/s
274
+ Step 15110/50462 | Loss: 2.7062 | LR: 4.04e-04 | 4003.8ms/step | 15.0 samples/s
275
+ Step 15120/50462 | Loss: 2.7052 | LR: 4.04e-04 | 4009.8ms/step | 15.0 samples/s
276
+ Step 15130/50462 | Loss: 2.7006 | LR: 4.04e-04 | 4006.7ms/step | 15.0 samples/s
277
+ Step 15140/50462 | Loss: 2.7194 | LR: 4.04e-04 | 3999.6ms/step | 15.0 samples/s
278
+ Step 15150/50462 | Loss: 2.6905 | LR: 4.03e-04 | 4009.8ms/step | 15.0 samples/s
279
+ Step 15160/50462 | Loss: 2.7150 | LR: 4.03e-04 | 4015.7ms/step | 14.9 samples/s
280
+ Step 15170/50462 | Loss: 2.7039 | LR: 4.03e-04 | 4004.8ms/step | 15.0 samples/s
281
+ Step 15180/50462 | Loss: 2.6536 | LR: 4.03e-04 | 4005.5ms/step | 15.0 samples/s
282
+ Step 15190/50462 | Loss: 2.6936 | LR: 4.03e-04 | 4004.7ms/step | 15.0 samples/s
283
+ Step 15200/50462 | Loss: 2.6943 | LR: 4.03e-04 | 4009.2ms/step | 15.0 samples/s
284
+ Step 15210/50462 | Loss: 2.6824 | LR: 4.03e-04 | 4009.4ms/step | 15.0 samples/s
285
+ Step 15220/50462 | Loss: 2.7155 | LR: 4.03e-04 | 3991.9ms/step | 15.0 samples/s
286
+ Step 15230/50462 | Loss: 2.6915 | LR: 4.02e-04 | 4009.5ms/step | 15.0 samples/s
287
+ Step 15240/50462 | Loss: 2.7130 | LR: 4.02e-04 | 4030.7ms/step | 14.9 samples/s
288
+ Step 15250/50462 | Loss: 2.7361 | LR: 4.02e-04 | 4010.2ms/step | 15.0 samples/s
289
+ Validation at step 15250: Loss = 2.7188, Perplexity = 15.16
290
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-15250
291
+ Step 15260/50462 | Loss: 2.6959 | LR: 4.02e-04 | 5970.9ms/step | 10.0 samples/s
292
+ Step 15270/50462 | Loss: 2.6981 | LR: 4.02e-04 | 4004.4ms/step | 15.0 samples/s
293
+ Step 15280/50462 | Loss: 2.6940 | LR: 4.02e-04 | 3997.0ms/step | 15.0 samples/s
294
+ Step 15290/50462 | Loss: 2.6852 | LR: 4.02e-04 | 4011.7ms/step | 15.0 samples/s
295
+ Step 15300/50462 | Loss: 2.7302 | LR: 4.02e-04 | 4020.5ms/step | 14.9 samples/s
296
+ Step 15310/50462 | Loss: 2.6914 | LR: 4.01e-04 | 4017.4ms/step | 14.9 samples/s
297
+ Step 15320/50462 | Loss: 2.7140 | LR: 4.01e-04 | 4018.8ms/step | 14.9 samples/s
298
+ Step 15330/50462 | Loss: 2.7012 | LR: 4.01e-04 | 4007.9ms/step | 15.0 samples/s
299
+ Step 15340/50462 | Loss: 2.7100 | LR: 4.01e-04 | 4009.0ms/step | 15.0 samples/s
300
+ Step 15350/50462 | Loss: 2.7091 | LR: 4.01e-04 | 4020.9ms/step | 14.9 samples/s
301
+ Step 15360/50462 | Loss: 2.7052 | LR: 4.01e-04 | 4010.3ms/step | 15.0 samples/s
302
+ Step 15370/50462 | Loss: 2.7023 | LR: 4.01e-04 | 4014.2ms/step | 14.9 samples/s
303
+ Step 15380/50462 | Loss: 2.7065 | LR: 4.01e-04 | 4002.1ms/step | 15.0 samples/s
304
+ Step 15390/50462 | Loss: 2.7167 | LR: 4.00e-04 | 4003.9ms/step | 15.0 samples/s
305
+ Step 15400/50462 | Loss: 2.6940 | LR: 4.00e-04 | 4001.1ms/step | 15.0 samples/s
306
+ Step 15410/50462 | Loss: 2.6652 | LR: 4.00e-04 | 4004.2ms/step | 15.0 samples/s
307
+ Step 15420/50462 | Loss: 2.6945 | LR: 4.00e-04 | 4015.3ms/step | 14.9 samples/s
308
+ Step 15430/50462 | Loss: 2.6710 | LR: 4.00e-04 | 3986.0ms/step | 15.1 samples/s
309
+ Step 15440/50462 | Loss: 2.7190 | LR: 4.00e-04 | 4012.5ms/step | 15.0 samples/s
310
+ Step 15450/50462 | Loss: 2.7146 | LR: 4.00e-04 | 4012.5ms/step | 15.0 samples/s
311
+ Step 15460/50462 | Loss: 2.7072 | LR: 4.00e-04 | 4003.1ms/step | 15.0 samples/s
312
+ Step 15470/50462 | Loss: 2.6928 | LR: 3.99e-04 | 4005.6ms/step | 15.0 samples/s
313
+ Step 15480/50462 | Loss: 2.6840 | LR: 3.99e-04 | 4002.3ms/step | 15.0 samples/s
314
+ Step 15490/50462 | Loss: 2.6617 | LR: 3.99e-04 | 4001.5ms/step | 15.0 samples/s
315
+ Step 15500/50462 | Loss: 2.7044 | LR: 3.99e-04 | 4007.2ms/step | 15.0 samples/s
316
+ Validation at step 15500: Loss = 2.7139, Perplexity = 15.09
317
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-15500
318
+ Step 15510/50462 | Loss: 2.7023 | LR: 3.99e-04 | 5960.9ms/step | 10.1 samples/s
319
+ Step 15520/50462 | Loss: 2.7193 | LR: 3.99e-04 | 4002.8ms/step | 15.0 samples/s
320
+ Step 15530/50462 | Loss: 2.6816 | LR: 3.99e-04 | 4002.9ms/step | 15.0 samples/s
321
+ Step 15540/50462 | Loss: 2.6860 | LR: 3.99e-04 | 4010.8ms/step | 15.0 samples/s
322
+ Step 15550/50462 | Loss: 2.6809 | LR: 3.98e-04 | 4003.1ms/step | 15.0 samples/s
323
+ Step 15560/50462 | Loss: 2.6873 | LR: 3.98e-04 | 4002.8ms/step | 15.0 samples/s
324
+ Step 15570/50462 | Loss: 2.6867 | LR: 3.98e-04 | 4005.3ms/step | 15.0 samples/s
325
+ Step 15580/50462 | Loss: 2.6985 | LR: 3.98e-04 | 4001.0ms/step | 15.0 samples/s
326
+ Step 15590/50462 | Loss: 2.7084 | LR: 3.98e-04 | 4011.2ms/step | 15.0 samples/s
327
+ Step 15600/50462 | Loss: 2.6910 | LR: 3.98e-04 | 4014.4ms/step | 14.9 samples/s
328
+ Step 15610/50462 | Loss: 2.6679 | LR: 3.98e-04 | 4018.4ms/step | 14.9 samples/s
329
+ Step 15620/50462 | Loss: 2.7351 | LR: 3.98e-04 | 4005.1ms/step | 15.0 samples/s
330
+ Step 15630/50462 | Loss: 2.6759 | LR: 3.97e-04 | 4013.6ms/step | 14.9 samples/s
331
+ Step 15640/50462 | Loss: 2.6824 | LR: 3.97e-04 | 4020.1ms/step | 14.9 samples/s
332
+ Step 15650/50462 | Loss: 2.6704 | LR: 3.97e-04 | 3994.1ms/step | 15.0 samples/s
333
+ Step 15660/50462 | Loss: 2.7091 | LR: 3.97e-04 | 4011.5ms/step | 15.0 samples/s
334
+ Step 15670/50462 | Loss: 2.6894 | LR: 3.97e-04 | 4001.7ms/step | 15.0 samples/s
335
+ Step 15680/50462 | Loss: 2.6970 | LR: 3.97e-04 | 4003.2ms/step | 15.0 samples/s
336
+ Step 15690/50462 | Loss: 2.7111 | LR: 3.97e-04 | 4010.2ms/step | 15.0 samples/s
337
+ Step 15700/50462 | Loss: 2.6846 | LR: 3.96e-04 | 4002.2ms/step | 15.0 samples/s
338
+ Step 15710/50462 | Loss: 2.6951 | LR: 3.96e-04 | 3992.2ms/step | 15.0 samples/s
339
+ Step 15720/50462 | Loss: 2.6678 | LR: 3.96e-04 | 4002.7ms/step | 15.0 samples/s
340
+ Step 15730/50462 | Loss: 2.6703 | LR: 3.96e-04 | 4001.9ms/step | 15.0 samples/s
341
+ Step 15740/50462 | Loss: 2.6782 | LR: 3.96e-04 | 3995.7ms/step | 15.0 samples/s
342
+ Step 15750/50462 | Loss: 2.7153 | LR: 3.96e-04 | 4009.4ms/step | 15.0 samples/s
343
+ Validation at step 15750: Loss = 2.7112, Perplexity = 15.05
344
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-15750
345
+ Step 15760/50462 | Loss: 2.7194 | LR: 3.96e-04 | 5941.0ms/step | 10.1 samples/s
346
+ Step 15770/50462 | Loss: 2.7094 | LR: 3.96e-04 | 4002.1ms/step | 15.0 samples/s
347
+ Step 15780/50462 | Loss: 2.6973 | LR: 3.95e-04 | 4009.1ms/step | 15.0 samples/s
348
+ Step 15790/50462 | Loss: 2.7009 | LR: 3.95e-04 | 4007.7ms/step | 15.0 samples/s
349
+ Step 15800/50462 | Loss: 2.7200 | LR: 3.95e-04 | 4009.1ms/step | 15.0 samples/s
350
+ Step 15810/50462 | Loss: 2.6627 | LR: 3.95e-04 | 4000.3ms/step | 15.0 samples/s
351
+ Step 15820/50462 | Loss: 2.7031 | LR: 3.95e-04 | 4000.9ms/step | 15.0 samples/s
352
+ Step 15830/50462 | Loss: 2.6687 | LR: 3.95e-04 | 4007.0ms/step | 15.0 samples/s
353
+ Step 15840/50462 | Loss: 2.7059 | LR: 3.95e-04 | 4002.2ms/step | 15.0 samples/s
354
+ Step 15850/50462 | Loss: 2.6625 | LR: 3.95e-04 | 4005.6ms/step | 15.0 samples/s
355
+ Step 15860/50462 | Loss: 2.6886 | LR: 3.94e-04 | 4015.0ms/step | 14.9 samples/s
356
+ Step 15870/50462 | Loss: 2.6833 | LR: 3.94e-04 | 4005.3ms/step | 15.0 samples/s
357
+ Step 15880/50462 | Loss: 2.6951 | LR: 3.94e-04 | 3994.3ms/step | 15.0 samples/s
358
+ Step 15890/50462 | Loss: 2.7337 | LR: 3.94e-04 | 3992.5ms/step | 15.0 samples/s
359
+ Step 15900/50462 | Loss: 2.7154 | LR: 3.94e-04 | 4007.4ms/step | 15.0 samples/s
360
+ Step 15910/50462 | Loss: 2.6635 | LR: 3.94e-04 | 4007.0ms/step | 15.0 samples/s
361
+ Step 15920/50462 | Loss: 2.7058 | LR: 3.94e-04 | 3996.3ms/step | 15.0 samples/s
362
+ Step 15930/50462 | Loss: 2.6729 | LR: 3.94e-04 | 4016.6ms/step | 14.9 samples/s
363
+ Step 15940/50462 | Loss: 2.6849 | LR: 3.93e-04 | 4003.3ms/step | 15.0 samples/s
364
+ Step 15950/50462 | Loss: 2.6730 | LR: 3.93e-04 | 3998.2ms/step | 15.0 samples/s
365
+ Step 15960/50462 | Loss: 2.7168 | LR: 3.93e-04 | 4005.3ms/step | 15.0 samples/s
366
+ Step 15970/50462 | Loss: 2.6690 | LR: 3.93e-04 | 4008.4ms/step | 15.0 samples/s
367
+ Step 15980/50462 | Loss: 2.7202 | LR: 3.93e-04 | 4004.6ms/step | 15.0 samples/s
368
+ Step 15990/50462 | Loss: 2.6931 | LR: 3.93e-04 | 4008.0ms/step | 15.0 samples/s
369
+ Step 16000/50462 | Loss: 2.6749 | LR: 3.93e-04 | 4004.2ms/step | 15.0 samples/s
370
+ Validation at step 16000: Loss = 2.7094, Perplexity = 15.02
371
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-16000
372
+ Step 16010/50462 | Loss: 2.6764 | LR: 3.92e-04 | 5953.5ms/step | 10.1 samples/s
373
+ Step 16020/50462 | Loss: 2.6990 | LR: 3.92e-04 | 4003.4ms/step | 15.0 samples/s
374
+ Step 16030/50462 | Loss: 2.6906 | LR: 3.92e-04 | 4016.0ms/step | 14.9 samples/s
375
+ Step 16040/50462 | Loss: 2.6669 | LR: 3.92e-04 | 4011.4ms/step | 15.0 samples/s
376
+ Step 16050/50462 | Loss: 2.6529 | LR: 3.92e-04 | 4000.2ms/step | 15.0 samples/s
377
+ Step 16060/50462 | Loss: 2.7272 | LR: 3.92e-04 | 4002.6ms/step | 15.0 samples/s
378
+ Step 16070/50462 | Loss: 2.6908 | LR: 3.92e-04 | 4001.6ms/step | 15.0 samples/s
379
+ Step 16080/50462 | Loss: 2.6769 | LR: 3.92e-04 | 4013.6ms/step | 14.9 samples/s
380
+ Step 16090/50462 | Loss: 2.6822 | LR: 3.91e-04 | 4012.3ms/step | 15.0 samples/s
381
+ Step 16100/50462 | Loss: 2.6978 | LR: 3.91e-04 | 4018.9ms/step | 14.9 samples/s
382
+ Step 16110/50462 | Loss: 2.6784 | LR: 3.91e-04 | 4007.5ms/step | 15.0 samples/s
383
+ Step 16120/50462 | Loss: 2.7110 | LR: 3.91e-04 | 4007.8ms/step | 15.0 samples/s
384
+ Step 16130/50462 | Loss: 2.6998 | LR: 3.91e-04 | 4011.1ms/step | 15.0 samples/s
385
+ Step 16140/50462 | Loss: 2.6976 | LR: 3.91e-04 | 4009.0ms/step | 15.0 samples/s
386
+ Step 16150/50462 | Loss: 2.6950 | LR: 3.91e-04 | 4003.8ms/step | 15.0 samples/s
387
+ Step 16160/50462 | Loss: 2.7251 | LR: 3.91e-04 | 4007.1ms/step | 15.0 samples/s
388
+ Step 16170/50462 | Loss: 2.7289 | LR: 3.90e-04 | 4005.0ms/step | 15.0 samples/s
389
+ Step 16180/50462 | Loss: 2.6872 | LR: 3.90e-04 | 4010.6ms/step | 15.0 samples/s
390
+ Step 16190/50462 | Loss: 2.6466 | LR: 3.90e-04 | 4005.8ms/step | 15.0 samples/s
391
+ Step 16200/50462 | Loss: 2.6923 | LR: 3.90e-04 | 3999.3ms/step | 15.0 samples/s
392
+ Step 16210/50462 | Loss: 2.6912 | LR: 3.90e-04 | 4002.0ms/step | 15.0 samples/s
393
+ Step 16220/50462 | Loss: 2.6920 | LR: 3.90e-04 | 4006.2ms/step | 15.0 samples/s
394
+ Step 16230/50462 | Loss: 2.6841 | LR: 3.90e-04 | 4003.9ms/step | 15.0 samples/s
395
+ Step 16240/50462 | Loss: 2.6625 | LR: 3.89e-04 | 4007.4ms/step | 15.0 samples/s
396
+ Step 16250/50462 | Loss: 2.6641 | LR: 3.89e-04 | 4003.4ms/step | 15.0 samples/s
397
+ Validation at step 16250: Loss = 2.7057, Perplexity = 14.96
398
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-16250
399
+ Step 16260/50462 | Loss: 2.6943 | LR: 3.89e-04 | 5966.3ms/step | 10.1 samples/s
400
+ Step 16270/50462 | Loss: 2.7186 | LR: 3.89e-04 | 3994.7ms/step | 15.0 samples/s
401
+ Step 16280/50462 | Loss: 2.6708 | LR: 3.89e-04 | 3999.4ms/step | 15.0 samples/s
402
+ Step 16290/50462 | Loss: 2.6938 | LR: 3.89e-04 | 4004.9ms/step | 15.0 samples/s
403
+ Step 16300/50462 | Loss: 2.6975 | LR: 3.89e-04 | 4006.9ms/step | 15.0 samples/s
404
+ Step 16310/50462 | Loss: 2.6845 | LR: 3.89e-04 | 4003.0ms/step | 15.0 samples/s
405
+ Step 16320/50462 | Loss: 2.6763 | LR: 3.88e-04 | 4003.5ms/step | 15.0 samples/s
406
+ Step 16330/50462 | Loss: 2.6870 | LR: 3.88e-04 | 4015.1ms/step | 14.9 samples/s
407
+ Step 16340/50462 | Loss: 2.6754 | LR: 3.88e-04 | 4004.2ms/step | 15.0 samples/s
408
+ Step 16350/50462 | Loss: 2.6910 | LR: 3.88e-04 | 4003.2ms/step | 15.0 samples/s
409
+ Step 16360/50462 | Loss: 2.6605 | LR: 3.88e-04 | 4009.8ms/step | 15.0 samples/s
410
+ Step 16370/50462 | Loss: 2.6613 | LR: 3.88e-04 | 4003.8ms/step | 15.0 samples/s
411
+ Step 16380/50462 | Loss: 2.6861 | LR: 3.88e-04 | 4010.6ms/step | 15.0 samples/s
412
+ Step 16390/50462 | Loss: 2.6880 | LR: 3.88e-04 | 4002.6ms/step | 15.0 samples/s
413
+ Step 16400/50462 | Loss: 2.6949 | LR: 3.87e-04 | 4004.3ms/step | 15.0 samples/s
414
+ Step 16410/50462 | Loss: 2.6831 | LR: 3.87e-04 | 4010.0ms/step | 15.0 samples/s
415
+ Step 16420/50462 | Loss: 2.6614 | LR: 3.87e-04 | 4005.3ms/step | 15.0 samples/s
416
+ Step 16430/50462 | Loss: 2.6587 | LR: 3.87e-04 | 4003.1ms/step | 15.0 samples/s
417
+ Step 16440/50462 | Loss: 2.6647 | LR: 3.87e-04 | 3998.6ms/step | 15.0 samples/s
418
+ Step 16450/50462 | Loss: 2.6970 | LR: 3.87e-04 | 4002.5ms/step | 15.0 samples/s
419
+ Step 16460/50462 | Loss: 2.6944 | LR: 3.87e-04 | 4005.9ms/step | 15.0 samples/s
420
+ Step 16470/50462 | Loss: 2.7077 | LR: 3.86e-04 | 4009.0ms/step | 15.0 samples/s
421
+ Step 16480/50462 | Loss: 2.6760 | LR: 3.86e-04 | 4001.4ms/step | 15.0 samples/s
422
+ Step 16490/50462 | Loss: 2.6573 | LR: 3.86e-04 | 4007.0ms/step | 15.0 samples/s
423
+ Step 16500/50462 | Loss: 2.6644 | LR: 3.86e-04 | 4004.4ms/step | 15.0 samples/s
424
+ Validation at step 16500: Loss = 2.7030, Perplexity = 14.93
425
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-16500
426
+ Step 16510/50462 | Loss: 2.7129 | LR: 3.86e-04 | 5957.1ms/step | 10.1 samples/s
427
+ Step 16520/50462 | Loss: 2.7046 | LR: 3.86e-04 | 4012.9ms/step | 15.0 samples/s
428
+ Step 16530/50462 | Loss: 2.6725 | LR: 3.86e-04 | 3998.1ms/step | 15.0 samples/s
429
+ Step 16540/50462 | Loss: 2.7319 | LR: 3.86e-04 | 4002.6ms/step | 15.0 samples/s
430
+ Step 16550/50462 | Loss: 2.6677 | LR: 3.85e-04 | 4001.1ms/step | 15.0 samples/s
431
+ Step 16560/50462 | Loss: 2.6921 | LR: 3.85e-04 | 4009.7ms/step | 15.0 samples/s
432
+ Step 16570/50462 | Loss: 2.7119 | LR: 3.85e-04 | 3998.7ms/step | 15.0 samples/s
433
+ Step 16580/50462 | Loss: 2.6613 | LR: 3.85e-04 | 3998.2ms/step | 15.0 samples/s
434
+ Step 16590/50462 | Loss: 2.6677 | LR: 3.85e-04 | 4000.9ms/step | 15.0 samples/s
435
+ Step 16600/50462 | Loss: 2.6726 | LR: 3.85e-04 | 4002.5ms/step | 15.0 samples/s
436
+ Step 16610/50462 | Loss: 2.6994 | LR: 3.85e-04 | 4010.3ms/step | 15.0 samples/s
437
+ Step 16620/50462 | Loss: 2.6521 | LR: 3.84e-04 | 4004.5ms/step | 15.0 samples/s
438
+ Step 16630/50462 | Loss: 2.6906 | LR: 3.84e-04 | 4017.5ms/step | 14.9 samples/s
439
+ Step 16640/50462 | Loss: 2.6966 | LR: 3.84e-04 | 4006.7ms/step | 15.0 samples/s
440
+ Step 16650/50462 | Loss: 2.6711 | LR: 3.84e-04 | 3999.7ms/step | 15.0 samples/s
441
+ Step 16660/50462 | Loss: 2.6851 | LR: 3.84e-04 | 4006.7ms/step | 15.0 samples/s
442
+ Step 16670/50462 | Loss: 2.6805 | LR: 3.84e-04 | 4002.6ms/step | 15.0 samples/s
443
+ Step 16680/50462 | Loss: 2.6714 | LR: 3.84e-04 | 4012.6ms/step | 15.0 samples/s
444
+ Step 16690/50462 | Loss: 2.6885 | LR: 3.84e-04 | 4004.5ms/step | 15.0 samples/s
445
+ Step 16700/50462 | Loss: 2.6742 | LR: 3.83e-04 | 4004.4ms/step | 15.0 samples/s
446
+ Step 16710/50462 | Loss: 2.7379 | LR: 3.83e-04 | 3996.4ms/step | 15.0 samples/s
447
+ Step 16720/50462 | Loss: 2.7189 | LR: 3.83e-04 | 3997.8ms/step | 15.0 samples/s
448
+ Step 16730/50462 | Loss: 2.6946 | LR: 3.83e-04 | 4006.0ms/step | 15.0 samples/s
449
+ Step 16740/50462 | Loss: 2.6463 | LR: 3.83e-04 | 4013.1ms/step | 15.0 samples/s
450
+ Step 16750/50462 | Loss: 2.6776 | LR: 3.83e-04 | 3997.9ms/step | 15.0 samples/s
451
+ Validation at step 16750: Loss = 2.7005, Perplexity = 14.89
452
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-16750
453
+ Step 16760/50462 | Loss: 2.7002 | LR: 3.83e-04 | 5958.1ms/step | 10.1 samples/s
454
+ Step 16770/50462 | Loss: 2.6812 | LR: 3.82e-04 | 3997.2ms/step | 15.0 samples/s
455
+ Step 16780/50462 | Loss: 2.6598 | LR: 3.82e-04 | 4004.6ms/step | 15.0 samples/s
456
+ Step 16790/50462 | Loss: 2.6955 | LR: 3.82e-04 | 4004.0ms/step | 15.0 samples/s
457
+ Step 16800/50462 | Loss: 2.6777 | LR: 3.82e-04 | 3998.7ms/step | 15.0 samples/s
458
+ Step 16810/50462 | Loss: 2.6975 | LR: 3.82e-04 | 4001.3ms/step | 15.0 samples/s
459
+ Step 16820/50462 | Loss: 2.7005 | LR: 3.82e-04 | 3995.5ms/step | 15.0 samples/s
460
+ Step 16830/50462 | Loss: 2.7094 | LR: 3.82e-04 | 3995.0ms/step | 15.0 samples/s
461
+ Step 16840/50462 | Loss: 2.6376 | LR: 3.82e-04 | 4006.6ms/step | 15.0 samples/s
462
+ Step 16850/50462 | Loss: 2.6664 | LR: 3.81e-04 | 4009.7ms/step | 15.0 samples/s
463
+ Step 16860/50462 | Loss: 2.6472 | LR: 3.81e-04 | 3996.9ms/step | 15.0 samples/s
464
+ Step 16870/50462 | Loss: 2.6728 | LR: 3.81e-04 | 4014.1ms/step | 14.9 samples/s
465
+ Step 16880/50462 | Loss: 2.7066 | LR: 3.81e-04 | 3993.4ms/step | 15.0 samples/s
466
+ Step 16890/50462 | Loss: 2.6790 | LR: 3.81e-04 | 3990.8ms/step | 15.0 samples/s
467
+ Step 16900/50462 | Loss: 2.7042 | LR: 3.81e-04 | 3989.7ms/step | 15.0 samples/s
468
+ Step 16910/50462 | Loss: 2.7137 | LR: 3.81e-04 | 4002.4ms/step | 15.0 samples/s
469
+ Step 16920/50462 | Loss: 2.6579 | LR: 3.80e-04 | 4005.0ms/step | 15.0 samples/s
470
+ Step 16930/50462 | Loss: 2.6699 | LR: 3.80e-04 | 3998.3ms/step | 15.0 samples/s
471
+ Step 16940/50462 | Loss: 2.7022 | LR: 3.80e-04 | 4007.1ms/step | 15.0 samples/s
472
+ Step 16950/50462 | Loss: 2.6715 | LR: 3.80e-04 | 4001.5ms/step | 15.0 samples/s
473
+ Step 16960/50462 | Loss: 2.7294 | LR: 3.80e-04 | 4003.5ms/step | 15.0 samples/s
474
+ Step 16970/50462 | Loss: 2.6984 | LR: 3.80e-04 | 3997.7ms/step | 15.0 samples/s
475
+ Step 16980/50462 | Loss: 2.6808 | LR: 3.80e-04 | 3998.6ms/step | 15.0 samples/s
476
+ Step 16990/50462 | Loss: 2.6577 | LR: 3.79e-04 | 3995.9ms/step | 15.0 samples/s
477
+ Step 17000/50462 | Loss: 2.7006 | LR: 3.79e-04 | 4000.8ms/step | 15.0 samples/s
478
+ Validation at step 17000: Loss = 2.6973, Perplexity = 14.84
479
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-17000
480
+ Step 17010/50462 | Loss: 2.6568 | LR: 3.79e-04 | 5956.4ms/step | 10.1 samples/s
481
+ Step 17020/50462 | Loss: 2.6753 | LR: 3.79e-04 | 4008.2ms/step | 15.0 samples/s
482
+ Step 17030/50462 | Loss: 2.7091 | LR: 3.79e-04 | 4010.9ms/step | 15.0 samples/s
483
+ Step 17040/50462 | Loss: 2.6488 | LR: 3.79e-04 | 4001.8ms/step | 15.0 samples/s
484
+ Step 17050/50462 | Loss: 2.6620 | LR: 3.79e-04 | 4001.3ms/step | 15.0 samples/s
485
+ Step 17060/50462 | Loss: 2.6457 | LR: 3.79e-04 | 4000.9ms/step | 15.0 samples/s
486
+ Step 17070/50462 | Loss: 2.6778 | LR: 3.78e-04 | 3995.4ms/step | 15.0 samples/s
487
+ Step 17080/50462 | Loss: 2.6816 | LR: 3.78e-04 | 3997.0ms/step | 15.0 samples/s
488
+ Step 17090/50462 | Loss: 2.6860 | LR: 3.78e-04 | 4007.0ms/step | 15.0 samples/s
489
+ Step 17100/50462 | Loss: 2.6305 | LR: 3.78e-04 | 3998.7ms/step | 15.0 samples/s
490
+ Step 17110/50462 | Loss: 2.6647 | LR: 3.78e-04 | 3991.7ms/step | 15.0 samples/s
491
+ Step 17120/50462 | Loss: 2.6579 | LR: 3.78e-04 | 3995.5ms/step | 15.0 samples/s
492
+ Step 17130/50462 | Loss: 2.6697 | LR: 3.78e-04 | 3988.1ms/step | 15.0 samples/s
493
+ Step 17140/50462 | Loss: 2.6537 | LR: 3.77e-04 | 3995.8ms/step | 15.0 samples/s
494
+ Step 17150/50462 | Loss: 2.7189 | LR: 3.77e-04 | 3996.1ms/step | 15.0 samples/s
495
+ Step 17160/50462 | Loss: 2.6665 | LR: 3.77e-04 | 3985.2ms/step | 15.1 samples/s
496
+ Step 17170/50462 | Loss: 2.6668 | LR: 3.77e-04 | 4001.6ms/step | 15.0 samples/s
497
+ Step 17180/50462 | Loss: 2.6663 | LR: 3.77e-04 | 3989.1ms/step | 15.0 samples/s
498
+ Step 17190/50462 | Loss: 2.6568 | LR: 3.77e-04 | 4008.3ms/step | 15.0 samples/s
499
+ Step 17200/50462 | Loss: 2.6811 | LR: 3.77e-04 | 4008.4ms/step | 15.0 samples/s
500
+ Step 17210/50462 | Loss: 2.6844 | LR: 3.77e-04 | 4014.3ms/step | 14.9 samples/s
501
+ Step 17220/50462 | Loss: 2.6352 | LR: 3.76e-04 | 4002.5ms/step | 15.0 samples/s
502
+ Step 17230/50462 | Loss: 2.6169 | LR: 3.76e-04 | 4000.2ms/step | 15.0 samples/s
503
+ Step 17240/50462 | Loss: 2.6215 | LR: 3.76e-04 | 4010.1ms/step | 15.0 samples/s
504
+ Step 17250/50462 | Loss: 2.6859 | LR: 3.76e-04 | 3998.8ms/step | 15.0 samples/s
505
+ Validation at step 17250: Loss = 2.6957, Perplexity = 14.82
506
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-17250
507
+ Step 17260/50462 | Loss: 2.6604 | LR: 3.76e-04 | 5950.6ms/step | 10.1 samples/s
508
+ Step 17270/50462 | Loss: 2.6364 | LR: 3.76e-04 | 3998.6ms/step | 15.0 samples/s
509
+ Step 17280/50462 | Loss: 2.6862 | LR: 3.76e-04 | 3996.3ms/step | 15.0 samples/s
510
+ Step 17290/50462 | Loss: 2.6836 | LR: 3.75e-04 | 4009.7ms/step | 15.0 samples/s
511
+ Step 17300/50462 | Loss: 2.6791 | LR: 3.75e-04 | 3987.8ms/step | 15.0 samples/s
512
+ Step 17310/50462 | Loss: 2.6726 | LR: 3.75e-04 | 3993.1ms/step | 15.0 samples/s
513
+ Step 17320/50462 | Loss: 2.6638 | LR: 3.75e-04 | 3986.4ms/step | 15.1 samples/s
514
+ Step 17330/50462 | Loss: 2.6800 | LR: 3.75e-04 | 3989.9ms/step | 15.0 samples/s
515
+ Step 17340/50462 | Loss: 2.6341 | LR: 3.75e-04 | 4000.7ms/step | 15.0 samples/s
516
+ Step 17350/50462 | Loss: 2.6478 | LR: 3.75e-04 | 4003.2ms/step | 15.0 samples/s
517
+ Step 17360/50462 | Loss: 2.6689 | LR: 3.74e-04 | 4000.7ms/step | 15.0 samples/s
518
+ Step 17370/50462 | Loss: 2.6785 | LR: 3.74e-04 | 3996.3ms/step | 15.0 samples/s
519
+ Step 17380/50462 | Loss: 2.7064 | LR: 3.74e-04 | 4003.0ms/step | 15.0 samples/s
520
+ Step 17390/50462 | Loss: 2.6944 | LR: 3.74e-04 | 3988.2ms/step | 15.0 samples/s
521
+ Step 17400/50462 | Loss: 2.6056 | LR: 3.74e-04 | 3998.0ms/step | 15.0 samples/s
522
+ Step 17410/50462 | Loss: 2.6711 | LR: 3.74e-04 | 4001.4ms/step | 15.0 samples/s
523
+ Step 17420/50462 | Loss: 2.6917 | LR: 3.74e-04 | 4002.2ms/step | 15.0 samples/s
524
+ Step 17430/50462 | Loss: 2.6581 | LR: 3.74e-04 | 3995.4ms/step | 15.0 samples/s
525
+ Step 17440/50462 | Loss: 2.6632 | LR: 3.73e-04 | 3998.8ms/step | 15.0 samples/s
526
+ Step 17450/50462 | Loss: 2.6826 | LR: 3.73e-04 | 3995.1ms/step | 15.0 samples/s
527
+ Step 17460/50462 | Loss: 2.6484 | LR: 3.73e-04 | 3997.1ms/step | 15.0 samples/s
528
+ Step 17470/50462 | Loss: 2.6352 | LR: 3.73e-04 | 4009.3ms/step | 15.0 samples/s
529
+ Step 17480/50462 | Loss: 2.6596 | LR: 3.73e-04 | 4004.4ms/step | 15.0 samples/s
530
+ Step 17490/50462 | Loss: 2.6638 | LR: 3.73e-04 | 3996.8ms/step | 15.0 samples/s
531
+ Step 17500/50462 | Loss: 2.6902 | LR: 3.73e-04 | 3999.5ms/step | 15.0 samples/s
532
+ Validation at step 17500: Loss = 2.6923, Perplexity = 14.77
533
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-17500
534
+ Step 17510/50462 | Loss: 2.6864 | LR: 3.72e-04 | 5952.3ms/step | 10.1 samples/s
535
+ Step 17520/50462 | Loss: 2.6873 | LR: 3.72e-04 | 3993.6ms/step | 15.0 samples/s
536
+ Step 17530/50462 | Loss: 2.6844 | LR: 3.72e-04 | 4005.8ms/step | 15.0 samples/s
537
+ Step 17540/50462 | Loss: 2.7036 | LR: 3.72e-04 | 3998.7ms/step | 15.0 samples/s
538
+ Step 17550/50462 | Loss: 2.6538 | LR: 3.72e-04 | 4006.6ms/step | 15.0 samples/s
539
+ Step 17560/50462 | Loss: 2.6639 | LR: 3.72e-04 | 3999.9ms/step | 15.0 samples/s
540
+ Step 17570/50462 | Loss: 2.6558 | LR: 3.72e-04 | 4006.3ms/step | 15.0 samples/s
541
+ Step 17580/50462 | Loss: 2.6353 | LR: 3.71e-04 | 4000.6ms/step | 15.0 samples/s
542
+ Step 17590/50462 | Loss: 2.7124 | LR: 3.71e-04 | 4013.4ms/step | 14.9 samples/s
543
+ Step 17600/50462 | Loss: 2.6775 | LR: 3.71e-04 | 4008.6ms/step | 15.0 samples/s
544
+ Step 17610/50462 | Loss: 2.6747 | LR: 3.71e-04 | 4010.9ms/step | 15.0 samples/s
545
+ Step 17620/50462 | Loss: 2.6768 | LR: 3.71e-04 | 4002.0ms/step | 15.0 samples/s
546
+ Step 17630/50462 | Loss: 2.6851 | LR: 3.71e-04 | 4001.0ms/step | 15.0 samples/s
547
+ Step 17640/50462 | Loss: 2.6842 | LR: 3.71e-04 | 4007.4ms/step | 15.0 samples/s
548
+ Step 17650/50462 | Loss: 2.6638 | LR: 3.70e-04 | 4008.3ms/step | 15.0 samples/s
549
+ Step 17660/50462 | Loss: 2.6558 | LR: 3.70e-04 | 4001.9ms/step | 15.0 samples/s
550
+ Step 17670/50462 | Loss: 2.6426 | LR: 3.70e-04 | 4005.3ms/step | 15.0 samples/s
551
+ Step 17680/50462 | Loss: 2.6422 | LR: 3.70e-04 | 4016.7ms/step | 14.9 samples/s
552
+ Step 17690/50462 | Loss: 2.6680 | LR: 3.70e-04 | 4007.9ms/step | 15.0 samples/s
553
+ Step 17700/50462 | Loss: 2.6803 | LR: 3.70e-04 | 4004.2ms/step | 15.0 samples/s
554
+ Step 17710/50462 | Loss: 2.6786 | LR: 3.70e-04 | 4000.2ms/step | 15.0 samples/s
555
+ Step 17720/50462 | Loss: 2.6717 | LR: 3.69e-04 | 4008.5ms/step | 15.0 samples/s
556
+ Step 17730/50462 | Loss: 2.6446 | LR: 3.69e-04 | 4003.4ms/step | 15.0 samples/s
557
+ Step 17740/50462 | Loss: 2.6584 | LR: 3.69e-04 | 4010.2ms/step | 15.0 samples/s
558
+ Step 17750/50462 | Loss: 2.6746 | LR: 3.69e-04 | 3996.9ms/step | 15.0 samples/s
559
+ Validation at step 17750: Loss = 2.6887, Perplexity = 14.71
560
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-17750
561
+ Step 17760/50462 | Loss: 2.6620 | LR: 3.69e-04 | 5939.6ms/step | 10.1 samples/s
562
+ Step 17770/50462 | Loss: 2.6582 | LR: 3.69e-04 | 4008.4ms/step | 15.0 samples/s
563
+ Step 17780/50462 | Loss: 2.6950 | LR: 3.69e-04 | 4003.9ms/step | 15.0 samples/s
564
+ Step 17790/50462 | Loss: 2.6690 | LR: 3.69e-04 | 3999.2ms/step | 15.0 samples/s
565
+ Step 17800/50462 | Loss: 2.6893 | LR: 3.68e-04 | 4001.9ms/step | 15.0 samples/s
566
+ Step 17810/50462 | Loss: 2.6928 | LR: 3.68e-04 | 4016.9ms/step | 14.9 samples/s
567
+ Step 17820/50462 | Loss: 2.6590 | LR: 3.68e-04 | 4007.5ms/step | 15.0 samples/s
568
+ Step 17830/50462 | Loss: 2.6807 | LR: 3.68e-04 | 4013.5ms/step | 14.9 samples/s
569
+ Step 17840/50462 | Loss: 2.6754 | LR: 3.68e-04 | 3996.6ms/step | 15.0 samples/s
570
+ Step 17850/50462 | Loss: 2.6622 | LR: 3.68e-04 | 4000.9ms/step | 15.0 samples/s
571
+ Step 17860/50462 | Loss: 2.6735 | LR: 3.68e-04 | 4002.5ms/step | 15.0 samples/s
572
+ Step 17870/50462 | Loss: 2.6544 | LR: 3.67e-04 | 4009.8ms/step | 15.0 samples/s
573
+ Step 17880/50462 | Loss: 2.6668 | LR: 3.67e-04 | 4007.3ms/step | 15.0 samples/s
574
+ Step 17890/50462 | Loss: 2.6957 | LR: 3.67e-04 | 4005.8ms/step | 15.0 samples/s
575
+ Step 17900/50462 | Loss: 2.6566 | LR: 3.67e-04 | 4007.0ms/step | 15.0 samples/s
576
+ Step 17910/50462 | Loss: 2.6326 | LR: 3.67e-04 | 4002.4ms/step | 15.0 samples/s
577
+ Step 17920/50462 | Loss: 2.6632 | LR: 3.67e-04 | 3995.1ms/step | 15.0 samples/s
578
+ Step 17930/50462 | Loss: 2.6419 | LR: 3.67e-04 | 4008.0ms/step | 15.0 samples/s
579
+ Step 17940/50462 | Loss: 2.6567 | LR: 3.66e-04 | 4005.1ms/step | 15.0 samples/s
580
+ Step 17950/50462 | Loss: 2.7009 | LR: 3.66e-04 | 4001.4ms/step | 15.0 samples/s
581
+ Step 17960/50462 | Loss: 2.6663 | LR: 3.66e-04 | 4019.8ms/step | 14.9 samples/s
582
+ Step 17970/50462 | Loss: 2.6943 | LR: 3.66e-04 | 4002.4ms/step | 15.0 samples/s
583
+ Step 17980/50462 | Loss: 2.6419 | LR: 3.66e-04 | 3995.9ms/step | 15.0 samples/s
584
+ Step 17990/50462 | Loss: 2.6585 | LR: 3.66e-04 | 3995.7ms/step | 15.0 samples/s
585
+ Step 18000/50462 | Loss: 2.6245 | LR: 3.66e-04 | 4005.7ms/step | 15.0 samples/s
586
+ Validation at step 18000: Loss = 2.6858, Perplexity = 14.67
587
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-18000
588
+ Step 18010/50462 | Loss: 2.6801 | LR: 3.65e-04 | 5935.3ms/step | 10.1 samples/s
589
+ Step 18020/50462 | Loss: 2.6478 | LR: 3.65e-04 | 3990.1ms/step | 15.0 samples/s
590
+ Step 18030/50462 | Loss: 2.6509 | LR: 3.65e-04 | 4008.5ms/step | 15.0 samples/s
591
+ Step 18040/50462 | Loss: 2.6736 | LR: 3.65e-04 | 4007.2ms/step | 15.0 samples/s
592
+ Step 18050/50462 | Loss: 2.6733 | LR: 3.65e-04 | 4005.9ms/step | 15.0 samples/s
593
+ Step 18060/50462 | Loss: 2.6672 | LR: 3.65e-04 | 4002.0ms/step | 15.0 samples/s
594
+ Step 18070/50462 | Loss: 2.6884 | LR: 3.65e-04 | 4004.4ms/step | 15.0 samples/s
595
+ Step 18080/50462 | Loss: 2.6645 | LR: 3.64e-04 | 4008.4ms/step | 15.0 samples/s
596
+ Step 18090/50462 | Loss: 2.6779 | LR: 3.64e-04 | 4000.9ms/step | 15.0 samples/s
597
+ Step 18100/50462 | Loss: 2.6850 | LR: 3.64e-04 | 4002.6ms/step | 15.0 samples/s
598
+ Step 18110/50462 | Loss: 2.6742 | LR: 3.64e-04 | 4004.1ms/step | 15.0 samples/s
599
+ Step 18120/50462 | Loss: 2.6684 | LR: 3.64e-04 | 4001.0ms/step | 15.0 samples/s
600
+ Step 18130/50462 | Loss: 2.6450 | LR: 3.64e-04 | 4005.7ms/step | 15.0 samples/s
601
+ Step 18140/50462 | Loss: 2.6639 | LR: 3.64e-04 | 4014.4ms/step | 14.9 samples/s
602
+ Step 18150/50462 | Loss: 2.6533 | LR: 3.63e-04 | 3997.0ms/step | 15.0 samples/s
603
+ Step 18160/50462 | Loss: 2.6371 | LR: 3.63e-04 | 3994.2ms/step | 15.0 samples/s
604
+ Step 18170/50462 | Loss: 2.6778 | LR: 3.63e-04 | 3995.8ms/step | 15.0 samples/s
605
+ Step 18180/50462 | Loss: 2.6557 | LR: 3.63e-04 | 3997.7ms/step | 15.0 samples/s
606
+ Step 18190/50462 | Loss: 2.6563 | LR: 3.63e-04 | 4003.6ms/step | 15.0 samples/s
607
+ Step 18200/50462 | Loss: 2.6830 | LR: 3.63e-04 | 3996.0ms/step | 15.0 samples/s
608
+ Step 18210/50462 | Loss: 2.6755 | LR: 3.63e-04 | 3999.6ms/step | 15.0 samples/s
609
+ Step 18220/50462 | Loss: 2.7039 | LR: 3.62e-04 | 4002.1ms/step | 15.0 samples/s
610
+ Step 18230/50462 | Loss: 2.6687 | LR: 3.62e-04 | 4008.1ms/step | 15.0 samples/s
611
+ Step 18240/50462 | Loss: 2.6407 | LR: 3.62e-04 | 3996.9ms/step | 15.0 samples/s
612
+ Step 18250/50462 | Loss: 2.6419 | LR: 3.62e-04 | 4007.8ms/step | 15.0 samples/s
613
+ Validation at step 18250: Loss = 2.6844, Perplexity = 14.65
614
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-18250
615
+ Step 18260/50462 | Loss: 2.6370 | LR: 3.62e-04 | 5946.7ms/step | 10.1 samples/s
616
+ Step 18270/50462 | Loss: 2.6649 | LR: 3.62e-04 | 4001.7ms/step | 15.0 samples/s
617
+ Step 18280/50462 | Loss: 2.6425 | LR: 3.62e-04 | 4005.4ms/step | 15.0 samples/s
618
+ Step 18290/50462 | Loss: 2.6713 | LR: 3.62e-04 | 3997.7ms/step | 15.0 samples/s
619
+ Step 18300/50462 | Loss: 2.6454 | LR: 3.61e-04 | 4001.9ms/step | 15.0 samples/s
620
+ Step 18310/50462 | Loss: 2.6764 | LR: 3.61e-04 | 3997.7ms/step | 15.0 samples/s
621
+ Step 18320/50462 | Loss: 2.6700 | LR: 3.61e-04 | 4006.0ms/step | 15.0 samples/s
622
+ Step 18330/50462 | Loss: 2.6804 | LR: 3.61e-04 | 3999.6ms/step | 15.0 samples/s
623
+ Step 18340/50462 | Loss: 2.6617 | LR: 3.61e-04 | 4007.5ms/step | 15.0 samples/s
624
+ Step 18350/50462 | Loss: 2.6611 | LR: 3.61e-04 | 4004.4ms/step | 15.0 samples/s
625
+ Step 18360/50462 | Loss: 2.6658 | LR: 3.61e-04 | 4010.6ms/step | 15.0 samples/s
626
+ Step 18370/50462 | Loss: 2.6419 | LR: 3.60e-04 | 4001.9ms/step | 15.0 samples/s
627
+ Step 18380/50462 | Loss: 2.6540 | LR: 3.60e-04 | 4008.3ms/step | 15.0 samples/s
628
+ Step 18390/50462 | Loss: 2.7048 | LR: 3.60e-04 | 4008.6ms/step | 15.0 samples/s
629
+ Step 18400/50462 | Loss: 2.6575 | LR: 3.60e-04 | 4016.6ms/step | 14.9 samples/s
630
+ Step 18410/50462 | Loss: 2.6435 | LR: 3.60e-04 | 4014.7ms/step | 14.9 samples/s
631
+ Step 18420/50462 | Loss: 2.6424 | LR: 3.60e-04 | 4012.0ms/step | 15.0 samples/s
632
+ Step 18430/50462 | Loss: 2.6533 | LR: 3.60e-04 | 4004.7ms/step | 15.0 samples/s
633
+ Step 18440/50462 | Loss: 2.6647 | LR: 3.59e-04 | 3998.2ms/step | 15.0 samples/s
634
+ Step 18450/50462 | Loss: 2.7072 | LR: 3.59e-04 | 4011.3ms/step | 15.0 samples/s
635
+ Step 18460/50462 | Loss: 2.6790 | LR: 3.59e-04 | 4003.4ms/step | 15.0 samples/s
636
+ Step 18470/50462 | Loss: 2.6778 | LR: 3.59e-04 | 4007.4ms/step | 15.0 samples/s
637
+ Step 18480/50462 | Loss: 2.6676 | LR: 3.59e-04 | 3996.5ms/step | 15.0 samples/s
638
+ Step 18490/50462 | Loss: 2.6544 | LR: 3.59e-04 | 3997.1ms/step | 15.0 samples/s
639
+ Step 18500/50462 | Loss: 2.6601 | LR: 3.59e-04 | 3999.2ms/step | 15.0 samples/s
640
+ Validation at step 18500: Loss = 2.6807, Perplexity = 14.59
641
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-18500
642
+ Step 18510/50462 | Loss: 2.6942 | LR: 3.58e-04 | 5973.6ms/step | 10.0 samples/s
643
+ Step 18520/50462 | Loss: 2.6265 | LR: 3.58e-04 | 3999.8ms/step | 15.0 samples/s
644
+ Step 18530/50462 | Loss: 2.6465 | LR: 3.58e-04 | 3991.4ms/step | 15.0 samples/s
645
+ Step 18540/50462 | Loss: 2.6725 | LR: 3.58e-04 | 4006.9ms/step | 15.0 samples/s
646
+ Step 18550/50462 | Loss: 2.6537 | LR: 3.58e-04 | 4004.1ms/step | 15.0 samples/s
647
+ Step 18560/50462 | Loss: 2.6459 | LR: 3.58e-04 | 3991.4ms/step | 15.0 samples/s
648
+ Step 18570/50462 | Loss: 2.6516 | LR: 3.58e-04 | 4005.0ms/step | 15.0 samples/s
649
+ Step 18580/50462 | Loss: 2.6492 | LR: 3.57e-04 | 3993.7ms/step | 15.0 samples/s
650
+ Step 18590/50462 | Loss: 2.6590 | LR: 3.57e-04 | 3996.0ms/step | 15.0 samples/s
651
+ Step 18600/50462 | Loss: 2.6901 | LR: 3.57e-04 | 4007.4ms/step | 15.0 samples/s
652
+ Step 18610/50462 | Loss: 2.6410 | LR: 3.57e-04 | 3999.9ms/step | 15.0 samples/s
653
+ Step 18620/50462 | Loss: 2.6458 | LR: 3.57e-04 | 3995.3ms/step | 15.0 samples/s
654
+ Step 18630/50462 | Loss: 2.6670 | LR: 3.57e-04 | 4002.1ms/step | 15.0 samples/s
655
+ Step 18640/50462 | Loss: 2.6625 | LR: 3.57e-04 | 4008.0ms/step | 15.0 samples/s
656
+ Step 18650/50462 | Loss: 2.6819 | LR: 3.56e-04 | 4003.1ms/step | 15.0 samples/s
657
+ Step 18660/50462 | Loss: 2.6678 | LR: 3.56e-04 | 4005.6ms/step | 15.0 samples/s
658
+ Step 18670/50462 | Loss: 2.6484 | LR: 3.56e-04 | 4004.6ms/step | 15.0 samples/s
659
+ Step 18680/50462 | Loss: 2.6623 | LR: 3.56e-04 | 4006.5ms/step | 15.0 samples/s
660
+ Step 18690/50462 | Loss: 2.6467 | LR: 3.56e-04 | 4003.5ms/step | 15.0 samples/s
661
+ Step 18700/50462 | Loss: 2.6393 | LR: 3.56e-04 | 3996.7ms/step | 15.0 samples/s
662
+ Step 18710/50462 | Loss: 2.6734 | LR: 3.56e-04 | 3999.1ms/step | 15.0 samples/s
663
+ Step 18720/50462 | Loss: 2.6708 | LR: 3.55e-04 | 4002.0ms/step | 15.0 samples/s
664
+ Step 18730/50462 | Loss: 2.6421 | LR: 3.55e-04 | 4004.2ms/step | 15.0 samples/s
665
+ Step 18740/50462 | Loss: 2.6579 | LR: 3.55e-04 | 4006.3ms/step | 15.0 samples/s
666
+ Step 18750/50462 | Loss: 2.6501 | LR: 3.55e-04 | 4003.3ms/step | 15.0 samples/s
667
+ Validation at step 18750: Loss = 2.6801, Perplexity = 14.59
668
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-18750
669
+ Step 18760/50462 | Loss: 2.6960 | LR: 3.55e-04 | 5960.4ms/step | 10.1 samples/s
670
+ Step 18770/50462 | Loss: 2.6667 | LR: 3.55e-04 | 4000.3ms/step | 15.0 samples/s
671
+ Step 18780/50462 | Loss: 2.6668 | LR: 3.55e-04 | 4009.2ms/step | 15.0 samples/s
672
+ Step 18790/50462 | Loss: 2.6423 | LR: 3.54e-04 | 4005.1ms/step | 15.0 samples/s
673
+ Step 18800/50462 | Loss: 2.6827 | LR: 3.54e-04 | 4003.8ms/step | 15.0 samples/s
674
+ Step 18810/50462 | Loss: 2.6730 | LR: 3.54e-04 | 4008.5ms/step | 15.0 samples/s
675
+ Step 18820/50462 | Loss: 2.6526 | LR: 3.54e-04 | 4009.4ms/step | 15.0 samples/s
676
+ Step 18830/50462 | Loss: 2.6389 | LR: 3.54e-04 | 4003.1ms/step | 15.0 samples/s
677
+ Step 18840/50462 | Loss: 2.6420 | LR: 3.54e-04 | 3995.8ms/step | 15.0 samples/s
678
+ Step 18850/50462 | Loss: 2.6694 | LR: 3.54e-04 | 4010.1ms/step | 15.0 samples/s
679
+ Step 18860/50462 | Loss: 2.6574 | LR: 3.53e-04 | 4013.1ms/step | 15.0 samples/s
680
+ Step 18870/50462 | Loss: 2.6471 | LR: 3.53e-04 | 3995.8ms/step | 15.0 samples/s
681
+ Step 18880/50462 | Loss: 2.6683 | LR: 3.53e-04 | 4012.9ms/step | 15.0 samples/s
682
+ Step 18890/50462 | Loss: 2.6929 | LR: 3.53e-04 | 4010.4ms/step | 15.0 samples/s
683
+ Step 18900/50462 | Loss: 2.6629 | LR: 3.53e-04 | 4004.9ms/step | 15.0 samples/s
684
+ Step 18910/50462 | Loss: 2.6520 | LR: 3.53e-04 | 4000.8ms/step | 15.0 samples/s
685
+ Step 18920/50462 | Loss: 2.6658 | LR: 3.53e-04 | 4011.7ms/step | 15.0 samples/s
686
+ Step 18930/50462 | Loss: 2.6847 | LR: 3.52e-04 | 4007.6ms/step | 15.0 samples/s
687
+ Step 18940/50462 | Loss: 2.6476 | LR: 3.52e-04 | 3985.2ms/step | 15.1 samples/s
688
+ Step 18950/50462 | Loss: 2.6610 | LR: 3.52e-04 | 3993.8ms/step | 15.0 samples/s
689
+ Step 18960/50462 | Loss: 2.6644 | LR: 3.52e-04 | 4007.6ms/step | 15.0 samples/s
690
+ Step 18970/50462 | Loss: 2.6739 | LR: 3.52e-04 | 4010.5ms/step | 15.0 samples/s
691
+ Step 18980/50462 | Loss: 2.6723 | LR: 3.52e-04 | 3998.6ms/step | 15.0 samples/s
692
+ Step 18990/50462 | Loss: 2.6539 | LR: 3.52e-04 | 4001.1ms/step | 15.0 samples/s
693
+ Step 19000/50462 | Loss: 2.6256 | LR: 3.51e-04 | 4007.1ms/step | 15.0 samples/s
694
+ Validation at step 19000: Loss = 2.6756, Perplexity = 14.52
695
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-19000
696
+ Step 19010/50462 | Loss: 2.6790 | LR: 3.51e-04 | 5973.2ms/step | 10.0 samples/s
697
+ Step 19020/50462 | Loss: 2.6541 | LR: 3.51e-04 | 3999.8ms/step | 15.0 samples/s
698
+ Step 19030/50462 | Loss: 2.6458 | LR: 3.51e-04 | 3991.4ms/step | 15.0 samples/s
699
+ Step 19040/50462 | Loss: 2.6665 | LR: 3.51e-04 | 3999.5ms/step | 15.0 samples/s
700
+ Step 19050/50462 | Loss: 2.6441 | LR: 3.51e-04 | 4015.5ms/step | 14.9 samples/s
701
+ Step 19060/50462 | Loss: 2.6624 | LR: 3.50e-04 | 4009.4ms/step | 15.0 samples/s
702
+ Step 19070/50462 | Loss: 2.6496 | LR: 3.50e-04 | 4002.7ms/step | 15.0 samples/s
703
+ Step 19080/50462 | Loss: 2.6674 | LR: 3.50e-04 | 4006.2ms/step | 15.0 samples/s
704
+ Step 19090/50462 | Loss: 2.6504 | LR: 3.50e-04 | 3993.5ms/step | 15.0 samples/s
705
+ Step 19100/50462 | Loss: 2.6479 | LR: 3.50e-04 | 3999.8ms/step | 15.0 samples/s
706
+ Step 19110/50462 | Loss: 2.6442 | LR: 3.50e-04 | 4004.9ms/step | 15.0 samples/s
707
+ Step 19120/50462 | Loss: 2.6102 | LR: 3.50e-04 | 3998.8ms/step | 15.0 samples/s
708
+ Step 19130/50462 | Loss: 2.6347 | LR: 3.49e-04 | 3996.1ms/step | 15.0 samples/s
709
+ Step 19140/50462 | Loss: 2.6667 | LR: 3.49e-04 | 3999.7ms/step | 15.0 samples/s
710
+ Step 19150/50462 | Loss: 2.6193 | LR: 3.49e-04 | 3998.5ms/step | 15.0 samples/s
711
+ Step 19160/50462 | Loss: 2.6156 | LR: 3.49e-04 | 4009.1ms/step | 15.0 samples/s
712
+ Step 19170/50462 | Loss: 2.6187 | LR: 3.49e-04 | 3995.2ms/step | 15.0 samples/s
713
+ Step 19180/50462 | Loss: 2.6396 | LR: 3.49e-04 | 4017.1ms/step | 14.9 samples/s
714
+ Step 19190/50462 | Loss: 2.6535 | LR: 3.49e-04 | 4001.5ms/step | 15.0 samples/s
715
+ Step 19200/50462 | Loss: 2.6557 | LR: 3.48e-04 | 4000.7ms/step | 15.0 samples/s
716
+ Step 19210/50462 | Loss: 2.6191 | LR: 3.48e-04 | 4008.9ms/step | 15.0 samples/s
717
+ Step 19220/50462 | Loss: 2.6222 | LR: 3.48e-04 | 4015.4ms/step | 14.9 samples/s
718
+ Step 19230/50462 | Loss: 2.6601 | LR: 3.48e-04 | 4002.1ms/step | 15.0 samples/s
719
+ Step 19240/50462 | Loss: 2.6335 | LR: 3.48e-04 | 4003.5ms/step | 15.0 samples/s
720
+ Step 19250/50462 | Loss: 2.6601 | LR: 3.48e-04 | 4008.6ms/step | 15.0 samples/s
721
+ Validation at step 19250: Loss = 2.6708, Perplexity = 14.45
722
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-19250
723
+ Step 19260/50462 | Loss: 2.6822 | LR: 3.48e-04 | 5945.3ms/step | 10.1 samples/s
724
+ Step 19270/50462 | Loss: 2.6450 | LR: 3.47e-04 | 4008.4ms/step | 15.0 samples/s
725
+ Step 19280/50462 | Loss: 2.6366 | LR: 3.47e-04 | 4011.1ms/step | 15.0 samples/s
726
+ Step 19290/50462 | Loss: 2.6327 | LR: 3.47e-04 | 4006.1ms/step | 15.0 samples/s
727
+ Step 19300/50462 | Loss: 2.6539 | LR: 3.47e-04 | 4013.7ms/step | 14.9 samples/s
728
+ Step 19310/50462 | Loss: 2.6160 | LR: 3.47e-04 | 3994.9ms/step | 15.0 samples/s
729
+ Step 19320/50462 | Loss: 2.6373 | LR: 3.47e-04 | 4003.1ms/step | 15.0 samples/s
730
+ Step 19330/50462 | Loss: 2.6368 | LR: 3.47e-04 | 4003.5ms/step | 15.0 samples/s
731
+ Step 19340/50462 | Loss: 2.6454 | LR: 3.46e-04 | 3992.8ms/step | 15.0 samples/s
732
+ Step 19350/50462 | Loss: 2.6624 | LR: 3.46e-04 | 3995.2ms/step | 15.0 samples/s
733
+ Step 19360/50462 | Loss: 2.6493 | LR: 3.46e-04 | 4000.6ms/step | 15.0 samples/s
734
+ Step 19370/50462 | Loss: 2.6734 | LR: 3.46e-04 | 4001.8ms/step | 15.0 samples/s
735
+ Step 19380/50462 | Loss: 2.6564 | LR: 3.46e-04 | 3991.1ms/step | 15.0 samples/s
736
+ Step 19390/50462 | Loss: 2.6560 | LR: 3.46e-04 | 3988.8ms/step | 15.0 samples/s
737
+ Step 19400/50462 | Loss: 2.6433 | LR: 3.46e-04 | 4012.5ms/step | 15.0 samples/s
738
+ Step 19410/50462 | Loss: 2.6549 | LR: 3.45e-04 | 4006.3ms/step | 15.0 samples/s
739
+ Step 19420/50462 | Loss: 2.6204 | LR: 3.45e-04 | 3996.0ms/step | 15.0 samples/s
740
+ Step 19430/50462 | Loss: 2.6457 | LR: 3.45e-04 | 4006.5ms/step | 15.0 samples/s
741
+ Step 19440/50462 | Loss: 2.6665 | LR: 3.45e-04 | 4005.7ms/step | 15.0 samples/s
742
+ Step 19450/50462 | Loss: 2.6405 | LR: 3.45e-04 | 4007.3ms/step | 15.0 samples/s
743
+ Step 19460/50462 | Loss: 2.6804 | LR: 3.45e-04 | 4010.9ms/step | 15.0 samples/s
744
+ Step 19470/50462 | Loss: 2.6648 | LR: 3.45e-04 | 4008.8ms/step | 15.0 samples/s
745
+ Step 19480/50462 | Loss: 2.6595 | LR: 3.44e-04 | 3999.9ms/step | 15.0 samples/s
746
+ Step 19490/50462 | Loss: 2.6370 | LR: 3.44e-04 | 3998.5ms/step | 15.0 samples/s
747
+ Step 19500/50462 | Loss: 2.6880 | LR: 3.44e-04 | 4011.4ms/step | 15.0 samples/s
748
+ Validation at step 19500: Loss = 2.6706, Perplexity = 14.45
749
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-19500
750
+ Step 19510/50462 | Loss: 2.6375 | LR: 3.44e-04 | 5960.0ms/step | 10.1 samples/s
751
+ Step 19520/50462 | Loss: 2.6721 | LR: 3.44e-04 | 4002.3ms/step | 15.0 samples/s
752
+ Step 19530/50462 | Loss: 2.6598 | LR: 3.44e-04 | 4007.1ms/step | 15.0 samples/s
753
+ Step 19540/50462 | Loss: 2.6564 | LR: 3.44e-04 | 4001.9ms/step | 15.0 samples/s
754
+ Step 19550/50462 | Loss: 2.6412 | LR: 3.43e-04 | 4003.8ms/step | 15.0 samples/s
755
+ Step 19560/50462 | Loss: 2.6374 | LR: 3.43e-04 | 3990.4ms/step | 15.0 samples/s
756
+ Step 19570/50462 | Loss: 2.6406 | LR: 3.43e-04 | 4006.3ms/step | 15.0 samples/s
757
+ Step 19580/50462 | Loss: 2.6410 | LR: 3.43e-04 | 3998.5ms/step | 15.0 samples/s
758
+ Step 19590/50462 | Loss: 2.6528 | LR: 3.43e-04 | 4018.8ms/step | 14.9 samples/s
759
+ Step 19600/50462 | Loss: 2.6368 | LR: 3.43e-04 | 3997.9ms/step | 15.0 samples/s
760
+ Step 19610/50462 | Loss: 2.6209 | LR: 3.42e-04 | 3997.9ms/step | 15.0 samples/s
761
+ Step 19620/50462 | Loss: 2.6327 | LR: 3.42e-04 | 4001.0ms/step | 15.0 samples/s
762
+ Step 19630/50462 | Loss: 2.6664 | LR: 3.42e-04 | 4001.9ms/step | 15.0 samples/s
763
+ Step 19640/50462 | Loss: 2.6181 | LR: 3.42e-04 | 4009.9ms/step | 15.0 samples/s
764
+ Step 19650/50462 | Loss: 2.6608 | LR: 3.42e-04 | 3997.8ms/step | 15.0 samples/s
765
+ Step 19660/50462 | Loss: 2.6503 | LR: 3.42e-04 | 4013.0ms/step | 15.0 samples/s
766
+ Step 19670/50462 | Loss: 2.6234 | LR: 3.42e-04 | 4006.6ms/step | 15.0 samples/s
767
+ Step 19680/50462 | Loss: 2.6369 | LR: 3.41e-04 | 4008.4ms/step | 15.0 samples/s
768
+ Step 19690/50462 | Loss: 2.6264 | LR: 3.41e-04 | 4006.7ms/step | 15.0 samples/s
769
+ Step 19700/50462 | Loss: 2.6407 | LR: 3.41e-04 | 4014.2ms/step | 14.9 samples/s
770
+ Step 19710/50462 | Loss: 2.6404 | LR: 3.41e-04 | 4007.3ms/step | 15.0 samples/s
771
+ Step 19720/50462 | Loss: 2.6365 | LR: 3.41e-04 | 4003.4ms/step | 15.0 samples/s
772
+ Step 19730/50462 | Loss: 2.6128 | LR: 3.41e-04 | 4009.5ms/step | 15.0 samples/s
773
+ Step 19740/50462 | Loss: 2.6289 | LR: 3.41e-04 | 4003.5ms/step | 15.0 samples/s
774
+ Step 19750/50462 | Loss: 2.6414 | LR: 3.40e-04 | 4002.7ms/step | 15.0 samples/s
775
+ Validation at step 19750: Loss = 2.6669, Perplexity = 14.40
776
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-19750
777
+ Step 19760/50462 | Loss: 2.6424 | LR: 3.40e-04 | 5954.4ms/step | 10.1 samples/s
778
+ Step 19770/50462 | Loss: 2.6420 | LR: 3.40e-04 | 3991.5ms/step | 15.0 samples/s
779
+ Step 19780/50462 | Loss: 2.6410 | LR: 3.40e-04 | 4005.8ms/step | 15.0 samples/s
780
+ Step 19790/50462 | Loss: 2.6211 | LR: 3.40e-04 | 4006.0ms/step | 15.0 samples/s
781
+ Step 19800/50462 | Loss: 2.6371 | LR: 3.40e-04 | 4004.5ms/step | 15.0 samples/s
782
+ Step 19810/50462 | Loss: 2.6587 | LR: 3.40e-04 | 4014.6ms/step | 14.9 samples/s
783
+ Step 19820/50462 | Loss: 2.6279 | LR: 3.39e-04 | 4010.5ms/step | 15.0 samples/s
784
+ Step 19830/50462 | Loss: 2.6487 | LR: 3.39e-04 | 3998.1ms/step | 15.0 samples/s
785
+ Step 19840/50462 | Loss: 2.6374 | LR: 3.39e-04 | 4002.8ms/step | 15.0 samples/s
786
+ Step 19850/50462 | Loss: 2.6494 | LR: 3.39e-04 | 4009.9ms/step | 15.0 samples/s
787
+ Step 19860/50462 | Loss: 2.6369 | LR: 3.39e-04 | 4010.2ms/step | 15.0 samples/s
788
+ Step 19870/50462 | Loss: 2.6609 | LR: 3.39e-04 | 4015.3ms/step | 14.9 samples/s
789
+ Step 19880/50462 | Loss: 2.6789 | LR: 3.39e-04 | 4011.8ms/step | 15.0 samples/s
790
+ Step 19890/50462 | Loss: 2.6626 | LR: 3.38e-04 | 4002.6ms/step | 15.0 samples/s
791
+ Step 19900/50462 | Loss: 2.6726 | LR: 3.38e-04 | 4001.1ms/step | 15.0 samples/s
792
+ Step 19910/50462 | Loss: 2.6345 | LR: 3.38e-04 | 4008.7ms/step | 15.0 samples/s
793
+ Step 19920/50462 | Loss: 2.6591 | LR: 3.38e-04 | 4002.3ms/step | 15.0 samples/s
794
+ Step 19930/50462 | Loss: 2.6479 | LR: 3.38e-04 | 4009.7ms/step | 15.0 samples/s
795
+ Step 19940/50462 | Loss: 2.6370 | LR: 3.38e-04 | 4020.3ms/step | 14.9 samples/s
796
+ Step 19950/50462 | Loss: 2.6254 | LR: 3.37e-04 | 4001.6ms/step | 15.0 samples/s
797
+ Step 19960/50462 | Loss: 2.6536 | LR: 3.37e-04 | 3994.4ms/step | 15.0 samples/s
798
+ Step 19970/50462 | Loss: 2.6330 | LR: 3.37e-04 | 4000.2ms/step | 15.0 samples/s
799
+ Step 19980/50462 | Loss: 2.6676 | LR: 3.37e-04 | 4000.1ms/step | 15.0 samples/s
800
+ Step 19990/50462 | Loss: 2.6593 | LR: 3.37e-04 | 3995.6ms/step | 15.0 samples/s
801
+ Step 20000/50462 | Loss: 2.6128 | LR: 3.37e-04 | 3998.7ms/step | 15.0 samples/s
802
+ Validation at step 20000: Loss = 2.6649, Perplexity = 14.37
803
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-20000
804
+ Step 20010/50462 | Loss: 2.6354 | LR: 3.37e-04 | 5984.9ms/step | 10.0 samples/s
805
+ Step 20020/50462 | Loss: 2.6230 | LR: 3.36e-04 | 4001.2ms/step | 15.0 samples/s
806
+ Step 20030/50462 | Loss: 2.6261 | LR: 3.36e-04 | 4016.5ms/step | 14.9 samples/s
807
+ Step 20040/50462 | Loss: 2.6341 | LR: 3.36e-04 | 4012.6ms/step | 15.0 samples/s
808
+ Step 20050/50462 | Loss: 2.6164 | LR: 3.36e-04 | 4012.4ms/step | 15.0 samples/s
809
+ Step 20060/50462 | Loss: 2.6517 | LR: 3.36e-04 | 4005.9ms/step | 15.0 samples/s
810
+ Step 20070/50462 | Loss: 2.6912 | LR: 3.36e-04 | 4003.5ms/step | 15.0 samples/s
811
+ Step 20080/50462 | Loss: 2.6404 | LR: 3.36e-04 | 4003.3ms/step | 15.0 samples/s
812
+ Step 20090/50462 | Loss: 2.6747 | LR: 3.35e-04 | 4000.6ms/step | 15.0 samples/s
813
+ Step 20100/50462 | Loss: 2.6467 | LR: 3.35e-04 | 4014.3ms/step | 14.9 samples/s
814
+ Step 20110/50462 | Loss: 2.6303 | LR: 3.35e-04 | 4002.7ms/step | 15.0 samples/s
815
+ Step 20120/50462 | Loss: 2.6492 | LR: 3.35e-04 | 4011.0ms/step | 15.0 samples/s
816
+ Step 20130/50462 | Loss: 2.6266 | LR: 3.35e-04 | 4001.1ms/step | 15.0 samples/s
817
+ Step 20140/50462 | Loss: 2.6636 | LR: 3.35e-04 | 4004.7ms/step | 15.0 samples/s
818
+ Step 20150/50462 | Loss: 2.6802 | LR: 3.35e-04 | 4016.6ms/step | 14.9 samples/s
819
+ Step 20160/50462 | Loss: 2.6423 | LR: 3.34e-04 | 4015.2ms/step | 14.9 samples/s
820
+ Step 20170/50462 | Loss: 2.6691 | LR: 3.34e-04 | 3999.7ms/step | 15.0 samples/s
821
+ Step 20180/50462 | Loss: 2.6367 | LR: 3.34e-04 | 4000.4ms/step | 15.0 samples/s
822
+ Step 20190/50462 | Loss: 2.6384 | LR: 3.34e-04 | 3993.5ms/step | 15.0 samples/s
823
+ Step 20200/50462 | Loss: 2.6213 | LR: 3.34e-04 | 4003.2ms/step | 15.0 samples/s
824
+ Step 20210/50462 | Loss: 2.6964 | LR: 3.34e-04 | 4006.6ms/step | 15.0 samples/s
825
+ Step 20220/50462 | Loss: 2.6396 | LR: 3.33e-04 | 4011.7ms/step | 15.0 samples/s
826
+ Step 20230/50462 | Loss: 2.6412 | LR: 3.33e-04 | 3997.4ms/step | 15.0 samples/s
827
+ Step 20240/50462 | Loss: 2.6271 | LR: 3.33e-04 | 3996.9ms/step | 15.0 samples/s
828
+ Step 20250/50462 | Loss: 2.6348 | LR: 3.33e-04 | 3996.2ms/step | 15.0 samples/s
829
+ Validation at step 20250: Loss = 2.6635, Perplexity = 14.35
830
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-20250
831
+ Step 20260/50462 | Loss: 2.6427 | LR: 3.33e-04 | 5965.3ms/step | 10.1 samples/s
832
+ Step 20270/50462 | Loss: 2.6295 | LR: 3.33e-04 | 4018.4ms/step | 14.9 samples/s
833
+ Step 20280/50462 | Loss: 2.6745 | LR: 3.33e-04 | 4010.0ms/step | 15.0 samples/s
834
+ Step 20290/50462 | Loss: 2.6570 | LR: 3.32e-04 | 4011.0ms/step | 15.0 samples/s
835
+ Step 20300/50462 | Loss: 2.6379 | LR: 3.32e-04 | 3999.9ms/step | 15.0 samples/s
836
+ Step 20310/50462 | Loss: 2.6506 | LR: 3.32e-04 | 4007.7ms/step | 15.0 samples/s
837
+ Step 20320/50462 | Loss: 2.6329 | LR: 3.32e-04 | 3998.9ms/step | 15.0 samples/s
838
+ Step 20330/50462 | Loss: 2.6735 | LR: 3.32e-04 | 3994.8ms/step | 15.0 samples/s
839
+ Step 20340/50462 | Loss: 2.6190 | LR: 3.32e-04 | 4001.6ms/step | 15.0 samples/s
840
+ Step 20350/50462 | Loss: 2.6659 | LR: 3.32e-04 | 4007.8ms/step | 15.0 samples/s
841
+ Step 20360/50462 | Loss: 2.6054 | LR: 3.31e-04 | 4010.1ms/step | 15.0 samples/s
842
+ Step 20370/50462 | Loss: 2.6336 | LR: 3.31e-04 | 4016.0ms/step | 14.9 samples/s
843
+ Step 20380/50462 | Loss: 2.6187 | LR: 3.31e-04 | 4006.4ms/step | 15.0 samples/s
844
+ Step 20390/50462 | Loss: 2.6423 | LR: 3.31e-04 | 3997.4ms/step | 15.0 samples/s
845
+ Step 20400/50462 | Loss: 2.6427 | LR: 3.31e-04 | 4019.4ms/step | 14.9 samples/s
846
+ Step 20410/50462 | Loss: 2.6222 | LR: 3.31e-04 | 4016.9ms/step | 14.9 samples/s
847
+ Step 20420/50462 | Loss: 2.6398 | LR: 3.30e-04 | 4011.1ms/step | 15.0 samples/s
848
+ Step 20430/50462 | Loss: 2.6249 | LR: 3.30e-04 | 4009.8ms/step | 15.0 samples/s
849
+ Step 20440/50462 | Loss: 2.6796 | LR: 3.30e-04 | 3998.3ms/step | 15.0 samples/s
850
+ Step 20450/50462 | Loss: 2.6233 | LR: 3.30e-04 | 4000.1ms/step | 15.0 samples/s
851
+ Step 20460/50462 | Loss: 2.6172 | LR: 3.30e-04 | 4002.6ms/step | 15.0 samples/s
852
+ Step 20470/50462 | Loss: 2.6429 | LR: 3.30e-04 | 4005.1ms/step | 15.0 samples/s
853
+ Step 20480/50462 | Loss: 2.6066 | LR: 3.30e-04 | 4000.9ms/step | 15.0 samples/s
854
+ Step 20490/50462 | Loss: 2.6644 | LR: 3.29e-04 | 3999.5ms/step | 15.0 samples/s
855
+ Step 20500/50462 | Loss: 2.6383 | LR: 3.29e-04 | 4007.9ms/step | 15.0 samples/s
856
+ Validation at step 20500: Loss = 2.6597, Perplexity = 14.29
857
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-20500
858
+ Step 20510/50462 | Loss: 2.6073 | LR: 3.29e-04 | 5974.4ms/step | 10.0 samples/s
859
+ Step 20520/50462 | Loss: 2.6449 | LR: 3.29e-04 | 4001.2ms/step | 15.0 samples/s
860
+ Step 20530/50462 | Loss: 2.6462 | LR: 3.29e-04 | 4010.2ms/step | 15.0 samples/s
861
+ Step 20540/50462 | Loss: 2.6504 | LR: 3.29e-04 | 4014.4ms/step | 14.9 samples/s
862
+ Step 20550/50462 | Loss: 2.6383 | LR: 3.29e-04 | 4003.3ms/step | 15.0 samples/s
863
+ Step 20560/50462 | Loss: 2.6437 | LR: 3.28e-04 | 4010.2ms/step | 15.0 samples/s
864
+ Step 20570/50462 | Loss: 2.6159 | LR: 3.28e-04 | 4010.8ms/step | 15.0 samples/s
865
+ Step 20580/50462 | Loss: 2.6614 | LR: 3.28e-04 | 4007.5ms/step | 15.0 samples/s
866
+ Step 20590/50462 | Loss: 2.6628 | LR: 3.28e-04 | 3999.8ms/step | 15.0 samples/s
867
+ Step 20600/50462 | Loss: 2.6595 | LR: 3.28e-04 | 4017.2ms/step | 14.9 samples/s
868
+ Step 20610/50462 | Loss: 2.6377 | LR: 3.28e-04 | 4014.9ms/step | 14.9 samples/s
869
+ Step 20620/50462 | Loss: 2.6246 | LR: 3.27e-04 | 4002.4ms/step | 15.0 samples/s
870
+ Step 20630/50462 | Loss: 2.6522 | LR: 3.27e-04 | 3998.5ms/step | 15.0 samples/s
871
+ Step 20640/50462 | Loss: 2.6422 | LR: 3.27e-04 | 4008.7ms/step | 15.0 samples/s
872
+ Step 20650/50462 | Loss: 2.6192 | LR: 3.27e-04 | 3998.0ms/step | 15.0 samples/s
873
+ Step 20660/50462 | Loss: 2.6114 | LR: 3.27e-04 | 3999.3ms/step | 15.0 samples/s
874
+ Step 20670/50462 | Loss: 2.6327 | LR: 3.27e-04 | 4012.2ms/step | 15.0 samples/s
875
+ Step 20680/50462 | Loss: 2.6492 | LR: 3.27e-04 | 4007.1ms/step | 15.0 samples/s
876
+ Step 20690/50462 | Loss: 2.6474 | LR: 3.26e-04 | 4019.2ms/step | 14.9 samples/s
877
+ Step 20700/50462 | Loss: 2.6226 | LR: 3.26e-04 | 4007.3ms/step | 15.0 samples/s
878
+ Step 20710/50462 | Loss: 2.6377 | LR: 3.26e-04 | 4003.8ms/step | 15.0 samples/s
879
+ Step 20720/50462 | Loss: 2.6381 | LR: 3.26e-04 | 4006.1ms/step | 15.0 samples/s
880
+ Step 20730/50462 | Loss: 2.6492 | LR: 3.26e-04 | 4010.9ms/step | 15.0 samples/s
881
+ Step 20740/50462 | Loss: 2.6740 | LR: 3.26e-04 | 4015.8ms/step | 14.9 samples/s
882
+ Step 20750/50462 | Loss: 2.6330 | LR: 3.26e-04 | 4007.6ms/step | 15.0 samples/s
883
+ Validation at step 20750: Loss = 2.6571, Perplexity = 14.26
884
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-20750
885
+ Step 20760/50462 | Loss: 2.6495 | LR: 3.25e-04 | 5995.1ms/step | 10.0 samples/s
886
+ Step 20770/50462 | Loss: 2.6452 | LR: 3.25e-04 | 4013.2ms/step | 15.0 samples/s
887
+ Step 20780/50462 | Loss: 2.6477 | LR: 3.25e-04 | 4015.4ms/step | 14.9 samples/s
888
+ Step 20790/50462 | Loss: 2.6483 | LR: 3.25e-04 | 4010.9ms/step | 15.0 samples/s
889
+ Step 20800/50462 | Loss: 2.6369 | LR: 3.25e-04 | 4008.1ms/step | 15.0 samples/s
890
+ Step 20810/50462 | Loss: 2.6143 | LR: 3.25e-04 | 4010.6ms/step | 15.0 samples/s
891
+ Step 20820/50462 | Loss: 2.6290 | LR: 3.24e-04 | 4011.9ms/step | 15.0 samples/s
892
+ Step 20830/50462 | Loss: 2.6276 | LR: 3.24e-04 | 4009.4ms/step | 15.0 samples/s
893
+ Step 20840/50462 | Loss: 2.6317 | LR: 3.24e-04 | 3998.2ms/step | 15.0 samples/s
894
+ Step 20850/50462 | Loss: 2.6649 | LR: 3.24e-04 | 4013.2ms/step | 15.0 samples/s
895
+ Step 20860/50462 | Loss: 2.6567 | LR: 3.24e-04 | 4001.3ms/step | 15.0 samples/s
896
+ Step 20870/50462 | Loss: 2.6315 | LR: 3.24e-04 | 4006.1ms/step | 15.0 samples/s
897
+ Step 20880/50462 | Loss: 2.6331 | LR: 3.24e-04 | 4011.5ms/step | 15.0 samples/s
898
+ Step 20890/50462 | Loss: 2.6351 | LR: 3.23e-04 | 4003.8ms/step | 15.0 samples/s
899
+ Step 20900/50462 | Loss: 2.6551 | LR: 3.23e-04 | 4007.2ms/step | 15.0 samples/s
900
+ Step 20910/50462 | Loss: 2.6499 | LR: 3.23e-04 | 4014.0ms/step | 14.9 samples/s
901
+ Step 20920/50462 | Loss: 2.6091 | LR: 3.23e-04 | 4010.7ms/step | 15.0 samples/s
902
+ Step 20930/50462 | Loss: 2.6454 | LR: 3.23e-04 | 3992.5ms/step | 15.0 samples/s
903
+ Step 20940/50462 | Loss: 2.6154 | LR: 3.23e-04 | 4012.9ms/step | 15.0 samples/s
904
+ Step 20950/50462 | Loss: 2.6719 | LR: 3.23e-04 | 4003.7ms/step | 15.0 samples/s
905
+ Step 20960/50462 | Loss: 2.6467 | LR: 3.22e-04 | 4000.6ms/step | 15.0 samples/s
906
+ Step 20970/50462 | Loss: 2.6633 | LR: 3.22e-04 | 3989.9ms/step | 15.0 samples/s
907
+ Step 20980/50462 | Loss: 2.6406 | LR: 3.22e-04 | 4012.7ms/step | 15.0 samples/s
908
+ Step 20990/50462 | Loss: 2.5894 | LR: 3.22e-04 | 4012.0ms/step | 15.0 samples/s
909
+ Step 21000/50462 | Loss: 2.6307 | LR: 3.22e-04 | 4004.6ms/step | 15.0 samples/s
910
+ Validation at step 21000: Loss = 2.6543, Perplexity = 14.22
911
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-21000
912
+ Step 21010/50462 | Loss: 2.6267 | LR: 3.22e-04 | 5962.1ms/step | 10.1 samples/s
913
+ Step 21020/50462 | Loss: 2.6385 | LR: 3.21e-04 | 4005.9ms/step | 15.0 samples/s
914
+ Step 21030/50462 | Loss: 2.6430 | LR: 3.21e-04 | 3996.0ms/step | 15.0 samples/s
915
+ Step 21040/50462 | Loss: 2.6384 | LR: 3.21e-04 | 4018.2ms/step | 14.9 samples/s
916
+ Step 21050/50462 | Loss: 2.6689 | LR: 3.21e-04 | 4018.9ms/step | 14.9 samples/s
917
+ Step 21060/50462 | Loss: 2.6542 | LR: 3.21e-04 | 4022.4ms/step | 14.9 samples/s
918
+ Step 21070/50462 | Loss: 2.6381 | LR: 3.21e-04 | 4002.8ms/step | 15.0 samples/s
919
+ Step 21080/50462 | Loss: 2.6572 | LR: 3.21e-04 | 3999.1ms/step | 15.0 samples/s
920
+ Step 21090/50462 | Loss: 2.6024 | LR: 3.20e-04 | 4002.3ms/step | 15.0 samples/s
921
+ Step 21100/50462 | Loss: 2.6111 | LR: 3.20e-04 | 4010.6ms/step | 15.0 samples/s
922
+ Step 21110/50462 | Loss: 2.6150 | LR: 3.20e-04 | 4012.9ms/step | 15.0 samples/s
923
+ Step 21120/50462 | Loss: 2.6344 | LR: 3.20e-04 | 4011.7ms/step | 15.0 samples/s
924
+ Step 21130/50462 | Loss: 2.6159 | LR: 3.20e-04 | 4004.3ms/step | 15.0 samples/s
925
+ Step 21140/50462 | Loss: 2.6100 | LR: 3.20e-04 | 4010.1ms/step | 15.0 samples/s
926
+ Step 21150/50462 | Loss: 2.6791 | LR: 3.19e-04 | 3995.5ms/step | 15.0 samples/s
927
+ Step 21160/50462 | Loss: 2.6648 | LR: 3.19e-04 | 4002.0ms/step | 15.0 samples/s
928
+ Step 21170/50462 | Loss: 2.6562 | LR: 3.19e-04 | 3999.1ms/step | 15.0 samples/s
929
+ Step 21180/50462 | Loss: 2.6268 | LR: 3.19e-04 | 4017.4ms/step | 14.9 samples/s
930
+ Step 21190/50462 | Loss: 2.6774 | LR: 3.19e-04 | 4005.9ms/step | 15.0 samples/s
931
+ Step 21200/50462 | Loss: 2.6283 | LR: 3.19e-04 | 4002.5ms/step | 15.0 samples/s
932
+ Step 21210/50462 | Loss: 2.6048 | LR: 3.19e-04 | 4003.7ms/step | 15.0 samples/s
933
+ Step 21220/50462 | Loss: 2.6490 | LR: 3.18e-04 | 3997.6ms/step | 15.0 samples/s
934
+ Step 21230/50462 | Loss: 2.6200 | LR: 3.18e-04 | 3999.4ms/step | 15.0 samples/s
935
+ Step 21240/50462 | Loss: 2.6210 | LR: 3.18e-04 | 4011.5ms/step | 15.0 samples/s
936
+ Step 21250/50462 | Loss: 2.6459 | LR: 3.18e-04 | 4003.3ms/step | 15.0 samples/s
937
+ Validation at step 21250: Loss = 2.6521, Perplexity = 14.18
938
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-21250
939
+ Step 21260/50462 | Loss: 2.6259 | LR: 3.18e-04 | 5966.7ms/step | 10.1 samples/s
940
+ Step 21270/50462 | Loss: 2.6344 | LR: 3.18e-04 | 4004.7ms/step | 15.0 samples/s
941
+ Step 21280/50462 | Loss: 2.6122 | LR: 3.17e-04 | 4014.1ms/step | 14.9 samples/s
942
+ Step 21290/50462 | Loss: 2.5803 | LR: 3.17e-04 | 4001.3ms/step | 15.0 samples/s
943
+ Step 21300/50462 | Loss: 2.6222 | LR: 3.17e-04 | 4001.8ms/step | 15.0 samples/s
944
+ Step 21310/50462 | Loss: 2.6511 | LR: 3.17e-04 | 4004.2ms/step | 15.0 samples/s
945
+ Step 21320/50462 | Loss: 2.6321 | LR: 3.17e-04 | 4006.3ms/step | 15.0 samples/s
946
+ Step 21330/50462 | Loss: 2.6061 | LR: 3.17e-04 | 3999.6ms/step | 15.0 samples/s
947
+ Step 21340/50462 | Loss: 2.6301 | LR: 3.17e-04 | 4001.8ms/step | 15.0 samples/s
948
+ Step 21350/50462 | Loss: 2.6312 | LR: 3.16e-04 | 4008.3ms/step | 15.0 samples/s
949
+ Step 21360/50462 | Loss: 2.6439 | LR: 3.16e-04 | 4004.9ms/step | 15.0 samples/s
950
+ Step 21370/50462 | Loss: 2.6441 | LR: 3.16e-04 | 4017.1ms/step | 14.9 samples/s
951
+ Step 21380/50462 | Loss: 2.6424 | LR: 3.16e-04 | 4001.1ms/step | 15.0 samples/s
952
+ Step 21390/50462 | Loss: 2.6294 | LR: 3.16e-04 | 4010.4ms/step | 15.0 samples/s
953
+ Step 21400/50462 | Loss: 2.6692 | LR: 3.16e-04 | 4001.5ms/step | 15.0 samples/s
954
+ Step 21410/50462 | Loss: 2.6275 | LR: 3.16e-04 | 4014.6ms/step | 14.9 samples/s
955
+ Step 21420/50462 | Loss: 2.6245 | LR: 3.15e-04 | 3999.2ms/step | 15.0 samples/s
956
+ Step 21430/50462 | Loss: 2.6152 | LR: 3.15e-04 | 3998.2ms/step | 15.0 samples/s
957
+ Step 21440/50462 | Loss: 2.6191 | LR: 3.15e-04 | 4003.0ms/step | 15.0 samples/s
958
+ Step 21450/50462 | Loss: 2.6135 | LR: 3.15e-04 | 4001.3ms/step | 15.0 samples/s
959
+ Step 21460/50462 | Loss: 2.6541 | LR: 3.15e-04 | 3996.8ms/step | 15.0 samples/s
960
+ Step 21470/50462 | Loss: 2.6335 | LR: 3.15e-04 | 4006.7ms/step | 15.0 samples/s
961
+ Step 21480/50462 | Loss: 2.6397 | LR: 3.14e-04 | 4012.4ms/step | 15.0 samples/s
962
+ Step 21490/50462 | Loss: 2.5786 | LR: 3.14e-04 | 4007.4ms/step | 15.0 samples/s
963
+ Step 21500/50462 | Loss: 2.6485 | LR: 3.14e-04 | 4005.8ms/step | 15.0 samples/s
964
+ Validation at step 21500: Loss = 2.6479, Perplexity = 14.13
965
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-21500
966
+ Step 21510/50462 | Loss: 2.6672 | LR: 3.14e-04 | 5966.6ms/step | 10.1 samples/s
967
+ Step 21520/50462 | Loss: 2.6242 | LR: 3.14e-04 | 3998.7ms/step | 15.0 samples/s
968
+ Step 21530/50462 | Loss: 2.5860 | LR: 3.14e-04 | 4008.4ms/step | 15.0 samples/s
969
+ Step 21540/50462 | Loss: 2.6400 | LR: 3.14e-04 | 4002.4ms/step | 15.0 samples/s
970
+ Step 21550/50462 | Loss: 2.6261 | LR: 3.13e-04 | 4016.3ms/step | 14.9 samples/s
971
+ Step 21560/50462 | Loss: 2.5982 | LR: 3.13e-04 | 4006.6ms/step | 15.0 samples/s
972
+ Step 21570/50462 | Loss: 2.5984 | LR: 3.13e-04 | 4008.0ms/step | 15.0 samples/s
973
+ Step 21580/50462 | Loss: 2.6052 | LR: 3.13e-04 | 4000.3ms/step | 15.0 samples/s
974
+ Step 21590/50462 | Loss: 2.6095 | LR: 3.13e-04 | 4009.8ms/step | 15.0 samples/s
975
+ Step 21600/50462 | Loss: 2.6514 | LR: 3.13e-04 | 4002.4ms/step | 15.0 samples/s
976
+ Step 21610/50462 | Loss: 2.6292 | LR: 3.12e-04 | 4004.5ms/step | 15.0 samples/s
977
+ Step 21620/50462 | Loss: 2.6378 | LR: 3.12e-04 | 4014.7ms/step | 14.9 samples/s
978
+ Step 21630/50462 | Loss: 2.6525 | LR: 3.12e-04 | 4006.4ms/step | 15.0 samples/s
979
+ Step 21640/50462 | Loss: 2.6274 | LR: 3.12e-04 | 4005.1ms/step | 15.0 samples/s
980
+ Step 21650/50462 | Loss: 2.5873 | LR: 3.12e-04 | 4012.1ms/step | 15.0 samples/s
981
+ Step 21660/50462 | Loss: 2.6309 | LR: 3.12e-04 | 4009.4ms/step | 15.0 samples/s
982
+ Step 21670/50462 | Loss: 2.6316 | LR: 3.12e-04 | 4002.2ms/step | 15.0 samples/s
983
+ Step 21680/50462 | Loss: 2.6125 | LR: 3.11e-04 | 4008.7ms/step | 15.0 samples/s
984
+ Step 21690/50462 | Loss: 2.6919 | LR: 3.11e-04 | 4014.5ms/step | 14.9 samples/s
985
+ Step 21700/50462 | Loss: 2.6551 | LR: 3.11e-04 | 4017.4ms/step | 14.9 samples/s
986
+ Step 21710/50462 | Loss: 2.6606 | LR: 3.11e-04 | 4012.2ms/step | 15.0 samples/s
987
+ Step 21720/50462 | Loss: 2.6456 | LR: 3.11e-04 | 4006.1ms/step | 15.0 samples/s
988
+ Step 21730/50462 | Loss: 2.6390 | LR: 3.11e-04 | 4002.0ms/step | 15.0 samples/s
989
+ Step 21740/50462 | Loss: 2.6200 | LR: 3.10e-04 | 4005.3ms/step | 15.0 samples/s
990
+ Step 21750/50462 | Loss: 2.6005 | LR: 3.10e-04 | 4015.6ms/step | 14.9 samples/s
991
+ Validation at step 21750: Loss = 2.6475, Perplexity = 14.12
992
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-21750
993
+ Step 21760/50462 | Loss: 2.6143 | LR: 3.10e-04 | 5955.5ms/step | 10.1 samples/s
994
+ Step 21770/50462 | Loss: 2.6404 | LR: 3.10e-04 | 3994.4ms/step | 15.0 samples/s
995
+ Step 21780/50462 | Loss: 2.6388 | LR: 3.10e-04 | 4001.9ms/step | 15.0 samples/s
996
+ Step 21790/50462 | Loss: 2.6183 | LR: 3.10e-04 | 4010.0ms/step | 15.0 samples/s
997
+ Step 21800/50462 | Loss: 2.6010 | LR: 3.10e-04 | 4002.6ms/step | 15.0 samples/s
998
+ Step 21810/50462 | Loss: 2.6415 | LR: 3.09e-04 | 3998.2ms/step | 15.0 samples/s
999
+ Step 21820/50462 | Loss: 2.6460 | LR: 3.09e-04 | 4008.8ms/step | 15.0 samples/s
1000
+ Step 21830/50462 | Loss: 2.6497 | LR: 3.09e-04 | 4011.5ms/step | 15.0 samples/s
1001
+ Step 21840/50462 | Loss: 2.6401 | LR: 3.09e-04 | 4017.0ms/step | 14.9 samples/s
1002
+ Step 21850/50462 | Loss: 2.6465 | LR: 3.09e-04 | 4006.3ms/step | 15.0 samples/s
1003
+ Step 21860/50462 | Loss: 2.5876 | LR: 3.09e-04 | 4005.9ms/step | 15.0 samples/s
1004
+ Step 21870/50462 | Loss: 2.6071 | LR: 3.08e-04 | 4007.7ms/step | 15.0 samples/s
1005
+ Step 21880/50462 | Loss: 2.6490 | LR: 3.08e-04 | 4007.0ms/step | 15.0 samples/s
1006
+ Step 21890/50462 | Loss: 2.6361 | LR: 3.08e-04 | 4010.9ms/step | 15.0 samples/s
1007
+ Step 21900/50462 | Loss: 2.6417 | LR: 3.08e-04 | 4007.3ms/step | 15.0 samples/s
1008
+ Step 21910/50462 | Loss: 2.6221 | LR: 3.08e-04 | 4009.5ms/step | 15.0 samples/s
1009
+ Step 21920/50462 | Loss: 2.5879 | LR: 3.08e-04 | 3999.9ms/step | 15.0 samples/s
1010
+ Step 21930/50462 | Loss: 2.6453 | LR: 3.08e-04 | 4013.6ms/step | 14.9 samples/s
1011
+ Step 21940/50462 | Loss: 2.6315 | LR: 3.07e-04 | 4009.3ms/step | 15.0 samples/s
1012
+ Step 21950/50462 | Loss: 2.6133 | LR: 3.07e-04 | 4005.9ms/step | 15.0 samples/s
1013
+ Step 21960/50462 | Loss: 2.6553 | LR: 3.07e-04 | 4009.4ms/step | 15.0 samples/s
1014
+ Step 21970/50462 | Loss: 2.6124 | LR: 3.07e-04 | 4016.6ms/step | 14.9 samples/s
1015
+ Step 21980/50462 | Loss: 2.6047 | LR: 3.07e-04 | 4005.5ms/step | 15.0 samples/s
1016
+ Step 21990/50462 | Loss: 2.5866 | LR: 3.07e-04 | 4000.6ms/step | 15.0 samples/s
1017
+ Step 22000/50462 | Loss: 2.6220 | LR: 3.06e-04 | 4007.3ms/step | 15.0 samples/s
1018
+ Validation at step 22000: Loss = 2.6449, Perplexity = 14.08
1019
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-22000
1020
+ Step 22010/50462 | Loss: 2.6228 | LR: 3.06e-04 | 5988.9ms/step | 10.0 samples/s
1021
+ Step 22020/50462 | Loss: 2.6226 | LR: 3.06e-04 | 4005.8ms/step | 15.0 samples/s
1022
+ Step 22030/50462 | Loss: 2.6470 | LR: 3.06e-04 | 4001.1ms/step | 15.0 samples/s
1023
+ Step 22040/50462 | Loss: 2.6404 | LR: 3.06e-04 | 4009.7ms/step | 15.0 samples/s
1024
+ Step 22050/50462 | Loss: 2.6177 | LR: 3.06e-04 | 4011.0ms/step | 15.0 samples/s
1025
+ Step 22060/50462 | Loss: 2.6332 | LR: 3.06e-04 | 4002.2ms/step | 15.0 samples/s
1026
+ Step 22070/50462 | Loss: 2.6405 | LR: 3.05e-04 | 4011.2ms/step | 15.0 samples/s
1027
+ Step 22080/50462 | Loss: 2.6133 | LR: 3.05e-04 | 4005.9ms/step | 15.0 samples/s
1028
+ Step 22090/50462 | Loss: 2.6173 | LR: 3.05e-04 | 4002.8ms/step | 15.0 samples/s
1029
+ Step 22100/50462 | Loss: 2.6406 | LR: 3.05e-04 | 4001.1ms/step | 15.0 samples/s
1030
+ Step 22110/50462 | Loss: 2.6615 | LR: 3.05e-04 | 4002.2ms/step | 15.0 samples/s
1031
+ Step 22120/50462 | Loss: 2.6281 | LR: 3.05e-04 | 4019.3ms/step | 14.9 samples/s
1032
+ Step 22130/50462 | Loss: 2.6237 | LR: 3.04e-04 | 4017.6ms/step | 14.9 samples/s
1033
+ Step 22140/50462 | Loss: 2.6134 | LR: 3.04e-04 | 4028.8ms/step | 14.9 samples/s
1034
+ Step 22150/50462 | Loss: 2.6452 | LR: 3.04e-04 | 4014.7ms/step | 14.9 samples/s
1035
+ Step 22160/50462 | Loss: 2.6180 | LR: 3.04e-04 | 4016.1ms/step | 14.9 samples/s
1036
+ Step 22170/50462 | Loss: 2.6267 | LR: 3.04e-04 | 4010.3ms/step | 15.0 samples/s
1037
+ Step 22180/50462 | Loss: 2.6417 | LR: 3.04e-04 | 4011.6ms/step | 15.0 samples/s
1038
+ Step 22190/50462 | Loss: 2.5881 | LR: 3.04e-04 | 4007.2ms/step | 15.0 samples/s
1039
+ Step 22200/50462 | Loss: 2.5934 | LR: 3.03e-04 | 4007.0ms/step | 15.0 samples/s
1040
+ Step 22210/50462 | Loss: 2.6597 | LR: 3.03e-04 | 4008.0ms/step | 15.0 samples/s
1041
+ Step 22220/50462 | Loss: 2.6139 | LR: 3.03e-04 | 4015.3ms/step | 14.9 samples/s
1042
+ Step 22230/50462 | Loss: 2.6634 | LR: 3.03e-04 | 4006.6ms/step | 15.0 samples/s
1043
+ Step 22240/50462 | Loss: 2.6018 | LR: 3.03e-04 | 4013.0ms/step | 15.0 samples/s
1044
+ Step 22250/50462 | Loss: 2.6288 | LR: 3.03e-04 | 4012.1ms/step | 15.0 samples/s
1045
+ Validation at step 22250: Loss = 2.6417, Perplexity = 14.04
1046
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-22250
1047
+ Step 22260/50462 | Loss: 2.5873 | LR: 3.02e-04 | 5957.4ms/step | 10.1 samples/s
1048
+ Step 22270/50462 | Loss: 2.6253 | LR: 3.02e-04 | 4006.0ms/step | 15.0 samples/s
1049
+ Step 22280/50462 | Loss: 2.6069 | LR: 3.02e-04 | 4013.7ms/step | 14.9 samples/s
1050
+ Step 22290/50462 | Loss: 2.6210 | LR: 3.02e-04 | 4008.6ms/step | 15.0 samples/s
1051
+ Step 22300/50462 | Loss: 2.6417 | LR: 3.02e-04 | 3996.0ms/step | 15.0 samples/s
1052
+ Step 22310/50462 | Loss: 2.6185 | LR: 3.02e-04 | 4010.6ms/step | 15.0 samples/s
1053
+ Step 22320/50462 | Loss: 2.6246 | LR: 3.02e-04 | 3997.9ms/step | 15.0 samples/s
1054
+ Step 22330/50462 | Loss: 2.6251 | LR: 3.01e-04 | 4001.9ms/step | 15.0 samples/s
1055
+ Step 22340/50462 | Loss: 2.6018 | LR: 3.01e-04 | 4002.5ms/step | 15.0 samples/s
1056
+ Step 22350/50462 | Loss: 2.6280 | LR: 3.01e-04 | 4005.9ms/step | 15.0 samples/s
1057
+ Step 22360/50462 | Loss: 2.6298 | LR: 3.01e-04 | 4009.2ms/step | 15.0 samples/s
1058
+ Step 22370/50462 | Loss: 2.6132 | LR: 3.01e-04 | 4011.7ms/step | 15.0 samples/s
1059
+ Step 22380/50462 | Loss: 2.5792 | LR: 3.01e-04 | 3998.4ms/step | 15.0 samples/s
1060
+ Step 22390/50462 | Loss: 2.6007 | LR: 3.00e-04 | 4007.1ms/step | 15.0 samples/s
1061
+ Step 22400/50462 | Loss: 2.6085 | LR: 3.00e-04 | 4011.7ms/step | 15.0 samples/s
1062
+ Step 22410/50462 | Loss: 2.5719 | LR: 3.00e-04 | 4003.8ms/step | 15.0 samples/s
1063
+ Step 22420/50462 | Loss: 2.5762 | LR: 3.00e-04 | 4009.4ms/step | 15.0 samples/s
1064
+ Step 22430/50462 | Loss: 2.5716 | LR: 3.00e-04 | 3986.1ms/step | 15.1 samples/s
1065
+ Step 22440/50462 | Loss: 2.5952 | LR: 3.00e-04 | 3989.7ms/step | 15.0 samples/s
1066
+ Step 22450/50462 | Loss: 2.6018 | LR: 3.00e-04 | 3994.1ms/step | 15.0 samples/s
1067
+ Step 22460/50462 | Loss: 2.6011 | LR: 2.99e-04 | 3997.3ms/step | 15.0 samples/s
1068
+ Step 22470/50462 | Loss: 2.6138 | LR: 2.99e-04 | 3993.4ms/step | 15.0 samples/s
1069
+ Step 22480/50462 | Loss: 2.6503 | LR: 2.99e-04 | 4002.1ms/step | 15.0 samples/s
1070
+ Step 22490/50462 | Loss: 2.5991 | LR: 2.99e-04 | 4001.9ms/step | 15.0 samples/s
1071
+ Step 22500/50462 | Loss: 2.6328 | LR: 2.99e-04 | 4002.7ms/step | 15.0 samples/s
1072
+ Validation at step 22500: Loss = 2.6410, Perplexity = 14.03
1073
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-22500
1074
+ Step 22510/50462 | Loss: 2.5897 | LR: 2.99e-04 | 5980.9ms/step | 10.0 samples/s
1075
+ Step 22520/50462 | Loss: 2.6149 | LR: 2.98e-04 | 4012.4ms/step | 15.0 samples/s
1076
+ Step 22530/50462 | Loss: 2.6444 | LR: 2.98e-04 | 4020.2ms/step | 14.9 samples/s
1077
+ Step 22540/50462 | Loss: 2.6261 | LR: 2.98e-04 | 4009.0ms/step | 15.0 samples/s
1078
+ Step 22550/50462 | Loss: 2.6331 | LR: 2.98e-04 | 4017.1ms/step | 14.9 samples/s
1079
+ Step 22560/50462 | Loss: 2.6433 | LR: 2.98e-04 | 4000.1ms/step | 15.0 samples/s
1080
+ Step 22570/50462 | Loss: 2.6199 | LR: 2.98e-04 | 3999.6ms/step | 15.0 samples/s
1081
+ Step 22580/50462 | Loss: 2.6381 | LR: 2.98e-04 | 4005.3ms/step | 15.0 samples/s
1082
+ Step 22590/50462 | Loss: 2.6192 | LR: 2.97e-04 | 4013.9ms/step | 14.9 samples/s
1083
+ Step 22600/50462 | Loss: 2.6339 | LR: 2.97e-04 | 4005.8ms/step | 15.0 samples/s
1084
+ Step 22610/50462 | Loss: 2.6185 | LR: 2.97e-04 | 4007.6ms/step | 15.0 samples/s
1085
+ Step 22620/50462 | Loss: 2.6527 | LR: 2.97e-04 | 4016.5ms/step | 14.9 samples/s
1086
+ Step 22630/50462 | Loss: 2.6344 | LR: 2.97e-04 | 3997.3ms/step | 15.0 samples/s
1087
+ Step 22640/50462 | Loss: 2.6493 | LR: 2.97e-04 | 4004.9ms/step | 15.0 samples/s
1088
+ Step 22650/50462 | Loss: 2.6389 | LR: 2.96e-04 | 3996.1ms/step | 15.0 samples/s
1089
+ Step 22660/50462 | Loss: 2.6013 | LR: 2.96e-04 | 4003.0ms/step | 15.0 samples/s
1090
+ Step 22670/50462 | Loss: 2.6135 | LR: 2.96e-04 | 3999.8ms/step | 15.0 samples/s
1091
+ Step 22680/50462 | Loss: 2.6150 | LR: 2.96e-04 | 3997.8ms/step | 15.0 samples/s
1092
+ Step 22690/50462 | Loss: 2.6396 | LR: 2.96e-04 | 4012.8ms/step | 15.0 samples/s
1093
+ Step 22700/50462 | Loss: 2.6053 | LR: 2.96e-04 | 4021.3ms/step | 14.9 samples/s
1094
+ Step 22710/50462 | Loss: 2.6630 | LR: 2.95e-04 | 4004.6ms/step | 15.0 samples/s
1095
+ Step 22720/50462 | Loss: 2.5961 | LR: 2.95e-04 | 3995.9ms/step | 15.0 samples/s
1096
+ Step 22730/50462 | Loss: 2.6382 | LR: 2.95e-04 | 4002.0ms/step | 15.0 samples/s
1097
+ Step 22740/50462 | Loss: 2.6343 | LR: 2.95e-04 | 4003.4ms/step | 15.0 samples/s
1098
+ Step 22750/50462 | Loss: 2.6366 | LR: 2.95e-04 | 3999.2ms/step | 15.0 samples/s
1099
+ Validation at step 22750: Loss = 2.6376, Perplexity = 13.98
1100
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-22750
1101
+ Step 22760/50462 | Loss: 2.6488 | LR: 2.95e-04 | 5950.6ms/step | 10.1 samples/s
1102
+ Step 22770/50462 | Loss: 2.6684 | LR: 2.95e-04 | 4011.1ms/step | 15.0 samples/s
1103
+ Step 22780/50462 | Loss: 2.6091 | LR: 2.94e-04 | 4000.2ms/step | 15.0 samples/s
1104
+ Step 22790/50462 | Loss: 2.6113 | LR: 2.94e-04 | 3998.5ms/step | 15.0 samples/s
1105
+ Step 22800/50462 | Loss: 2.6245 | LR: 2.94e-04 | 4006.2ms/step | 15.0 samples/s
1106
+ Step 22810/50462 | Loss: 2.6338 | LR: 2.94e-04 | 4005.0ms/step | 15.0 samples/s
1107
+ Step 22820/50462 | Loss: 2.6272 | LR: 2.94e-04 | 4003.0ms/step | 15.0 samples/s
1108
+ Step 22830/50462 | Loss: 2.6186 | LR: 2.94e-04 | 4002.7ms/step | 15.0 samples/s
1109
+ Step 22840/50462 | Loss: 2.5957 | LR: 2.93e-04 | 3995.2ms/step | 15.0 samples/s
1110
+ Step 22850/50462 | Loss: 2.6050 | LR: 2.93e-04 | 3992.5ms/step | 15.0 samples/s
1111
+ Step 22860/50462 | Loss: 2.5948 | LR: 2.93e-04 | 4003.0ms/step | 15.0 samples/s
1112
+ Step 22870/50462 | Loss: 2.5889 | LR: 2.93e-04 | 3998.8ms/step | 15.0 samples/s
1113
+ Step 22880/50462 | Loss: 2.6501 | LR: 2.93e-04 | 4004.2ms/step | 15.0 samples/s
1114
+ Step 22890/50462 | Loss: 2.6278 | LR: 2.93e-04 | 4010.0ms/step | 15.0 samples/s
1115
+ Step 22900/50462 | Loss: 2.5878 | LR: 2.93e-04 | 4003.3ms/step | 15.0 samples/s
1116
+ Step 22910/50462 | Loss: 2.6447 | LR: 2.92e-04 | 4008.4ms/step | 15.0 samples/s
1117
+ Step 22920/50462 | Loss: 2.6473 | LR: 2.92e-04 | 3993.1ms/step | 15.0 samples/s
1118
+ Step 22930/50462 | Loss: 2.5799 | LR: 2.92e-04 | 4001.6ms/step | 15.0 samples/s
1119
+ Step 22940/50462 | Loss: 2.6025 | LR: 2.92e-04 | 3996.2ms/step | 15.0 samples/s
1120
+ Step 22950/50462 | Loss: 2.6350 | LR: 2.92e-04 | 4004.4ms/step | 15.0 samples/s
1121
+ Step 22960/50462 | Loss: 2.6134 | LR: 2.92e-04 | 4009.0ms/step | 15.0 samples/s
1122
+ Step 22970/50462 | Loss: 2.6466 | LR: 2.91e-04 | 4004.5ms/step | 15.0 samples/s
1123
+ Step 22980/50462 | Loss: 2.5974 | LR: 2.91e-04 | 4006.8ms/step | 15.0 samples/s
1124
+ Step 22990/50462 | Loss: 2.6163 | LR: 2.91e-04 | 3995.8ms/step | 15.0 samples/s
1125
+ Step 23000/50462 | Loss: 2.6234 | LR: 2.91e-04 | 3993.2ms/step | 15.0 samples/s
1126
+ Validation at step 23000: Loss = 2.6367, Perplexity = 13.97
1127
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23000
1128
+ Step 23010/50462 | Loss: 2.5959 | LR: 2.91e-04 | 5953.9ms/step | 10.1 samples/s
1129
+ Step 23020/50462 | Loss: 2.6254 | LR: 2.91e-04 | 4002.4ms/step | 15.0 samples/s
1130
+ Step 23030/50462 | Loss: 2.6163 | LR: 2.91e-04 | 4013.8ms/step | 14.9 samples/s
1131
+ Step 23040/50462 | Loss: 2.6081 | LR: 2.90e-04 | 4004.9ms/step | 15.0 samples/s
1132
+ Step 23050/50462 | Loss: 2.5849 | LR: 2.90e-04 | 4000.3ms/step | 15.0 samples/s
1133
+ Step 23060/50462 | Loss: 2.5772 | LR: 2.90e-04 | 3986.0ms/step | 15.1 samples/s
1134
+ Step 23070/50462 | Loss: 2.5910 | LR: 2.90e-04 | 4009.9ms/step | 15.0 samples/s
1135
+ Step 23080/50462 | Loss: 2.5829 | LR: 2.90e-04 | 3992.2ms/step | 15.0 samples/s
1136
+ Step 23090/50462 | Loss: 2.5939 | LR: 2.90e-04 | 4001.5ms/step | 15.0 samples/s
1137
+ Step 23100/50462 | Loss: 2.6401 | LR: 2.89e-04 | 3997.7ms/step | 15.0 samples/s
1138
+ Step 23110/50462 | Loss: 2.6514 | LR: 2.89e-04 | 4012.0ms/step | 15.0 samples/s
1139
+ Step 23120/50462 | Loss: 2.6355 | LR: 2.89e-04 | 3999.0ms/step | 15.0 samples/s
1140
+ Step 23130/50462 | Loss: 2.6076 | LR: 2.89e-04 | 4010.4ms/step | 15.0 samples/s
1141
+ Step 23140/50462 | Loss: 2.6292 | LR: 2.89e-04 | 4010.4ms/step | 15.0 samples/s
1142
+ Step 23150/50462 | Loss: 2.6159 | LR: 2.89e-04 | 4001.6ms/step | 15.0 samples/s
1143
+ Step 23160/50462 | Loss: 2.6047 | LR: 2.88e-04 | 3988.1ms/step | 15.0 samples/s
1144
+ Step 23170/50462 | Loss: 2.6590 | LR: 2.88e-04 | 4008.8ms/step | 15.0 samples/s
1145
+ Step 23180/50462 | Loss: 2.6245 | LR: 2.88e-04 | 3991.7ms/step | 15.0 samples/s
1146
+ Step 23190/50462 | Loss: 2.6085 | LR: 2.88e-04 | 3992.5ms/step | 15.0 samples/s
1147
+ Step 23200/50462 | Loss: 2.6079 | LR: 2.88e-04 | 4001.6ms/step | 15.0 samples/s
1148
+ Step 23210/50462 | Loss: 2.6139 | LR: 2.88e-04 | 3998.9ms/step | 15.0 samples/s
1149
+ Step 23220/50462 | Loss: 2.6606 | LR: 2.88e-04 | 3999.7ms/step | 15.0 samples/s
1150
+ Step 23230/50462 | Loss: 2.6321 | LR: 2.87e-04 | 4007.0ms/step | 15.0 samples/s
1151
+ Step 23240/50462 | Loss: 2.6257 | LR: 2.87e-04 | 3999.9ms/step | 15.0 samples/s
1152
+ Step 23250/50462 | Loss: 2.6137 | LR: 2.87e-04 | 4003.6ms/step | 15.0 samples/s
1153
+ Validation at step 23250: Loss = 2.6327, Perplexity = 13.91
1154
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23250
1155
+ Step 23260/50462 | Loss: 2.6251 | LR: 2.87e-04 | 5945.6ms/step | 10.1 samples/s
1156
+ Step 23270/50462 | Loss: 2.5998 | LR: 2.87e-04 | 4016.8ms/step | 14.9 samples/s
1157
+ Step 23280/50462 | Loss: 2.5940 | LR: 2.87e-04 | 3994.1ms/step | 15.0 samples/s
1158
+ Step 23290/50462 | Loss: 2.6170 | LR: 2.86e-04 | 3996.0ms/step | 15.0 samples/s
1159
+ Step 23300/50462 | Loss: 2.6363 | LR: 2.86e-04 | 4017.0ms/step | 14.9 samples/s
1160
+ Step 23310/50462 | Loss: 2.5861 | LR: 2.86e-04 | 4001.0ms/step | 15.0 samples/s
1161
+ Step 23320/50462 | Loss: 2.6018 | LR: 2.86e-04 | 4003.6ms/step | 15.0 samples/s
1162
+ Step 23330/50462 | Loss: 2.6104 | LR: 2.86e-04 | 4006.9ms/step | 15.0 samples/s
1163
+ Step 23340/50462 | Loss: 2.5833 | LR: 2.86e-04 | 4005.2ms/step | 15.0 samples/s
1164
+ Step 23350/50462 | Loss: 2.6548 | LR: 2.86e-04 | 4001.2ms/step | 15.0 samples/s
1165
+ Step 23360/50462 | Loss: 2.5941 | LR: 2.85e-04 | 4006.7ms/step | 15.0 samples/s
1166
+ Step 23370/50462 | Loss: 2.6247 | LR: 2.85e-04 | 3994.8ms/step | 15.0 samples/s
1167
+ Step 23380/50462 | Loss: 2.5749 | LR: 2.85e-04 | 4007.4ms/step | 15.0 samples/s
1168
+ Step 23390/50462 | Loss: 2.6246 | LR: 2.85e-04 | 4006.1ms/step | 15.0 samples/s
1169
+ Step 23400/50462 | Loss: 2.5765 | LR: 2.85e-04 | 4006.2ms/step | 15.0 samples/s
1170
+ Step 23410/50462 | Loss: 2.6196 | LR: 2.85e-04 | 4012.2ms/step | 15.0 samples/s
epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/files/wandb-metadata.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-28T05:36:35.010459Z",
5
+ "args": [
6
+ "--config",
7
+ "configs/p2_1p2b_mixed_seed72_4gpu_resume12750.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/SparseAtt.git",
14
+ "commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-4-gpu-3-0-13",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 4,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "6151577186304"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-85844f64-d144-1c84-306d-f88b5f232a77"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-c65885f7-52dc-6f23-a3dd-6addf8b518a2"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-7e349d73-ad76-d62d-635c-266997fedcc1"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
61
+ }
62
+ ],
63
+ "cudaVersion": "13.0",
64
+ "writerId": "vkvf50jw3igjakr9xkk5u3wpr02bo3sw"
65
+ }
epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-28 05:36:35,042 INFO MainThread:199 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-28 05:36:35,042 INFO MainThread:199 [wandb_setup.py:_flush():81] Configure stats pid to 199
3
+ 2026-08-28 05:36:35,042 INFO MainThread:199 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-28 05:36:35,043 INFO MainThread:199 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_053635-z1i60278/logs/debug.log
5
+ 2026-08-28 05:36:35,043 INFO MainThread:199 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_053635-z1i60278/logs/debug-internal.log
6
+ 2026-08-28 05:36:35,043 INFO MainThread:199 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-28 05:36:35,043 INFO MainThread:199 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-12750', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 72, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed72 6gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed72'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 72, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-28 05:36:35,043 INFO MainThread:199 [wandb_init.py:init():898] starting backend
10
+ 2026-08-28 05:36:35,918 INFO MainThread:199 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-28 05:36:36,380 INFO MainThread:199 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-28 05:36:36,383 INFO MainThread:199 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-28 05:36:36,421 INFO MainThread:199 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-28 05:36:36,884 INFO MainThread:199 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-28 05:36:36,886 INFO MainThread:199 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-28 05:36:37,138 INFO MainThread:199 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-28 05:36:37,138 INFO MainThread:199 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-28 05:36:37,138 INFO MainThread:199 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-28 05:36:37,138 INFO MainThread:199 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-28 05:36:37,146 INFO MainThread:199 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260828_053635-z1i60278/run-z1i60278.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cf29f3ab263254edb58af650876a8c8997160fd4b4e6d8542212a45fc447e93a
3
+ size 14680064
epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/files/output.log ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 28500: Loss = 2.5505, Perplexity = 12.81
3
+ wandb: WARNING Tried to log to step 28500 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 28501: sparse_stochastic (configured sparse attention)
5
+ Step 28510/50462 | Loss: 2.5451 | LR: 2.04e-04 | 76514.7ms/step | 0.8 samples/s
6
+ wandb: WARNING Tried to log to step 28510 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 28520/50462 | Loss: 2.5720 | LR: 2.04e-04 | 3980.4ms/step | 15.1 samples/s
8
+ wandb: WARNING Tried to log to step 28520 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 28530/50462 | Loss: 2.5474 | LR: 2.04e-04 | 3977.4ms/step | 15.1 samples/s
10
+ wandb: WARNING Tried to log to step 28530 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 28540/50462 | Loss: 2.5644 | LR: 2.04e-04 | 3982.4ms/step | 15.1 samples/s
12
+ wandb: WARNING Tried to log to step 28540 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 28550/50462 | Loss: 2.5906 | LR: 2.04e-04 | 3987.8ms/step | 15.0 samples/s
14
+ wandb: WARNING Tried to log to step 28550 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 28560/50462 | Loss: 2.5604 | LR: 2.04e-04 | 3985.3ms/step | 15.1 samples/s
16
+ wandb: WARNING Tried to log to step 28560 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 28570/50462 | Loss: 2.5596 | LR: 2.03e-04 | 3974.8ms/step | 15.1 samples/s
18
+ wandb: WARNING Tried to log to step 28570 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
19
+ Step 28580/50462 | Loss: 2.5638 | LR: 2.03e-04 | 4002.9ms/step | 15.0 samples/s
20
+ wandb: WARNING Tried to log to step 28580 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
21
+ Step 28590/50462 | Loss: 2.5860 | LR: 2.03e-04 | 3995.8ms/step | 15.0 samples/s
22
+ wandb: WARNING Tried to log to step 28590 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
23
+ Step 28600/50462 | Loss: 2.5439 | LR: 2.03e-04 | 3992.3ms/step | 15.0 samples/s
24
+ wandb: WARNING Tried to log to step 28600 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
25
+ Step 28610/50462 | Loss: 2.5935 | LR: 2.03e-04 | 3982.6ms/step | 15.1 samples/s
26
+ wandb: WARNING Tried to log to step 28610 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
27
+ Step 28620/50462 | Loss: 2.5405 | LR: 2.03e-04 | 3981.1ms/step | 15.1 samples/s
28
+ wandb: WARNING Tried to log to step 28620 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
29
+ Step 28630/50462 | Loss: 2.5792 | LR: 2.03e-04 | 3996.1ms/step | 15.0 samples/s
30
+ wandb: WARNING Tried to log to step 28630 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
31
+ Step 28640/50462 | Loss: 2.5795 | LR: 2.02e-04 | 3988.0ms/step | 15.0 samples/s
32
+ wandb: WARNING Tried to log to step 28640 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
33
+ Step 28650/50462 | Loss: 2.5703 | LR: 2.02e-04 | 3988.3ms/step | 15.0 samples/s
34
+ wandb: WARNING Tried to log to step 28650 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
35
+ Step 28660/50462 | Loss: 2.5650 | LR: 2.02e-04 | 3983.2ms/step | 15.1 samples/s
36
+ wandb: WARNING Tried to log to step 28660 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
37
+ Step 28670/50462 | Loss: 2.5692 | LR: 2.02e-04 | 3988.2ms/step | 15.0 samples/s
38
+ wandb: WARNING Tried to log to step 28670 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
39
+ Step 28680/50462 | Loss: 2.5620 | LR: 2.02e-04 | 3980.9ms/step | 15.1 samples/s
40
+ wandb: WARNING Tried to log to step 28680 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
41
+ Step 28690/50462 | Loss: 2.5730 | LR: 2.02e-04 | 3987.5ms/step | 15.0 samples/s
42
+ wandb: WARNING Tried to log to step 28690 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
43
+ Step 28700/50462 | Loss: 2.5995 | LR: 2.01e-04 | 3992.0ms/step | 15.0 samples/s
44
+ wandb: WARNING Tried to log to step 28700 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
45
+ Step 28710/50462 | Loss: 2.5724 | LR: 2.01e-04 | 3986.5ms/step | 15.1 samples/s
46
+ wandb: WARNING Tried to log to step 28710 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
47
+ Step 28720/50462 | Loss: 2.5222 | LR: 2.01e-04 | 3992.0ms/step | 15.0 samples/s
48
+ wandb: WARNING Tried to log to step 28720 that is less than the current step 28721. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
49
+ Step 28730/50462 | Loss: 2.5623 | LR: 2.01e-04 | 3997.3ms/step | 15.0 samples/s
50
+ Step 28740/50462 | Loss: 2.5654 | LR: 2.01e-04 | 3992.9ms/step | 15.0 samples/s
51
+ Step 28750/50462 | Loss: 2.5627 | LR: 2.01e-04 | 3990.6ms/step | 15.0 samples/s
52
+ Validation at step 28750: Loss = 2.5506, Perplexity = 12.82
53
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28750
54
+ Step 28760/50462 | Loss: 2.5691 | LR: 2.00e-04 | 5924.9ms/step | 10.1 samples/s
55
+ Step 28770/50462 | Loss: 2.5540 | LR: 2.00e-04 | 3984.7ms/step | 15.1 samples/s
56
+ Step 28780/50462 | Loss: 2.5849 | LR: 2.00e-04 | 3990.8ms/step | 15.0 samples/s
57
+ Step 28790/50462 | Loss: 2.5539 | LR: 2.00e-04 | 3992.7ms/step | 15.0 samples/s
58
+ Step 28800/50462 | Loss: 2.5415 | LR: 2.00e-04 | 3981.7ms/step | 15.1 samples/s
59
+ Step 28810/50462 | Loss: 2.5533 | LR: 2.00e-04 | 3988.3ms/step | 15.0 samples/s
60
+ Step 28820/50462 | Loss: 2.5651 | LR: 2.00e-04 | 3983.2ms/step | 15.1 samples/s
61
+ Step 28830/50462 | Loss: 2.5387 | LR: 1.99e-04 | 3982.7ms/step | 15.1 samples/s
62
+ Step 28840/50462 | Loss: 2.5934 | LR: 1.99e-04 | 3981.2ms/step | 15.1 samples/s
63
+ Step 28850/50462 | Loss: 2.5488 | LR: 1.99e-04 | 3988.4ms/step | 15.0 samples/s
64
+ Step 28860/50462 | Loss: 2.5718 | LR: 1.99e-04 | 3991.5ms/step | 15.0 samples/s
65
+ Step 28870/50462 | Loss: 2.5898 | LR: 1.99e-04 | 3994.7ms/step | 15.0 samples/s
66
+ Step 28880/50462 | Loss: 2.5456 | LR: 1.99e-04 | 3983.8ms/step | 15.1 samples/s
67
+ Step 28890/50462 | Loss: 2.5258 | LR: 1.98e-04 | 3989.6ms/step | 15.0 samples/s
68
+ Step 28900/50462 | Loss: 2.5468 | LR: 1.98e-04 | 3988.6ms/step | 15.0 samples/s
69
+ Step 28910/50462 | Loss: 2.5241 | LR: 1.98e-04 | 3991.9ms/step | 15.0 samples/s
70
+ Step 28920/50462 | Loss: 2.5744 | LR: 1.98e-04 | 3994.6ms/step | 15.0 samples/s
71
+ Step 28930/50462 | Loss: 2.5797 | LR: 1.98e-04 | 3976.5ms/step | 15.1 samples/s
72
+ Step 28940/50462 | Loss: 2.5382 | LR: 1.98e-04 | 3990.5ms/step | 15.0 samples/s
73
+ Step 28950/50462 | Loss: 2.5434 | LR: 1.98e-04 | 3979.6ms/step | 15.1 samples/s
74
+ Step 28960/50462 | Loss: 2.5487 | LR: 1.97e-04 | 3984.5ms/step | 15.1 samples/s
75
+ Step 28970/50462 | Loss: 2.5581 | LR: 1.97e-04 | 3980.7ms/step | 15.1 samples/s
76
+ Step 28980/50462 | Loss: 2.5621 | LR: 1.97e-04 | 3991.3ms/step | 15.0 samples/s
77
+ Step 28990/50462 | Loss: 2.5556 | LR: 1.97e-04 | 3981.1ms/step | 15.1 samples/s
78
+ Step 29000/50462 | Loss: 2.5778 | LR: 1.97e-04 | 3979.1ms/step | 15.1 samples/s
79
+ Validation at step 29000: Loss = 2.5477, Perplexity = 12.78
80
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000
81
+ Step 29010/50462 | Loss: 2.5469 | LR: 1.97e-04 | 5909.8ms/step | 10.2 samples/s
82
+ Step 29020/50462 | Loss: 2.5159 | LR: 1.96e-04 | 3986.9ms/step | 15.0 samples/s
83
+ Step 29030/50462 | Loss: 2.5460 | LR: 1.96e-04 | 3991.0ms/step | 15.0 samples/s
84
+ Step 29040/50462 | Loss: 2.5689 | LR: 1.96e-04 | 3994.9ms/step | 15.0 samples/s
85
+ Step 29050/50462 | Loss: 2.5351 | LR: 1.96e-04 | 3979.8ms/step | 15.1 samples/s
86
+ Step 29060/50462 | Loss: 2.5869 | LR: 1.96e-04 | 3985.0ms/step | 15.1 samples/s
87
+ Step 29070/50462 | Loss: 2.5512 | LR: 1.96e-04 | 3996.0ms/step | 15.0 samples/s
epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/files/wandb-metadata.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-28T13:36:29.833225Z",
5
+ "args": [
6
+ "--config",
7
+ "configs/p2_1p2b_mixed_seed71_4gpu_resume28500.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/SparseAtt.git",
14
+ "commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-train-4-gpu-0-25",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 4,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9371709865984",
27
+ "used": "6613076279296"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-98029aa4-c589-b65b-3ab7-6495256f31ed"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-37a1c6c2-d3df-3064-44df-33fa82304bd0"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-24b8d952-dc08-33b9-d5df-54ce871fd288"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-e4ba0f44-6ff8-2e43-5721-346902be5759"
61
+ }
62
+ ],
63
+ "cudaVersion": "13.0",
64
+ "writerId": "vwsreox19o6t39uc0knehc85wrbcueio"
65
+ }
epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/logs/debug-internal.log ADDED
@@ -0,0 +1,440 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"time":"2026-08-28T13:36:30.618512563Z","level":"INFO","msg":"wandb-core"}
2
+ {"time":"2026-08-28T13:36:30.624577226Z","level":"INFO","msg":"stream: starting","core version":"0.26.1"}
3
+ {"time":"2026-08-28T13:36:31.042824593Z","level":"INFO","msg":"stream: created new stream","id":"8ynn1we6"}
4
+ {"time":"2026-08-28T13:36:31.042973455Z","level":"INFO","msg":"handler: started"}
5
+ {"time":"2026-08-28T13:36:31.044892396Z","level":"INFO","msg":"stream: started"}
6
+ {"time":"2026-08-28T13:36:31.046065651Z","level":"INFO","msg":"writer: started","stream_id":"8ynn1we6"}
7
+ {"time":"2026-08-28T13:36:31.046067464Z","level":"INFO","msg":"sender: started"}
8
+ {"time":"2026-08-28T13:36:31.660479346Z","level":"INFO","msg":"filestream: sending request","total_files":1}
9
+ {"time":"2026-08-28T13:36:31.90791728Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
10
+ {"time":"2026-08-28T13:36:46.660663895Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1,"uploaded_len":2}
11
+ {"time":"2026-08-28T13:36:46.876081073Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
12
+ {"time":"2026-08-28T13:37:01.661164173Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":0,"events_lines":2}
13
+ {"time":"2026-08-28T13:37:01.862061026Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
14
+ {"time":"2026-08-28T13:37:16.661090114Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":2,"events_lines":2}
15
+ {"time":"2026-08-28T13:37:16.870072902Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
16
+ {"time":"2026-08-28T13:37:18.665647664Z","level":"WARN","msg":"handler: ignoring partial history record","step":28500,"current":28721}
17
+ {"time":"2026-08-28T13:37:31.661321612Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":4,"events_lines":2,"console_offset":1,"console_lines":2}
18
+ {"time":"2026-08-28T13:37:31.898608116Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
19
+ {"time":"2026-08-28T13:37:46.660858941Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":6,"events_lines":2}
20
+ {"time":"2026-08-28T13:37:46.859388885Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
21
+ {"time":"2026-08-28T13:38:01.660620096Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":8,"events_lines":2}
22
+ {"time":"2026-08-28T13:38:01.878056359Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
23
+ {"time":"2026-08-28T13:38:16.661452915Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":10,"events_lines":2}
24
+ {"time":"2026-08-28T13:38:16.872959866Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
25
+ {"time":"2026-08-28T13:38:31.660756265Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":12,"events_lines":2}
26
+ {"time":"2026-08-28T13:38:31.856476723Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
27
+ {"time":"2026-08-28T13:38:46.660897759Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":14,"events_lines":2}
28
+ {"time":"2026-08-28T13:38:46.860165681Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
29
+ {"time":"2026-08-28T13:39:01.660687297Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":16,"events_lines":2}
30
+ {"time":"2026-08-28T13:39:01.86364133Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
31
+ {"time":"2026-08-28T13:39:16.660460381Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":18,"events_lines":2}
32
+ {"time":"2026-08-28T13:39:16.884347857Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
33
+ {"time":"2026-08-28T13:39:31.660519231Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":20,"events_lines":2}
34
+ {"time":"2026-08-28T13:39:31.860482238Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
35
+ {"time":"2026-08-28T13:39:46.661017589Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":22,"events_lines":2}
36
+ {"time":"2026-08-28T13:39:46.858759215Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
37
+ {"time":"2026-08-28T13:40:01.661055678Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":24,"events_lines":2}
38
+ {"time":"2026-08-28T13:40:01.865298291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
39
+ {"time":"2026-08-28T13:40:16.661337832Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":26,"events_lines":2}
40
+ {"time":"2026-08-28T13:40:16.870561212Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
41
+ {"time":"2026-08-28T13:40:31.660642712Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":28,"events_lines":2}
42
+ {"time":"2026-08-28T13:40:31.862861502Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
43
+ {"time":"2026-08-28T13:40:46.660865242Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":30,"events_lines":2}
44
+ {"time":"2026-08-28T13:40:46.901766951Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
45
+ {"time":"2026-08-28T13:41:01.66091163Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":32,"events_lines":2}
46
+ {"time":"2026-08-28T13:41:01.891824078Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
47
+ {"time":"2026-08-28T13:41:16.661395291Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":34,"events_lines":2}
48
+ {"time":"2026-08-28T13:41:16.885626042Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
49
+ {"time":"2026-08-28T13:41:31.660618329Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":36,"events_lines":2}
50
+ {"time":"2026-08-28T13:41:31.882492498Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
51
+ {"time":"2026-08-28T13:41:46.661398616Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":38,"events_lines":2}
52
+ {"time":"2026-08-28T13:41:46.867572364Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
53
+ {"time":"2026-08-28T13:42:01.660572952Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":40,"events_lines":2}
54
+ {"time":"2026-08-28T13:42:01.854657417Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
55
+ {"time":"2026-08-28T13:42:16.661412187Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":42,"events_lines":2}
56
+ {"time":"2026-08-28T13:42:16.849975899Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
57
+ {"time":"2026-08-28T13:42:31.660762621Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":44,"events_lines":2}
58
+ {"time":"2026-08-28T13:42:31.84427001Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
59
+ {"time":"2026-08-28T13:42:46.6613193Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":46,"events_lines":2}
60
+ {"time":"2026-08-28T13:42:46.851506112Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
61
+ {"time":"2026-08-28T13:43:01.660511029Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":48,"events_lines":2}
62
+ {"time":"2026-08-28T13:43:01.871260863Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
63
+ {"time":"2026-08-28T13:43:16.661105363Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":50,"events_lines":2}
64
+ {"time":"2026-08-28T13:43:16.864360717Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
65
+ {"time":"2026-08-28T13:43:31.660824811Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":52,"events_lines":2}
66
+ {"time":"2026-08-28T13:43:31.850943726Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
67
+ {"time":"2026-08-28T13:43:46.661223691Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":54,"events_lines":2}
68
+ {"time":"2026-08-28T13:43:46.852905102Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
69
+ {"time":"2026-08-28T13:44:01.661348102Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":56,"events_lines":2}
70
+ {"time":"2026-08-28T13:44:01.8734416Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
71
+ {"time":"2026-08-28T13:44:16.661301016Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
72
+ {"time":"2026-08-28T13:44:16.866764094Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
73
+ {"time":"2026-08-28T13:44:31.66104438Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":60,"events_lines":2}
74
+ {"time":"2026-08-28T13:44:31.846327492Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
75
+ {"time":"2026-08-28T13:44:46.660619848Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":62,"events_lines":2}
76
+ {"time":"2026-08-28T13:44:46.844375967Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
77
+ {"time":"2026-08-28T13:45:01.660933839Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":64,"events_lines":2}
78
+ {"time":"2026-08-28T13:45:01.901109018Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
79
+ {"time":"2026-08-28T13:45:16.661049875Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":66,"events_lines":2}
80
+ {"time":"2026-08-28T13:45:16.850627176Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
81
+ {"time":"2026-08-28T13:45:31.660552423Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":68,"events_lines":2}
82
+ {"time":"2026-08-28T13:45:31.869893355Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
83
+ {"time":"2026-08-28T13:45:46.660373716Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":70,"events_lines":2}
84
+ {"time":"2026-08-28T13:45:46.85521645Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
85
+ {"time":"2026-08-28T13:46:01.660754078Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":72,"events_lines":2}
86
+ {"time":"2026-08-28T13:46:01.892097983Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
87
+ {"time":"2026-08-28T13:46:16.661220929Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":74,"events_lines":2}
88
+ {"time":"2026-08-28T13:46:16.857831005Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
89
+ {"time":"2026-08-28T13:46:31.660697981Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":76,"events_lines":2}
90
+ {"time":"2026-08-28T13:46:31.851775036Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
91
+ {"time":"2026-08-28T13:46:46.660905147Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":78,"events_lines":2}
92
+ {"time":"2026-08-28T13:46:46.843098937Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
93
+ {"time":"2026-08-28T13:47:01.660774472Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":80,"events_lines":2}
94
+ {"time":"2026-08-28T13:47:01.884557876Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
95
+ {"time":"2026-08-28T13:47:16.660569386Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":82,"events_lines":2}
96
+ {"time":"2026-08-28T13:47:16.854226648Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
97
+ {"time":"2026-08-28T13:47:31.660900151Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":84,"events_lines":2,"console_offset":3,"console_lines":1}
98
+ {"time":"2026-08-28T13:47:31.878021168Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
99
+ {"time":"2026-08-28T13:47:46.660748353Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":86,"events_lines":2}
100
+ {"time":"2026-08-28T13:47:46.849458198Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
101
+ {"time":"2026-08-28T13:48:01.660957396Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":88,"events_lines":2}
102
+ {"time":"2026-08-28T13:48:01.876312125Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
103
+ {"time":"2026-08-28T13:48:16.660663938Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":90,"events_lines":2}
104
+ {"time":"2026-08-28T13:48:16.874298065Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
105
+ {"time":"2026-08-28T13:48:31.661166969Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":92,"events_lines":2}
106
+ {"time":"2026-08-28T13:48:31.868477184Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
107
+ {"time":"2026-08-28T13:48:46.661322492Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":94,"events_lines":2}
108
+ {"time":"2026-08-28T13:48:46.845563918Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
109
+ {"time":"2026-08-28T13:49:01.661101146Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":96,"events_lines":2}
110
+ {"time":"2026-08-28T13:49:01.875156935Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
111
+ {"time":"2026-08-28T13:49:16.660482833Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":98,"events_lines":2}
112
+ {"time":"2026-08-28T13:49:16.866970139Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
113
+ {"time":"2026-08-28T13:49:17.120759194Z","level":"WARN","msg":"handler: ignoring partial history record","step":28510,"current":28721}
114
+ {"time":"2026-08-28T13:49:31.661445684Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":100,"events_lines":2,"console_offset":4,"console_lines":2}
115
+ {"time":"2026-08-28T13:49:31.866851934Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
116
+ {"time":"2026-08-28T13:49:46.661304584Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":102,"events_lines":2}
117
+ {"time":"2026-08-28T13:49:46.847988197Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
118
+ {"time":"2026-08-28T13:49:56.924591935Z","level":"WARN","msg":"handler: ignoring partial history record","step":28520,"current":28721}
119
+ {"time":"2026-08-28T13:50:01.661280388Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":104,"events_lines":2,"console_offset":6,"console_lines":1}
120
+ {"time":"2026-08-28T13:50:01.874956144Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
121
+ {"time":"2026-08-28T13:50:16.660464254Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":106,"events_lines":2,"console_offset":7,"console_lines":1}
122
+ {"time":"2026-08-28T13:50:16.863994314Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
123
+ {"time":"2026-08-28T13:50:31.661182373Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":108,"events_lines":2}
124
+ {"time":"2026-08-28T13:50:31.857167975Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
125
+ {"time":"2026-08-28T13:50:36.698042753Z","level":"WARN","msg":"handler: ignoring partial history record","step":28530,"current":28721}
126
+ {"time":"2026-08-28T13:50:46.660897142Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":110,"events_lines":2,"console_offset":8,"console_lines":2}
127
+ {"time":"2026-08-28T13:50:46.86373652Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
128
+ {"time":"2026-08-28T13:51:01.661011547Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":112,"events_lines":2}
129
+ {"time":"2026-08-28T13:51:01.891731445Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
130
+ {"time":"2026-08-28T13:51:16.521817797Z","level":"WARN","msg":"handler: ignoring partial history record","step":28540,"current":28721}
131
+ {"time":"2026-08-28T13:51:16.661272595Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":114,"events_lines":2,"console_offset":10,"console_lines":1}
132
+ {"time":"2026-08-28T13:51:16.870073687Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
133
+ {"time":"2026-08-28T13:51:31.660762299Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":116,"events_lines":2,"console_offset":11,"console_lines":1}
134
+ {"time":"2026-08-28T13:51:31.864664277Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
135
+ {"time":"2026-08-28T13:51:46.6614268Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":118,"events_lines":2}
136
+ {"time":"2026-08-28T13:51:46.881252232Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
137
+ {"time":"2026-08-28T13:51:56.399810818Z","level":"WARN","msg":"handler: ignoring partial history record","step":28550,"current":28721}
138
+ {"time":"2026-08-28T13:52:01.660780873Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":120,"events_lines":2,"console_offset":12,"console_lines":1}
139
+ {"time":"2026-08-28T13:52:01.923689529Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
140
+ {"time":"2026-08-28T13:52:16.661456538Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":122,"events_lines":2,"console_offset":13,"console_lines":1}
141
+ {"time":"2026-08-28T13:52:16.862538301Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
142
+ {"time":"2026-08-28T13:52:31.660836018Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":124,"events_lines":2}
143
+ {"time":"2026-08-28T13:52:31.875286937Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
144
+ {"time":"2026-08-28T13:52:36.25306161Z","level":"WARN","msg":"handler: ignoring partial history record","step":28560,"current":28721}
145
+ {"time":"2026-08-28T13:52:46.6605304Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":126,"events_lines":2,"console_offset":14,"console_lines":2}
146
+ {"time":"2026-08-28T13:52:46.922436418Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
147
+ {"time":"2026-08-28T13:53:01.660738199Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":128,"events_lines":2}
148
+ {"time":"2026-08-28T13:53:01.884242581Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
149
+ {"time":"2026-08-28T13:53:16.0009212Z","level":"WARN","msg":"handler: ignoring partial history record","step":28570,"current":28721}
150
+ {"time":"2026-08-28T13:53:16.661353121Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":130,"events_lines":2,"console_offset":16,"console_lines":1}
151
+ {"time":"2026-08-28T13:53:16.859497106Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
152
+ {"time":"2026-08-28T13:53:31.661149537Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":132,"events_lines":2,"console_offset":17,"console_lines":1}
153
+ {"time":"2026-08-28T13:53:31.868560666Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
154
+ {"time":"2026-08-28T13:53:46.660533246Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":134,"events_lines":2}
155
+ {"time":"2026-08-28T13:53:46.848538085Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
156
+ {"time":"2026-08-28T13:53:56.029853235Z","level":"WARN","msg":"handler: ignoring partial history record","step":28580,"current":28721}
157
+ {"time":"2026-08-28T13:54:01.661247905Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":136,"events_lines":2,"console_offset":18,"console_lines":1}
158
+ {"time":"2026-08-28T13:54:01.851440374Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
159
+ {"time":"2026-08-28T13:54:16.661377293Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":138,"events_lines":2,"console_offset":19,"console_lines":1}
160
+ {"time":"2026-08-28T13:54:16.851241857Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
161
+ {"time":"2026-08-28T13:54:31.660421801Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":140,"events_lines":2}
162
+ {"time":"2026-08-28T13:54:31.884680008Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
163
+ {"time":"2026-08-28T13:54:35.987923683Z","level":"WARN","msg":"handler: ignoring partial history record","step":28590,"current":28721}
164
+ {"time":"2026-08-28T13:54:46.661231566Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":142,"events_lines":2,"console_offset":20,"console_lines":2}
165
+ {"time":"2026-08-28T13:54:46.870323171Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
166
+ {"time":"2026-08-28T13:55:01.66084941Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":144,"events_lines":2}
167
+ {"time":"2026-08-28T13:55:01.883648165Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
168
+ {"time":"2026-08-28T13:55:15.911148538Z","level":"WARN","msg":"handler: ignoring partial history record","step":28600,"current":28721}
169
+ {"time":"2026-08-28T13:55:16.660830697Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":146,"events_lines":2,"console_offset":22,"console_lines":1}
170
+ {"time":"2026-08-28T13:55:16.880987042Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
171
+ {"time":"2026-08-28T13:55:31.661069994Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":148,"events_lines":2,"console_offset":23,"console_lines":1}
172
+ {"time":"2026-08-28T13:55:31.857905789Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
173
+ {"time":"2026-08-28T13:55:46.660547588Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":150,"events_lines":2}
174
+ {"time":"2026-08-28T13:55:46.876937227Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
175
+ {"time":"2026-08-28T13:55:55.737047273Z","level":"WARN","msg":"handler: ignoring partial history record","step":28610,"current":28721}
176
+ {"time":"2026-08-28T13:56:01.660884246Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":152,"events_lines":2,"console_offset":24,"console_lines":1}
177
+ {"time":"2026-08-28T13:56:01.847932612Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
178
+ {"time":"2026-08-28T13:56:16.660508008Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":154,"events_lines":2,"console_offset":25,"console_lines":1}
179
+ {"time":"2026-08-28T13:56:16.889029128Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
180
+ {"time":"2026-08-28T13:56:31.661326823Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":156,"events_lines":2}
181
+ {"time":"2026-08-28T13:56:31.849682811Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
182
+ {"time":"2026-08-28T13:56:35.548577483Z","level":"WARN","msg":"handler: ignoring partial history record","step":28620,"current":28721}
183
+ {"time":"2026-08-28T13:56:46.660862422Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":158,"events_lines":2,"console_offset":26,"console_lines":2}
184
+ {"time":"2026-08-28T13:56:46.869651484Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
185
+ {"time":"2026-08-28T13:57:01.661563192Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":160,"events_lines":2}
186
+ {"time":"2026-08-28T13:57:01.862833248Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
187
+ {"time":"2026-08-28T13:57:15.509420223Z","level":"WARN","msg":"handler: ignoring partial history record","step":28630,"current":28721}
188
+ {"time":"2026-08-28T13:57:16.660906953Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":162,"events_lines":2,"console_offset":28,"console_lines":1}
189
+ {"time":"2026-08-28T13:57:16.849130593Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
190
+ {"time":"2026-08-28T13:57:31.660752601Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":164,"events_lines":2,"console_offset":29,"console_lines":1}
191
+ {"time":"2026-08-28T13:57:31.851472937Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
192
+ {"time":"2026-08-28T13:57:46.66058534Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":166,"events_lines":2}
193
+ {"time":"2026-08-28T13:57:46.931963291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
194
+ {"time":"2026-08-28T13:57:55.389671859Z","level":"WARN","msg":"handler: ignoring partial history record","step":28640,"current":28721}
195
+ {"time":"2026-08-28T13:58:01.660956393Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":168,"events_lines":2,"console_offset":30,"console_lines":1}
196
+ {"time":"2026-08-28T13:58:01.881265961Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
197
+ {"time":"2026-08-28T13:58:16.660814991Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":170,"events_lines":2,"console_offset":31,"console_lines":1}
198
+ {"time":"2026-08-28T13:58:16.881648873Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
199
+ {"time":"2026-08-28T13:58:31.661399584Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":172,"events_lines":2}
200
+ {"time":"2026-08-28T13:58:31.871836247Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
201
+ {"time":"2026-08-28T13:58:35.272305632Z","level":"WARN","msg":"handler: ignoring partial history record","step":28650,"current":28721}
202
+ {"time":"2026-08-28T13:58:46.661225858Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":174,"events_lines":2,"console_offset":32,"console_lines":2}
203
+ {"time":"2026-08-28T13:58:46.857175317Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
204
+ {"time":"2026-08-28T13:59:01.660808547Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":176,"events_lines":2}
205
+ {"time":"2026-08-28T13:59:01.864206337Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
206
+ {"time":"2026-08-28T13:59:15.104768337Z","level":"WARN","msg":"handler: ignoring partial history record","step":28660,"current":28721}
207
+ {"time":"2026-08-28T13:59:16.660544315Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":178,"events_lines":2,"console_offset":34,"console_lines":1}
208
+ {"time":"2026-08-28T13:59:16.854060948Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
209
+ {"time":"2026-08-28T13:59:31.661269191Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":180,"events_lines":2,"console_offset":35,"console_lines":1}
210
+ {"time":"2026-08-28T13:59:31.883341919Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
211
+ {"time":"2026-08-28T13:59:46.660927364Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":182,"events_lines":2}
212
+ {"time":"2026-08-28T13:59:46.8709915Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
213
+ {"time":"2026-08-28T13:59:54.98692638Z","level":"WARN","msg":"handler: ignoring partial history record","step":28670,"current":28721}
214
+ {"time":"2026-08-28T14:00:01.661265574Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":184,"events_lines":2,"console_offset":36,"console_lines":1}
215
+ {"time":"2026-08-28T14:00:01.867032899Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
216
+ {"time":"2026-08-28T14:00:16.660548916Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":186,"events_lines":2,"console_offset":37,"console_lines":1}
217
+ {"time":"2026-08-28T14:00:16.871326579Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
218
+ {"time":"2026-08-28T14:00:31.66086986Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":188,"events_lines":2}
219
+ {"time":"2026-08-28T14:00:31.879528741Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
220
+ {"time":"2026-08-28T14:00:34.796227227Z","level":"WARN","msg":"handler: ignoring partial history record","step":28680,"current":28721}
221
+ {"time":"2026-08-28T14:00:46.660852881Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":190,"events_lines":2,"console_offset":38,"console_lines":2}
222
+ {"time":"2026-08-28T14:00:46.86182431Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
223
+ {"time":"2026-08-28T14:01:01.660407085Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":192,"events_lines":2}
224
+ {"time":"2026-08-28T14:01:01.84611958Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
225
+ {"time":"2026-08-28T14:01:14.671178851Z","level":"WARN","msg":"handler: ignoring partial history record","step":28690,"current":28721}
226
+ {"time":"2026-08-28T14:01:16.661383854Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":194,"events_lines":2,"console_offset":40,"console_lines":1}
227
+ {"time":"2026-08-28T14:01:16.879088119Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
228
+ {"time":"2026-08-28T14:01:31.660514905Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":196,"events_lines":2,"console_offset":41,"console_lines":1}
229
+ {"time":"2026-08-28T14:01:31.860860399Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
230
+ {"time":"2026-08-28T14:01:46.661135939Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":198,"events_lines":2}
231
+ {"time":"2026-08-28T14:01:46.865804957Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
232
+ {"time":"2026-08-28T14:01:54.591782672Z","level":"WARN","msg":"handler: ignoring partial history record","step":28700,"current":28721}
233
+ {"time":"2026-08-28T14:02:01.660516283Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":200,"events_lines":2,"console_offset":42,"console_lines":1}
234
+ {"time":"2026-08-28T14:02:01.903910317Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
235
+ {"time":"2026-08-28T14:02:16.660904132Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":202,"events_lines":2,"console_offset":43,"console_lines":1}
236
+ {"time":"2026-08-28T14:02:16.877028909Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
237
+ {"time":"2026-08-28T14:02:31.660653237Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":204,"events_lines":2}
238
+ {"time":"2026-08-28T14:02:31.862469533Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
239
+ {"time":"2026-08-28T14:02:34.455545835Z","level":"WARN","msg":"handler: ignoring partial history record","step":28710,"current":28721}
240
+ {"time":"2026-08-28T14:02:46.660577804Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":206,"events_lines":2,"console_offset":44,"console_lines":2}
241
+ {"time":"2026-08-28T14:02:46.865677666Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
242
+ {"time":"2026-08-28T14:03:01.660895279Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":208,"events_lines":2}
243
+ {"time":"2026-08-28T14:03:01.853748585Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
244
+ {"time":"2026-08-28T14:03:14.375213761Z","level":"WARN","msg":"handler: ignoring partial history record","step":28720,"current":28721}
245
+ {"time":"2026-08-28T14:03:16.661198048Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":210,"events_lines":2,"console_offset":46,"console_lines":1}
246
+ {"time":"2026-08-28T14:03:16.865899538Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
247
+ {"time":"2026-08-28T14:03:31.660951165Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":212,"events_lines":2,"console_offset":47,"console_lines":1}
248
+ {"time":"2026-08-28T14:03:31.881876508Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
249
+ {"time":"2026-08-28T14:03:46.660486086Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":214,"events_lines":2}
250
+ {"time":"2026-08-28T14:03:46.859886395Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
251
+ {"time":"2026-08-28T14:04:01.660872757Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":216,"events_lines":2,"console_offset":48,"console_lines":1}
252
+ {"time":"2026-08-28T14:04:01.856826839Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
253
+ {"time":"2026-08-28T14:04:16.660703714Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":218,"events_lines":2}
254
+ {"time":"2026-08-28T14:04:16.867598918Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
255
+ {"time":"2026-08-28T14:04:31.661316899Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":220,"events_lines":2}
256
+ {"time":"2026-08-28T14:04:31.854769334Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
257
+ {"time":"2026-08-28T14:04:46.661222948Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2872,"history_lines":1,"events_offset":222,"events_lines":2,"console_offset":49,"console_lines":1}
258
+ {"time":"2026-08-28T14:04:46.879191701Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
259
+ {"time":"2026-08-28T14:05:01.660804155Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":224,"events_lines":2}
260
+ {"time":"2026-08-28T14:05:01.895751771Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
261
+ {"time":"2026-08-28T14:05:16.660919893Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2873,"history_lines":1,"events_offset":226,"events_lines":2,"console_offset":50,"console_lines":1}
262
+ {"time":"2026-08-28T14:05:16.866175634Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
263
+ {"time":"2026-08-28T14:05:31.660597526Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":228,"events_lines":2,"console_offset":51,"console_lines":1}
264
+ {"time":"2026-08-28T14:05:31.851916047Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
265
+ {"time":"2026-08-28T14:05:46.661210127Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":230,"events_lines":2,"console_offset":52,"console_lines":1}
266
+ {"time":"2026-08-28T14:05:46.850342803Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
267
+ {"time":"2026-08-28T14:06:01.660492721Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":232,"events_lines":2}
268
+ {"time":"2026-08-28T14:06:01.847993899Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
269
+ {"time":"2026-08-28T14:06:16.660995277Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2874,"history_lines":1,"events_offset":234,"events_lines":2,"console_offset":53,"console_lines":1}
270
+ {"time":"2026-08-28T14:06:17.773922804Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
271
+ {"time":"2026-08-28T14:06:31.661115531Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":236,"events_lines":2}
272
+ {"time":"2026-08-28T14:06:31.95403474Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
273
+ {"time":"2026-08-28T14:06:46.660883559Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":238,"events_lines":2}
274
+ {"time":"2026-08-28T14:06:46.867960333Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
275
+ {"time":"2026-08-28T14:07:01.661104242Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2875,"history_lines":1,"events_offset":240,"events_lines":2,"console_offset":54,"console_lines":1}
276
+ {"time":"2026-08-28T14:07:01.963073972Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
277
+ {"time":"2026-08-28T14:07:16.661315183Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":242,"events_lines":2}
278
+ {"time":"2026-08-28T14:07:16.858459521Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
279
+ {"time":"2026-08-28T14:07:31.661265599Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":244,"events_lines":2}
280
+ {"time":"2026-08-28T14:07:31.892679408Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
281
+ {"time":"2026-08-28T14:07:46.660946012Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2876,"history_lines":1,"events_offset":246,"events_lines":2,"console_offset":55,"console_lines":1}
282
+ {"time":"2026-08-28T14:07:46.901603041Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
283
+ {"time":"2026-08-28T14:08:01.660861826Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":248,"events_lines":2}
284
+ {"time":"2026-08-28T14:08:01.890080356Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
285
+ {"time":"2026-08-28T14:08:16.661025578Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2877,"history_lines":1,"events_offset":250,"events_lines":2,"console_offset":56,"console_lines":1}
286
+ {"time":"2026-08-28T14:08:16.895176651Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
287
+ {"time":"2026-08-28T14:08:31.66065701Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":252,"events_lines":2}
288
+ {"time":"2026-08-28T14:08:31.904430289Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
289
+ {"time":"2026-08-28T14:08:46.661096148Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":254,"events_lines":2}
290
+ {"time":"2026-08-28T14:08:46.870396138Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
291
+ {"time":"2026-08-28T14:09:01.661376449Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2878,"history_lines":1,"events_offset":256,"events_lines":2,"console_offset":57,"console_lines":1}
292
+ {"time":"2026-08-28T14:09:01.892421254Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
293
+ {"time":"2026-08-28T14:09:16.661149953Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":258,"events_lines":2}
294
+ {"time":"2026-08-28T14:09:16.869526174Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
295
+ {"time":"2026-08-28T14:09:31.660871054Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":260,"events_lines":2}
296
+ {"time":"2026-08-28T14:09:31.887165265Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
297
+ {"time":"2026-08-28T14:09:46.661168042Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2879,"history_lines":1,"events_offset":262,"events_lines":2,"console_offset":58,"console_lines":1}
298
+ {"time":"2026-08-28T14:09:46.872203842Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
299
+ {"time":"2026-08-28T14:10:01.661322006Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":264,"events_lines":2}
300
+ {"time":"2026-08-28T14:10:01.866412917Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
301
+ {"time":"2026-08-28T14:10:16.660857926Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2880,"history_lines":1,"events_offset":266,"events_lines":2,"console_offset":59,"console_lines":1}
302
+ {"time":"2026-08-28T14:10:17.140845713Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
303
+ {"time":"2026-08-28T14:10:31.661440453Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":268,"events_lines":2}
304
+ {"time":"2026-08-28T14:10:31.857795184Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
305
+ {"time":"2026-08-28T14:10:46.660968017Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":270,"events_lines":2}
306
+ {"time":"2026-08-28T14:10:46.878052893Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
307
+ {"time":"2026-08-28T14:11:01.661231661Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2881,"history_lines":1,"events_offset":272,"events_lines":2,"console_offset":60,"console_lines":1}
308
+ {"time":"2026-08-28T14:11:01.902098904Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
309
+ {"time":"2026-08-28T14:11:16.661226224Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":274,"events_lines":2}
310
+ {"time":"2026-08-28T14:11:16.870651865Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
311
+ {"time":"2026-08-28T14:11:31.661416642Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":276,"events_lines":2}
312
+ {"time":"2026-08-28T14:11:31.854131055Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
313
+ {"time":"2026-08-28T14:11:46.661475124Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2882,"history_lines":1,"events_offset":278,"events_lines":2,"console_offset":61,"console_lines":1}
314
+ {"time":"2026-08-28T14:11:46.922727039Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
315
+ {"time":"2026-08-28T14:12:01.661497212Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":280,"events_lines":2}
316
+ {"time":"2026-08-28T14:12:01.85365287Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
317
+ {"time":"2026-08-28T14:12:16.661571708Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2883,"history_lines":1,"events_offset":282,"events_lines":2,"console_offset":62,"console_lines":1}
318
+ {"time":"2026-08-28T14:12:17.092967029Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
319
+ {"time":"2026-08-28T14:12:31.661160658Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":284,"events_lines":2}
320
+ {"time":"2026-08-28T14:12:31.862589392Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
321
+ {"time":"2026-08-28T14:12:46.661420248Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":286,"events_lines":2}
322
+ {"time":"2026-08-28T14:12:46.855370393Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
323
+ {"time":"2026-08-28T14:13:01.66096093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2884,"history_lines":1,"events_offset":288,"events_lines":2,"console_offset":63,"console_lines":1}
324
+ {"time":"2026-08-28T14:13:01.883937955Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
325
+ {"time":"2026-08-28T14:13:16.660840248Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":290,"events_lines":2}
326
+ {"time":"2026-08-28T14:13:16.864948881Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
327
+ {"time":"2026-08-28T14:13:31.661225281Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":292,"events_lines":2}
328
+ {"time":"2026-08-28T14:13:31.904635565Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
329
+ {"time":"2026-08-28T14:13:46.660640474Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2885,"history_lines":1,"events_offset":294,"events_lines":2,"console_offset":64,"console_lines":1}
330
+ {"time":"2026-08-28T14:13:46.889048948Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
331
+ {"time":"2026-08-28T14:14:01.661070103Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":296,"events_lines":2}
332
+ {"time":"2026-08-28T14:14:01.851361517Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
333
+ {"time":"2026-08-28T14:14:16.661422508Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2886,"history_lines":1,"events_offset":298,"events_lines":2,"console_offset":65,"console_lines":1}
334
+ {"time":"2026-08-28T14:14:16.881614516Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
335
+ {"time":"2026-08-28T14:14:31.661101547Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":300,"events_lines":2}
336
+ {"time":"2026-08-28T14:14:31.846221433Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
337
+ {"time":"2026-08-28T14:14:46.660922382Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":302,"events_lines":2}
338
+ {"time":"2026-08-28T14:14:46.878038865Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
339
+ {"time":"2026-08-28T14:15:01.661054622Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2887,"history_lines":1,"events_offset":304,"events_lines":2,"console_offset":66,"console_lines":1}
340
+ {"time":"2026-08-28T14:15:08.348024614Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
341
+ {"time":"2026-08-28T14:15:16.66109145Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":306,"events_lines":2}
342
+ {"time":"2026-08-28T14:15:16.881728393Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
343
+ {"time":"2026-08-28T14:15:31.661375418Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2888,"history_lines":1,"events_offset":308,"events_lines":2,"console_offset":67,"console_lines":1}
344
+ {"time":"2026-08-28T14:15:31.883347365Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
345
+ {"time":"2026-08-28T14:15:46.661021732Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":310,"events_lines":2}
346
+ {"time":"2026-08-28T14:15:46.846488831Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
347
+ {"time":"2026-08-28T14:16:01.661045432Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":312,"events_lines":2}
348
+ {"time":"2026-08-28T14:16:01.884575782Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
349
+ {"time":"2026-08-28T14:16:16.661495389Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2889,"history_lines":1,"events_offset":314,"events_lines":2,"console_offset":68,"console_lines":1}
350
+ {"time":"2026-08-28T14:16:16.872179636Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
351
+ {"time":"2026-08-28T14:16:31.660647558Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":316,"events_lines":2}
352
+ {"time":"2026-08-28T14:16:31.877017169Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
353
+ {"time":"2026-08-28T14:16:46.66133509Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":318,"events_lines":2}
354
+ {"time":"2026-08-28T14:16:46.850700624Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
355
+ {"time":"2026-08-28T14:17:01.661426834Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2890,"history_lines":1,"events_offset":320,"events_lines":2,"console_offset":69,"console_lines":1}
356
+ {"time":"2026-08-28T14:17:01.867656077Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
357
+ {"time":"2026-08-28T14:17:16.660758067Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":322,"events_lines":2}
358
+ {"time":"2026-08-28T14:17:16.855655879Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
359
+ {"time":"2026-08-28T14:17:31.661228071Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2891,"history_lines":1,"events_offset":324,"events_lines":2,"console_offset":70,"console_lines":1}
360
+ {"time":"2026-08-28T14:17:31.886346247Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
361
+ {"time":"2026-08-28T14:17:46.660679323Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":326,"events_lines":2}
362
+ {"time":"2026-08-28T14:17:46.862640279Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
363
+ {"time":"2026-08-28T14:18:01.660983886Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":328,"events_lines":2}
364
+ {"time":"2026-08-28T14:18:01.859780189Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
365
+ {"time":"2026-08-28T14:18:16.661493673Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2892,"history_lines":1,"events_offset":330,"events_lines":2,"console_offset":71,"console_lines":1}
366
+ {"time":"2026-08-28T14:18:16.876198503Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
367
+ {"time":"2026-08-28T14:18:31.661130986Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":332,"events_lines":2}
368
+ {"time":"2026-08-28T14:18:31.873617513Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
369
+ {"time":"2026-08-28T14:18:46.66041298Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":334,"events_lines":2}
370
+ {"time":"2026-08-28T14:18:46.88521828Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
371
+ {"time":"2026-08-28T14:19:01.661206204Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2893,"history_lines":1,"events_offset":336,"events_lines":2,"console_offset":72,"console_lines":1}
372
+ {"time":"2026-08-28T14:19:01.922018943Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
373
+ {"time":"2026-08-28T14:19:16.660559201Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":338,"events_lines":2}
374
+ {"time":"2026-08-28T14:19:16.852506483Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
375
+ {"time":"2026-08-28T14:19:31.660854285Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2894,"history_lines":1,"events_offset":340,"events_lines":2,"console_offset":73,"console_lines":1}
376
+ {"time":"2026-08-28T14:19:31.879452975Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
377
+ {"time":"2026-08-28T14:19:46.661425221Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":342,"events_lines":2}
378
+ {"time":"2026-08-28T14:19:46.873361828Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
379
+ {"time":"2026-08-28T14:20:01.660984506Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":344,"events_lines":2}
380
+ {"time":"2026-08-28T14:20:01.860148621Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
381
+ {"time":"2026-08-28T14:20:16.66134186Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2895,"history_lines":1,"events_offset":346,"events_lines":2,"console_offset":74,"console_lines":1}
382
+ {"time":"2026-08-28T14:20:16.913175801Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
383
+ {"time":"2026-08-28T14:20:31.661172305Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":348,"events_lines":2}
384
+ {"time":"2026-08-28T14:20:31.864049972Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
385
+ {"time":"2026-08-28T14:20:46.660771516Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":350,"events_lines":2}
386
+ {"time":"2026-08-28T14:20:46.877823099Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
387
+ {"time":"2026-08-28T14:21:01.661142039Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2896,"history_lines":1,"events_offset":352,"events_lines":2,"console_offset":75,"console_lines":1}
388
+ {"time":"2026-08-28T14:21:01.942108076Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
389
+ {"time":"2026-08-28T14:21:16.661060243Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":354,"events_lines":2}
390
+ {"time":"2026-08-28T14:21:16.89483095Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
391
+ {"time":"2026-08-28T14:21:31.661291961Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2897,"history_lines":1,"events_offset":356,"events_lines":2,"console_offset":76,"console_lines":1}
392
+ {"time":"2026-08-28T14:21:31.889364785Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
393
+ {"time":"2026-08-28T14:21:46.66130326Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":358,"events_lines":2}
394
+ {"time":"2026-08-28T14:21:46.858487801Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
395
+ {"time":"2026-08-28T14:22:01.661173309Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":360,"events_lines":2}
396
+ {"time":"2026-08-28T14:22:01.859909678Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
397
+ {"time":"2026-08-28T14:22:16.661213811Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2898,"history_lines":1,"events_offset":362,"events_lines":2,"console_offset":77,"console_lines":1}
398
+ {"time":"2026-08-28T14:22:16.888455083Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
399
+ {"time":"2026-08-28T14:22:31.661122848Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":364,"events_lines":2,"console_offset":78,"console_lines":2}
400
+ {"time":"2026-08-28T14:22:31.882364835Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
401
+ {"time":"2026-08-28T14:22:46.660785085Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":366,"events_lines":2}
402
+ {"time":"2026-08-28T14:22:46.855456321Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
403
+ {"time":"2026-08-28T14:23:01.660344039Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":368,"events_lines":2}
404
+ {"time":"2026-08-28T14:23:01.892818009Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
405
+ {"time":"2026-08-28T14:23:16.660901083Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2899,"history_lines":1,"events_offset":370,"events_lines":2,"console_offset":80,"console_lines":1}
406
+ {"time":"2026-08-28T14:23:16.877991047Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
407
+ {"time":"2026-08-28T14:23:31.66076469Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":372,"events_lines":2}
408
+ {"time":"2026-08-28T14:23:31.860517288Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
409
+ {"time":"2026-08-28T14:23:46.661232095Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":374,"events_lines":2}
410
+ {"time":"2026-08-28T14:23:46.872268314Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
411
+ {"time":"2026-08-28T14:24:01.661171132Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2900,"history_lines":1,"events_offset":376,"events_lines":2,"console_offset":81,"console_lines":1}
412
+ {"time":"2026-08-28T14:24:01.901541321Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
413
+ {"time":"2026-08-28T14:24:16.661306261Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":378,"events_lines":2}
414
+ {"time":"2026-08-28T14:24:16.851162082Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
415
+ {"time":"2026-08-28T14:24:31.660768336Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2901,"history_lines":1,"events_offset":380,"events_lines":2,"console_offset":82,"console_lines":1}
416
+ {"time":"2026-08-28T14:24:31.903341913Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
417
+ {"time":"2026-08-28T14:24:46.660735845Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":382,"events_lines":2}
418
+ {"time":"2026-08-28T14:24:46.851914016Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
419
+ {"time":"2026-08-28T14:25:01.66107547Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":384,"events_lines":2}
420
+ {"time":"2026-08-28T14:25:01.859385968Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
421
+ {"time":"2026-08-28T14:25:16.660957531Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2902,"history_lines":1,"events_offset":386,"events_lines":2,"console_offset":83,"console_lines":1}
422
+ {"time":"2026-08-28T14:25:16.901611119Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
423
+ {"time":"2026-08-28T14:25:31.660924349Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":388,"events_lines":2}
424
+ {"time":"2026-08-28T14:25:31.852930844Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
425
+ {"time":"2026-08-28T14:25:46.660854035Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":390,"events_lines":2}
426
+ {"time":"2026-08-28T14:25:46.847675061Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
427
+ {"time":"2026-08-28T14:26:01.660650602Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2903,"history_lines":1,"events_offset":392,"events_lines":2,"console_offset":84,"console_lines":1}
428
+ {"time":"2026-08-28T14:26:01.876181184Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
429
+ {"time":"2026-08-28T14:26:16.661052463Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":394,"events_lines":2}
430
+ {"time":"2026-08-28T14:26:16.84476823Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
431
+ {"time":"2026-08-28T14:26:31.661029239Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2904,"history_lines":1,"events_offset":396,"events_lines":2,"console_offset":85,"console_lines":1}
432
+ {"time":"2026-08-28T14:26:31.881621555Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
433
+ {"time":"2026-08-28T14:26:46.6609888Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":398,"events_lines":2}
434
+ {"time":"2026-08-28T14:26:46.876665101Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
435
+ {"time":"2026-08-28T14:27:01.661479187Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":400,"events_lines":2}
436
+ {"time":"2026-08-28T14:27:01.858987946Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
437
+ {"time":"2026-08-28T14:27:16.660921578Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2905,"history_lines":1,"events_offset":402,"events_lines":2,"console_offset":86,"console_lines":1}
438
+ {"time":"2026-08-28T14:27:16.864643038Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
439
+ {"time":"2026-08-28T14:27:31.661308019Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":404,"events_lines":2}
440
+ {"time":"2026-08-28T14:27:31.880107797Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-28 13:36:29,853 INFO MainThread:198 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-28 13:36:29,853 INFO MainThread:198 [wandb_setup.py:_flush():81] Configure stats pid to 198
3
+ 2026-08-28 13:36:29,853 INFO MainThread:198 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-28 13:36:29,853 INFO MainThread:198 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_133629-8ynn1we6/logs/debug.log
5
+ 2026-08-28 13:36:29,853 INFO MainThread:198 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_133629-8ynn1we6/logs/debug-internal.log
6
+ 2026-08-28 13:36:29,853 INFO MainThread:198 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-28 13:36:29,853 INFO MainThread:198 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-28500', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 71, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed71 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 71, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-28 13:36:29,853 INFO MainThread:198 [wandb_init.py:init():898] starting backend
10
+ 2026-08-28 13:36:30,601 INFO MainThread:198 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-28 13:36:31,045 INFO MainThread:198 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-28 13:36:31,052 INFO MainThread:198 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-28 13:36:31,082 INFO MainThread:198 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-28 13:36:31,656 INFO MainThread:198 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-28 13:36:31,658 INFO MainThread:198 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-28 13:36:31,947 INFO MainThread:198 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-28 13:36:31,947 INFO MainThread:198 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-28 13:36:31,947 INFO MainThread:198 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-28 13:36:31,947 INFO MainThread:198 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-28 13:36:31,970 INFO MainThread:198 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260828_133629-8ynn1we6/run-8ynn1we6.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:41a407a46dc50b9af6686b90b85264bed88a59a93dbf88d61f651dcd88c1af76
3
+ size 950272
epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/files/output.log ADDED
@@ -0,0 +1,234 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 23250: Loss = 2.6326, Perplexity = 13.91
3
+ wandb: WARNING Tried to log to step 23250 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 23251: sparse_stochastic (configured sparse attention)
5
+ Step 23260/50462 | Loss: 2.6253 | LR: 2.87e-04 | 80261.3ms/step | 0.7 samples/s
6
+ wandb: WARNING Tried to log to step 23260 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 23270/50462 | Loss: 2.5993 | LR: 2.87e-04 | 4070.3ms/step | 14.7 samples/s
8
+ wandb: WARNING Tried to log to step 23270 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 23280/50462 | Loss: 2.5940 | LR: 2.87e-04 | 4070.5ms/step | 14.7 samples/s
10
+ wandb: WARNING Tried to log to step 23280 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 23290/50462 | Loss: 2.6180 | LR: 2.86e-04 | 4084.0ms/step | 14.7 samples/s
12
+ wandb: WARNING Tried to log to step 23290 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 23300/50462 | Loss: 2.6354 | LR: 2.86e-04 | 4075.9ms/step | 14.7 samples/s
14
+ wandb: WARNING Tried to log to step 23300 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 23310/50462 | Loss: 2.5866 | LR: 2.86e-04 | 4067.4ms/step | 14.8 samples/s
16
+ wandb: WARNING Tried to log to step 23310 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 23320/50462 | Loss: 2.6033 | LR: 2.86e-04 | 4075.2ms/step | 14.7 samples/s
18
+ wandb: WARNING Tried to log to step 23320 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
19
+ Step 23330/50462 | Loss: 2.6093 | LR: 2.86e-04 | 4076.5ms/step | 14.7 samples/s
20
+ wandb: WARNING Tried to log to step 23330 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
21
+ Step 23340/50462 | Loss: 2.5829 | LR: 2.86e-04 | 4077.9ms/step | 14.7 samples/s
22
+ wandb: WARNING Tried to log to step 23340 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
23
+ Step 23350/50462 | Loss: 2.6551 | LR: 2.86e-04 | 4092.4ms/step | 14.7 samples/s
24
+ wandb: WARNING Tried to log to step 23350 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
25
+ Step 23360/50462 | Loss: 2.5932 | LR: 2.85e-04 | 7608.3ms/step | 7.9 samples/s
26
+ wandb: WARNING Tried to log to step 23360 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
27
+ Step 23370/50462 | Loss: 2.6252 | LR: 2.85e-04 | 12642.8ms/step | 4.7 samples/s
28
+ wandb: WARNING Tried to log to step 23370 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
29
+ Step 23380/50462 | Loss: 2.5743 | LR: 2.85e-04 | 12113.3ms/step | 5.0 samples/s
30
+ wandb: WARNING Tried to log to step 23380 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
31
+ Step 23390/50462 | Loss: 2.6239 | LR: 2.85e-04 | 11988.2ms/step | 5.0 samples/s
32
+ wandb: WARNING Tried to log to step 23390 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
33
+ Step 23400/50462 | Loss: 2.5765 | LR: 2.85e-04 | 11945.6ms/step | 5.0 samples/s
34
+ wandb: WARNING Tried to log to step 23400 that is less than the current step 23401. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
35
+ Step 23410/50462 | Loss: 2.6190 | LR: 2.85e-04 | 11844.8ms/step | 5.1 samples/s
36
+ Step 23420/50462 | Loss: 2.5710 | LR: 2.84e-04 | 12096.3ms/step | 5.0 samples/s
37
+ Step 23430/50462 | Loss: 2.6226 | LR: 2.84e-04 | 11941.3ms/step | 5.0 samples/s
38
+ Step 23440/50462 | Loss: 2.6118 | LR: 2.84e-04 | 11778.9ms/step | 5.1 samples/s
39
+ Step 23450/50462 | Loss: 2.6270 | LR: 2.84e-04 | 12013.1ms/step | 5.0 samples/s
40
+ Step 23460/50462 | Loss: 2.6307 | LR: 2.84e-04 | 11944.9ms/step | 5.0 samples/s
41
+ Step 23470/50462 | Loss: 2.6120 | LR: 2.84e-04 | 12128.2ms/step | 4.9 samples/s
42
+ Step 23480/50462 | Loss: 2.6121 | LR: 2.83e-04 | 12104.6ms/step | 5.0 samples/s
43
+ Step 23490/50462 | Loss: 2.6154 | LR: 2.83e-04 | 12127.0ms/step | 4.9 samples/s
44
+ Step 23500/50462 | Loss: 2.6215 | LR: 2.83e-04 | 12148.3ms/step | 4.9 samples/s
45
+ Validation at step 23500: Loss = 2.6301, Perplexity = 13.88
46
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23500
47
+ Step 23510/50462 | Loss: 2.6426 | LR: 2.83e-04 | 17277.9ms/step | 3.5 samples/s
48
+ Step 23520/50462 | Loss: 2.6220 | LR: 2.83e-04 | 12106.3ms/step | 5.0 samples/s
49
+ Step 23530/50462 | Loss: 2.5945 | LR: 2.83e-04 | 12221.3ms/step | 4.9 samples/s
50
+ Step 23540/50462 | Loss: 2.6364 | LR: 2.83e-04 | 12031.3ms/step | 5.0 samples/s
51
+ Step 23550/50462 | Loss: 2.6192 | LR: 2.82e-04 | 12154.0ms/step | 4.9 samples/s
52
+ Step 23560/50462 | Loss: 2.6352 | LR: 2.82e-04 | 12085.8ms/step | 5.0 samples/s
53
+ Step 23570/50462 | Loss: 2.6031 | LR: 2.82e-04 | 11980.8ms/step | 5.0 samples/s
54
+ Step 23580/50462 | Loss: 2.6069 | LR: 2.82e-04 | 11999.9ms/step | 5.0 samples/s
55
+ Step 23590/50462 | Loss: 2.6162 | LR: 2.82e-04 | 11854.0ms/step | 5.1 samples/s
56
+ Step 23600/50462 | Loss: 2.6115 | LR: 2.82e-04 | 12081.8ms/step | 5.0 samples/s
57
+ Step 23610/50462 | Loss: 2.6253 | LR: 2.81e-04 | 9508.5ms/step | 6.3 samples/s
58
+ Step 23620/50462 | Loss: 2.6078 | LR: 2.81e-04 | 12246.2ms/step | 4.9 samples/s
59
+ Step 23630/50462 | Loss: 2.6459 | LR: 2.81e-04 | 11993.7ms/step | 5.0 samples/s
60
+ Step 23640/50462 | Loss: 2.6063 | LR: 2.81e-04 | 12061.3ms/step | 5.0 samples/s
61
+ Step 23650/50462 | Loss: 2.6141 | LR: 2.81e-04 | 12042.5ms/step | 5.0 samples/s
62
+ Step 23660/50462 | Loss: 2.5993 | LR: 2.81e-04 | 12004.0ms/step | 5.0 samples/s
63
+ Step 23670/50462 | Loss: 2.6249 | LR: 2.81e-04 | 11903.6ms/step | 5.0 samples/s
64
+ Step 23680/50462 | Loss: 2.6420 | LR: 2.80e-04 | 12011.4ms/step | 5.0 samples/s
65
+ Step 23690/50462 | Loss: 2.6115 | LR: 2.80e-04 | 12237.4ms/step | 4.9 samples/s
66
+ Step 23700/50462 | Loss: 2.6102 | LR: 2.80e-04 | 11687.4ms/step | 5.1 samples/s
67
+ Step 23710/50462 | Loss: 2.6101 | LR: 2.80e-04 | 11889.5ms/step | 5.0 samples/s
68
+ Step 23720/50462 | Loss: 2.6356 | LR: 2.80e-04 | 12187.7ms/step | 4.9 samples/s
69
+ Step 23730/50462 | Loss: 2.6235 | LR: 2.80e-04 | 12027.9ms/step | 5.0 samples/s
70
+ Step 23740/50462 | Loss: 2.6183 | LR: 2.79e-04 | 11960.0ms/step | 5.0 samples/s
71
+ Step 23750/50462 | Loss: 2.6015 | LR: 2.79e-04 | 12163.0ms/step | 4.9 samples/s
72
+ Validation at step 23750: Loss = 2.6280, Perplexity = 13.85
73
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23750
74
+ Step 23760/50462 | Loss: 2.6270 | LR: 2.79e-04 | 17182.5ms/step | 3.5 samples/s
75
+ Step 23770/50462 | Loss: 2.6084 | LR: 2.79e-04 | 11838.3ms/step | 5.1 samples/s
76
+ Step 23780/50462 | Loss: 2.6061 | LR: 2.79e-04 | 12140.5ms/step | 4.9 samples/s
77
+ Step 23790/50462 | Loss: 2.5800 | LR: 2.79e-04 | 11911.8ms/step | 5.0 samples/s
78
+ Step 23800/50462 | Loss: 2.5625 | LR: 2.78e-04 | 11878.8ms/step | 5.1 samples/s
79
+ Step 23810/50462 | Loss: 2.6035 | LR: 2.78e-04 | 12143.5ms/step | 4.9 samples/s
80
+ Step 23820/50462 | Loss: 2.5922 | LR: 2.78e-04 | 11855.3ms/step | 5.1 samples/s
81
+ Step 23830/50462 | Loss: 2.6080 | LR: 2.78e-04 | 12000.6ms/step | 5.0 samples/s
82
+ Step 23840/50462 | Loss: 2.5943 | LR: 2.78e-04 | 11885.3ms/step | 5.0 samples/s
83
+ Step 23850/50462 | Loss: 2.6118 | LR: 2.78e-04 | 11897.3ms/step | 5.0 samples/s
84
+ Step 23860/50462 | Loss: 2.6130 | LR: 2.78e-04 | 9456.2ms/step | 6.3 samples/s
85
+ Step 23870/50462 | Loss: 2.6200 | LR: 2.77e-04 | 11828.7ms/step | 5.1 samples/s
86
+ Step 23880/50462 | Loss: 2.6045 | LR: 2.77e-04 | 11933.2ms/step | 5.0 samples/s
87
+ Step 23890/50462 | Loss: 2.6396 | LR: 2.77e-04 | 12110.2ms/step | 5.0 samples/s
88
+ Step 23900/50462 | Loss: 2.6033 | LR: 2.77e-04 | 11972.5ms/step | 5.0 samples/s
89
+ Step 23910/50462 | Loss: 2.6191 | LR: 2.77e-04 | 11834.6ms/step | 5.1 samples/s
90
+ Step 23920/50462 | Loss: 2.5841 | LR: 2.77e-04 | 11788.6ms/step | 5.1 samples/s
91
+ Step 23930/50462 | Loss: 2.5913 | LR: 2.76e-04 | 12339.2ms/step | 4.9 samples/s
92
+ Step 23940/50462 | Loss: 2.6208 | LR: 2.76e-04 | 11801.0ms/step | 5.1 samples/s
93
+ Step 23950/50462 | Loss: 2.5853 | LR: 2.76e-04 | 11890.6ms/step | 5.0 samples/s
94
+ Step 23960/50462 | Loss: 2.6300 | LR: 2.76e-04 | 11911.9ms/step | 5.0 samples/s
95
+ Step 23970/50462 | Loss: 2.6000 | LR: 2.76e-04 | 11882.7ms/step | 5.0 samples/s
96
+ Step 23980/50462 | Loss: 2.5970 | LR: 2.76e-04 | 12110.1ms/step | 5.0 samples/s
97
+ Step 23990/50462 | Loss: 2.5902 | LR: 2.75e-04 | 11856.5ms/step | 5.1 samples/s
98
+ Step 24000/50462 | Loss: 2.5884 | LR: 2.75e-04 | 11915.1ms/step | 5.0 samples/s
99
+ Validation at step 24000: Loss = 2.6265, Perplexity = 13.83
100
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-24000
101
+ Step 24010/50462 | Loss: 2.6338 | LR: 2.75e-04 | 17281.5ms/step | 3.5 samples/s
102
+ Step 24020/50462 | Loss: 2.6027 | LR: 2.75e-04 | 11859.8ms/step | 5.1 samples/s
103
+ Step 24030/50462 | Loss: 2.6219 | LR: 2.75e-04 | 11986.4ms/step | 5.0 samples/s
104
+ Step 24040/50462 | Loss: 2.6319 | LR: 2.75e-04 | 11926.7ms/step | 5.0 samples/s
105
+ Step 24050/50462 | Loss: 2.6178 | LR: 2.75e-04 | 11995.7ms/step | 5.0 samples/s
106
+ Step 24060/50462 | Loss: 2.5977 | LR: 2.74e-04 | 12094.5ms/step | 5.0 samples/s
107
+ Step 24070/50462 | Loss: 2.6420 | LR: 2.74e-04 | 12367.1ms/step | 4.9 samples/s
108
+ Step 24080/50462 | Loss: 2.5995 | LR: 2.74e-04 | 12079.6ms/step | 5.0 samples/s
109
+ Step 24090/50462 | Loss: 2.5821 | LR: 2.74e-04 | 12224.9ms/step | 4.9 samples/s
110
+ Step 24100/50462 | Loss: 2.6065 | LR: 2.74e-04 | 12132.9ms/step | 4.9 samples/s
111
+ Step 24110/50462 | Loss: 2.6058 | LR: 2.74e-04 | 9639.6ms/step | 6.2 samples/s
112
+ Step 24120/50462 | Loss: 2.6061 | LR: 2.73e-04 | 11938.3ms/step | 5.0 samples/s
113
+ Step 24130/50462 | Loss: 2.6218 | LR: 2.73e-04 | 11748.1ms/step | 5.1 samples/s
114
+ Step 24140/50462 | Loss: 2.6104 | LR: 2.73e-04 | 11806.0ms/step | 5.1 samples/s
115
+ Step 24150/50462 | Loss: 2.6216 | LR: 2.73e-04 | 11893.2ms/step | 5.0 samples/s
116
+ Step 24160/50462 | Loss: 2.5975 | LR: 2.73e-04 | 11797.4ms/step | 5.1 samples/s
117
+ Step 24170/50462 | Loss: 2.5691 | LR: 2.73e-04 | 12064.2ms/step | 5.0 samples/s
118
+ Step 24180/50462 | Loss: 2.6040 | LR: 2.72e-04 | 11933.7ms/step | 5.0 samples/s
119
+ Step 24190/50462 | Loss: 2.6052 | LR: 2.72e-04 | 12030.9ms/step | 5.0 samples/s
120
+ Step 24200/50462 | Loss: 2.6000 | LR: 2.72e-04 | 12020.9ms/step | 5.0 samples/s
121
+ Step 24210/50462 | Loss: 2.5935 | LR: 2.72e-04 | 11891.3ms/step | 5.0 samples/s
122
+ Step 24220/50462 | Loss: 2.5901 | LR: 2.72e-04 | 11890.3ms/step | 5.0 samples/s
123
+ Step 24230/50462 | Loss: 2.5935 | LR: 2.72e-04 | 12220.5ms/step | 4.9 samples/s
124
+ Step 24240/50462 | Loss: 2.6079 | LR: 2.72e-04 | 11877.2ms/step | 5.1 samples/s
125
+ Step 24250/50462 | Loss: 2.6220 | LR: 2.71e-04 | 12187.8ms/step | 4.9 samples/s
126
+ Validation at step 24250: Loss = 2.6239, Perplexity = 13.79
127
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-24250
128
+ Step 24260/50462 | Loss: 2.6042 | LR: 2.71e-04 | 17048.3ms/step | 3.5 samples/s
129
+ Step 24270/50462 | Loss: 2.5951 | LR: 2.71e-04 | 12094.9ms/step | 5.0 samples/s
130
+ Step 24280/50462 | Loss: 2.5874 | LR: 2.71e-04 | 12029.4ms/step | 5.0 samples/s
131
+ Step 24290/50462 | Loss: 2.6175 | LR: 2.71e-04 | 11879.0ms/step | 5.1 samples/s
132
+ Step 24300/50462 | Loss: 2.6419 | LR: 2.71e-04 | 11819.8ms/step | 5.1 samples/s
133
+ Step 24310/50462 | Loss: 2.5845 | LR: 2.70e-04 | 11832.2ms/step | 5.1 samples/s
134
+ Step 24320/50462 | Loss: 2.6221 | LR: 2.70e-04 | 12292.5ms/step | 4.9 samples/s
135
+ Step 24330/50462 | Loss: 2.5635 | LR: 2.70e-04 | 11997.6ms/step | 5.0 samples/s
136
+ Step 24340/50462 | Loss: 2.6121 | LR: 2.70e-04 | 11967.0ms/step | 5.0 samples/s
137
+ Step 24350/50462 | Loss: 2.5734 | LR: 2.70e-04 | 12077.9ms/step | 5.0 samples/s
138
+ Step 24360/50462 | Loss: 2.6030 | LR: 2.70e-04 | 9483.4ms/step | 6.3 samples/s
139
+ Step 24370/50462 | Loss: 2.6186 | LR: 2.70e-04 | 12161.8ms/step | 4.9 samples/s
140
+ Step 24380/50462 | Loss: 2.6137 | LR: 2.69e-04 | 12198.6ms/step | 4.9 samples/s
141
+ Step 24390/50462 | Loss: 2.6172 | LR: 2.69e-04 | 11936.2ms/step | 5.0 samples/s
142
+ Step 24400/50462 | Loss: 2.6392 | LR: 2.69e-04 | 11776.2ms/step | 5.1 samples/s
143
+ Step 24410/50462 | Loss: 2.5887 | LR: 2.69e-04 | 11866.7ms/step | 5.1 samples/s
144
+ Step 24420/50462 | Loss: 2.6059 | LR: 2.69e-04 | 11838.4ms/step | 5.1 samples/s
145
+ Step 24430/50462 | Loss: 2.6020 | LR: 2.69e-04 | 12061.9ms/step | 5.0 samples/s
146
+ Step 24440/50462 | Loss: 2.5962 | LR: 2.68e-04 | 12016.5ms/step | 5.0 samples/s
147
+ Step 24450/50462 | Loss: 2.6325 | LR: 2.68e-04 | 11918.1ms/step | 5.0 samples/s
148
+ Step 24460/50462 | Loss: 2.5924 | LR: 2.68e-04 | 11831.5ms/step | 5.1 samples/s
149
+ Step 24470/50462 | Loss: 2.5972 | LR: 2.68e-04 | 11982.6ms/step | 5.0 samples/s
150
+ Step 24480/50462 | Loss: 2.6074 | LR: 2.68e-04 | 12642.1ms/step | 4.7 samples/s
151
+ Step 24490/50462 | Loss: 2.5694 | LR: 2.68e-04 | 12046.6ms/step | 5.0 samples/s
152
+ Step 24500/50462 | Loss: 2.6091 | LR: 2.67e-04 | 12059.1ms/step | 5.0 samples/s
153
+ Validation at step 24500: Loss = 2.6215, Perplexity = 13.76
154
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-24500
155
+ Step 24510/50462 | Loss: 2.6294 | LR: 2.67e-04 | 17120.7ms/step | 3.5 samples/s
156
+ Step 24520/50462 | Loss: 2.5950 | LR: 2.67e-04 | 11895.7ms/step | 5.0 samples/s
157
+ Step 24530/50462 | Loss: 2.5876 | LR: 2.67e-04 | 11586.1ms/step | 5.2 samples/s
158
+ Step 24540/50462 | Loss: 2.6070 | LR: 2.67e-04 | 11928.6ms/step | 5.0 samples/s
159
+ Step 24550/50462 | Loss: 2.6119 | LR: 2.67e-04 | 12012.3ms/step | 5.0 samples/s
160
+ Step 24560/50462 | Loss: 2.6301 | LR: 2.67e-04 | 11872.3ms/step | 5.1 samples/s
161
+ Step 24570/50462 | Loss: 2.5883 | LR: 2.66e-04 | 11877.8ms/step | 5.1 samples/s
162
+ Step 24580/50462 | Loss: 2.6169 | LR: 2.66e-04 | 11667.7ms/step | 5.1 samples/s
163
+ Step 24590/50462 | Loss: 2.5993 | LR: 2.66e-04 | 11986.7ms/step | 5.0 samples/s
164
+ Step 24600/50462 | Loss: 2.5934 | LR: 2.66e-04 | 12383.3ms/step | 4.8 samples/s
165
+ Step 24610/50462 | Loss: 2.6329 | LR: 2.66e-04 | 9442.0ms/step | 6.4 samples/s
166
+ Step 24620/50462 | Loss: 2.6142 | LR: 2.66e-04 | 12208.7ms/step | 4.9 samples/s
167
+ Step 24630/50462 | Loss: 2.5341 | LR: 2.65e-04 | 12087.7ms/step | 5.0 samples/s
168
+ Step 24640/50462 | Loss: 2.5782 | LR: 2.65e-04 | 11995.2ms/step | 5.0 samples/s
169
+ Step 24650/50462 | Loss: 2.6035 | LR: 2.65e-04 | 11987.3ms/step | 5.0 samples/s
170
+ Step 24660/50462 | Loss: 2.6002 | LR: 2.65e-04 | 11928.6ms/step | 5.0 samples/s
171
+ Step 24670/50462 | Loss: 2.6280 | LR: 2.65e-04 | 12468.0ms/step | 4.8 samples/s
172
+ Step 24680/50462 | Loss: 2.6387 | LR: 2.65e-04 | 11915.8ms/step | 5.0 samples/s
173
+ Step 24690/50462 | Loss: 2.5985 | LR: 2.64e-04 | 12109.5ms/step | 5.0 samples/s
174
+ Step 24700/50462 | Loss: 2.5447 | LR: 2.64e-04 | 12129.3ms/step | 4.9 samples/s
175
+ Step 24710/50462 | Loss: 2.6018 | LR: 2.64e-04 | 11922.3ms/step | 5.0 samples/s
176
+ Step 24720/50462 | Loss: 2.5804 | LR: 2.64e-04 | 11839.3ms/step | 5.1 samples/s
177
+ Step 24730/50462 | Loss: 2.5811 | LR: 2.64e-04 | 12104.3ms/step | 5.0 samples/s
178
+ Step 24740/50462 | Loss: 2.5862 | LR: 2.64e-04 | 12188.6ms/step | 4.9 samples/s
179
+ Step 24750/50462 | Loss: 2.6143 | LR: 2.64e-04 | 11888.8ms/step | 5.0 samples/s
180
+ Validation at step 24750: Loss = 2.6193, Perplexity = 13.73
181
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-24750
182
+ Step 24760/50462 | Loss: 2.5857 | LR: 2.63e-04 | 17248.4ms/step | 3.5 samples/s
183
+ Step 24770/50462 | Loss: 2.5990 | LR: 2.63e-04 | 11850.8ms/step | 5.1 samples/s
184
+ Step 24780/50462 | Loss: 2.5961 | LR: 2.63e-04 | 11824.7ms/step | 5.1 samples/s
185
+ Step 24790/50462 | Loss: 2.6206 | LR: 2.63e-04 | 12014.0ms/step | 5.0 samples/s
186
+ Step 24800/50462 | Loss: 2.5585 | LR: 2.63e-04 | 11987.8ms/step | 5.0 samples/s
187
+ Step 24810/50462 | Loss: 2.5824 | LR: 2.63e-04 | 11995.2ms/step | 5.0 samples/s
188
+ Step 24820/50462 | Loss: 2.5922 | LR: 2.62e-04 | 11928.5ms/step | 5.0 samples/s
189
+ Step 24830/50462 | Loss: 2.6009 | LR: 2.62e-04 | 12112.2ms/step | 5.0 samples/s
190
+ Step 24840/50462 | Loss: 2.6099 | LR: 2.62e-04 | 12171.5ms/step | 4.9 samples/s
191
+ Step 24850/50462 | Loss: 2.5924 | LR: 2.62e-04 | 11969.1ms/step | 5.0 samples/s
192
+ Step 24860/50462 | Loss: 2.5817 | LR: 2.62e-04 | 9445.9ms/step | 6.4 samples/s
193
+ Step 24870/50462 | Loss: 2.6098 | LR: 2.62e-04 | 11944.1ms/step | 5.0 samples/s
194
+ Step 24880/50462 | Loss: 2.5697 | LR: 2.61e-04 | 12182.9ms/step | 4.9 samples/s
195
+ Step 24890/50462 | Loss: 2.6025 | LR: 2.61e-04 | 11843.5ms/step | 5.1 samples/s
196
+ Step 24900/50462 | Loss: 2.6044 | LR: 2.61e-04 | 12171.4ms/step | 4.9 samples/s
197
+ Step 24910/50462 | Loss: 2.6097 | LR: 2.61e-04 | 11828.4ms/step | 5.1 samples/s
198
+ Step 24920/50462 | Loss: 2.6129 | LR: 2.61e-04 | 12051.1ms/step | 5.0 samples/s
199
+ Step 24930/50462 | Loss: 2.6081 | LR: 2.61e-04 | 11853.0ms/step | 5.1 samples/s
200
+ Step 24940/50462 | Loss: 2.6027 | LR: 2.61e-04 | 12009.5ms/step | 5.0 samples/s
201
+ Step 24950/50462 | Loss: 2.5778 | LR: 2.60e-04 | 11948.4ms/step | 5.0 samples/s
202
+ Step 24960/50462 | Loss: 2.5854 | LR: 2.60e-04 | 11925.8ms/step | 5.0 samples/s
203
+ Step 24970/50462 | Loss: 2.5831 | LR: 2.60e-04 | 12251.1ms/step | 4.9 samples/s
204
+ Step 24980/50462 | Loss: 2.5815 | LR: 2.60e-04 | 12275.8ms/step | 4.9 samples/s
205
+ Step 24990/50462 | Loss: 2.6082 | LR: 2.60e-04 | 11795.7ms/step | 5.1 samples/s
206
+ Step 25000/50462 | Loss: 2.6009 | LR: 2.60e-04 | 12407.4ms/step | 4.8 samples/s
207
+ Validation at step 25000: Loss = 2.6164, Perplexity = 13.69
208
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25000
209
+ Step 25010/50462 | Loss: 2.6280 | LR: 2.59e-04 | 16953.9ms/step | 3.5 samples/s
210
+ Step 25020/50462 | Loss: 2.5512 | LR: 2.59e-04 | 11947.6ms/step | 5.0 samples/s
211
+ Step 25030/50462 | Loss: 2.5974 | LR: 2.59e-04 | 12531.8ms/step | 4.8 samples/s
212
+ Step 25040/50462 | Loss: 2.5898 | LR: 2.59e-04 | 12009.5ms/step | 5.0 samples/s
213
+ Step 25050/50462 | Loss: 2.6044 | LR: 2.59e-04 | 9394.8ms/step | 6.4 samples/s
214
+ Step 25060/50462 | Loss: 2.5819 | LR: 2.59e-04 | 4007.8ms/step | 15.0 samples/s
215
+ Step 25070/50462 | Loss: 2.6098 | LR: 2.58e-04 | 4014.2ms/step | 14.9 samples/s
216
+ Step 25080/50462 | Loss: 2.6072 | LR: 2.58e-04 | 4019.8ms/step | 14.9 samples/s
217
+ Step 25090/50462 | Loss: 2.5939 | LR: 2.58e-04 | 4010.8ms/step | 15.0 samples/s
218
+ Step 25100/50462 | Loss: 2.5698 | LR: 2.58e-04 | 4016.8ms/step | 14.9 samples/s
219
+ Step 25110/50462 | Loss: 2.5632 | LR: 2.58e-04 | 4017.5ms/step | 14.9 samples/s
220
+ Step 25120/50462 | Loss: 2.5901 | LR: 2.58e-04 | 4010.2ms/step | 15.0 samples/s
221
+ Step 25130/50462 | Loss: 2.6156 | LR: 2.58e-04 | 4005.3ms/step | 15.0 samples/s
222
+ Step 25140/50462 | Loss: 2.6016 | LR: 2.57e-04 | 4006.9ms/step | 15.0 samples/s
223
+ Step 25150/50462 | Loss: 2.5955 | LR: 2.57e-04 | 4010.3ms/step | 15.0 samples/s
224
+ Step 25160/50462 | Loss: 2.5763 | LR: 2.57e-04 | 4008.5ms/step | 15.0 samples/s
225
+ Step 25170/50462 | Loss: 2.6167 | LR: 2.57e-04 | 4015.3ms/step | 14.9 samples/s
226
+ Step 25180/50462 | Loss: 2.5570 | LR: 2.57e-04 | 4011.2ms/step | 15.0 samples/s
227
+ Step 25190/50462 | Loss: 2.5725 | LR: 2.57e-04 | 4016.8ms/step | 14.9 samples/s
228
+ Step 25200/50462 | Loss: 2.6080 | LR: 2.56e-04 | 4009.8ms/step | 15.0 samples/s
229
+ Step 25210/50462 | Loss: 2.5904 | LR: 2.56e-04 | 4006.5ms/step | 15.0 samples/s
230
+ Step 25220/50462 | Loss: 2.5918 | LR: 2.56e-04 | 4011.7ms/step | 15.0 samples/s
231
+ Step 25230/50462 | Loss: 2.5817 | LR: 2.56e-04 | 4018.0ms/step | 14.9 samples/s
232
+ Step 25240/50462 | Loss: 2.5974 | LR: 2.56e-04 | 4010.5ms/step | 15.0 samples/s
233
+ Step 25250/50462 | Loss: 2.5872 | LR: 2.56e-04 | 4015.5ms/step | 14.9 samples/s
234
+ Validation at step 25250: Loss = 2.6152, Perplexity = 13.67
epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/files/wandb-metadata.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-28T21:45:24.452397Z",
5
+ "args": [
6
+ "--config",
7
+ "configs/p2_1p2b_mixed_seed72_4gpu_autoresume_current.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/SparseAtt.git",
14
+ "commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-4-gpu-3-0-14",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 4,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "6072048775168"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-e3358200-f776-7214-8269-c2c219f03c2e"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-788fbe73-59b3-e845-80c4-5eff0b2f64b5"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-123c77ea-9377-5bde-70b6-99ef1c791ff4"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-a3a94461-36d7-5e8a-af9b-6209869fa7f4"
61
+ }
62
+ ],
63
+ "cudaVersion": "13.0",
64
+ "writerId": "czr5778z4srpftl4x3sge29do5ngtdfi"
65
+ }
epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-28 21:45:24,509 INFO MainThread:233 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-28 21:45:24,509 INFO MainThread:233 [wandb_setup.py:_flush():81] Configure stats pid to 233
3
+ 2026-08-28 21:45:24,509 INFO MainThread:233 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-28 21:45:24,509 INFO MainThread:233 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_214524-z1i60278/logs/debug.log
5
+ 2026-08-28 21:45:24,509 INFO MainThread:233 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_214524-z1i60278/logs/debug-internal.log
6
+ 2026-08-28 21:45:24,509 INFO MainThread:233 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-28 21:45:24,509 INFO MainThread:233 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-23250', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 72, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed72 6gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed72'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 72, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-28 21:45:24,509 INFO MainThread:233 [wandb_init.py:init():898] starting backend
10
+ 2026-08-28 21:45:25,516 INFO MainThread:233 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-28 21:45:26,283 INFO MainThread:233 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-28 21:45:26,286 INFO MainThread:233 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-28 21:45:26,325 INFO MainThread:233 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-28 21:45:26,938 INFO MainThread:233 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-28 21:45:26,940 INFO MainThread:233 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-28 21:45:27,319 INFO MainThread:233 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-28 21:45:27,319 INFO MainThread:233 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-28 21:45:27,319 INFO MainThread:233 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-28 21:45:27,319 INFO MainThread:233 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-28 21:45:27,331 INFO MainThread:233 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260828_214524-z1i60278/run-z1i60278.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:03d435b26e2d9918718d483d4a7aa357cfd6189fd813c8270fe7a50b602f00ec
3
+ size 7176192
epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/files/output.log ADDED
@@ -0,0 +1,587 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 29000: Loss = 2.5472, Perplexity = 12.77
3
+ wandb: WARNING Tried to log to step 29000 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 29001: sparse_stochastic (configured sparse attention)
5
+ Step 29010/50462 | Loss: 2.5465 | LR: 1.97e-04 | 69761.1ms/step | 0.9 samples/s
6
+ wandb: WARNING Tried to log to step 29010 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 29020/50462 | Loss: 2.5167 | LR: 1.96e-04 | 3987.0ms/step | 15.0 samples/s
8
+ wandb: WARNING Tried to log to step 29020 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 29030/50462 | Loss: 2.5452 | LR: 1.96e-04 | 3985.3ms/step | 15.1 samples/s
10
+ wandb: WARNING Tried to log to step 29030 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 29040/50462 | Loss: 2.5690 | LR: 1.96e-04 | 3980.9ms/step | 15.1 samples/s
12
+ wandb: WARNING Tried to log to step 29040 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 29050/50462 | Loss: 2.5354 | LR: 1.96e-04 | 3989.2ms/step | 15.0 samples/s
14
+ wandb: WARNING Tried to log to step 29050 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 29060/50462 | Loss: 2.5873 | LR: 1.96e-04 | 3984.3ms/step | 15.1 samples/s
16
+ wandb: WARNING Tried to log to step 29060 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 29070/50462 | Loss: 2.5505 | LR: 1.96e-04 | 3989.7ms/step | 15.0 samples/s
18
+ Step 29080/50462 | Loss: 2.5596 | LR: 1.96e-04 | 3988.9ms/step | 15.0 samples/s
19
+ Step 29090/50462 | Loss: 2.5504 | LR: 1.95e-04 | 3992.3ms/step | 15.0 samples/s
20
+ Step 29100/50462 | Loss: 2.5628 | LR: 1.95e-04 | 3980.2ms/step | 15.1 samples/s
21
+ Step 29110/50462 | Loss: 2.5619 | LR: 1.95e-04 | 3993.2ms/step | 15.0 samples/s
22
+ Step 29120/50462 | Loss: 2.5659 | LR: 1.95e-04 | 3991.7ms/step | 15.0 samples/s
23
+ Step 29130/50462 | Loss: 2.5645 | LR: 1.95e-04 | 3995.5ms/step | 15.0 samples/s
24
+ Step 29140/50462 | Loss: 2.5995 | LR: 1.95e-04 | 3989.8ms/step | 15.0 samples/s
25
+ Step 29150/50462 | Loss: 2.5235 | LR: 1.94e-04 | 3983.8ms/step | 15.1 samples/s
26
+ Step 29160/50462 | Loss: 2.5563 | LR: 1.94e-04 | 3985.3ms/step | 15.1 samples/s
27
+ Step 29170/50462 | Loss: 2.5234 | LR: 1.94e-04 | 3976.5ms/step | 15.1 samples/s
28
+ Step 29180/50462 | Loss: 2.5304 | LR: 1.94e-04 | 3988.9ms/step | 15.0 samples/s
29
+ Step 29190/50462 | Loss: 2.5504 | LR: 1.94e-04 | 3986.2ms/step | 15.1 samples/s
30
+ Step 29200/50462 | Loss: 2.5488 | LR: 1.94e-04 | 3981.4ms/step | 15.1 samples/s
31
+ Step 29210/50462 | Loss: 2.5682 | LR: 1.94e-04 | 3982.7ms/step | 15.1 samples/s
32
+ Step 29220/50462 | Loss: 2.5340 | LR: 1.93e-04 | 3976.0ms/step | 15.1 samples/s
33
+ Step 29230/50462 | Loss: 2.5665 | LR: 1.93e-04 | 3982.2ms/step | 15.1 samples/s
34
+ Step 29240/50462 | Loss: 2.5868 | LR: 1.93e-04 | 3985.4ms/step | 15.1 samples/s
35
+ Step 29250/50462 | Loss: 2.5577 | LR: 1.93e-04 | 3965.7ms/step | 15.1 samples/s
36
+ Validation at step 29250: Loss = 2.5460, Perplexity = 12.76
37
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29250
38
+ Step 29260/50462 | Loss: 2.5749 | LR: 1.93e-04 | 5875.1ms/step | 10.2 samples/s
39
+ Step 29270/50462 | Loss: 2.5511 | LR: 1.93e-04 | 3991.9ms/step | 15.0 samples/s
40
+ Step 29280/50462 | Loss: 2.5505 | LR: 1.92e-04 | 3977.1ms/step | 15.1 samples/s
41
+ Step 29290/50462 | Loss: 2.5447 | LR: 1.92e-04 | 3978.9ms/step | 15.1 samples/s
42
+ Step 29300/50462 | Loss: 2.5608 | LR: 1.92e-04 | 3968.9ms/step | 15.1 samples/s
43
+ Step 29310/50462 | Loss: 2.5273 | LR: 1.92e-04 | 3989.0ms/step | 15.0 samples/s
44
+ Step 29320/50462 | Loss: 2.5709 | LR: 1.92e-04 | 3981.8ms/step | 15.1 samples/s
45
+ Step 29330/50462 | Loss: 2.5505 | LR: 1.92e-04 | 3980.9ms/step | 15.1 samples/s
46
+ Step 29340/50462 | Loss: 2.5607 | LR: 1.92e-04 | 3977.6ms/step | 15.1 samples/s
47
+ Step 29350/50462 | Loss: 2.5865 | LR: 1.91e-04 | 3962.8ms/step | 15.1 samples/s
48
+ Step 29360/50462 | Loss: 2.5565 | LR: 1.91e-04 | 3971.0ms/step | 15.1 samples/s
49
+ Step 29370/50462 | Loss: 2.5664 | LR: 1.91e-04 | 3986.1ms/step | 15.1 samples/s
50
+ Step 29380/50462 | Loss: 2.5507 | LR: 1.91e-04 | 3973.2ms/step | 15.1 samples/s
51
+ Step 29390/50462 | Loss: 2.5610 | LR: 1.91e-04 | 3987.8ms/step | 15.0 samples/s
52
+ Step 29400/50462 | Loss: 2.5456 | LR: 1.91e-04 | 3974.4ms/step | 15.1 samples/s
53
+ Step 29410/50462 | Loss: 2.5759 | LR: 1.90e-04 | 3983.0ms/step | 15.1 samples/s
54
+ Step 29420/50462 | Loss: 2.5709 | LR: 1.90e-04 | 3976.9ms/step | 15.1 samples/s
55
+ Step 29430/50462 | Loss: 2.5373 | LR: 1.90e-04 | 3991.1ms/step | 15.0 samples/s
56
+ Step 29440/50462 | Loss: 2.5516 | LR: 1.90e-04 | 3985.2ms/step | 15.1 samples/s
57
+ Step 29450/50462 | Loss: 2.5324 | LR: 1.90e-04 | 3992.1ms/step | 15.0 samples/s
58
+ Step 29460/50462 | Loss: 2.5588 | LR: 1.90e-04 | 3992.7ms/step | 15.0 samples/s
59
+ Step 29470/50462 | Loss: 2.5467 | LR: 1.90e-04 | 3989.2ms/step | 15.0 samples/s
60
+ Step 29480/50462 | Loss: 2.5578 | LR: 1.89e-04 | 3983.6ms/step | 15.1 samples/s
61
+ Step 29490/50462 | Loss: 2.5761 | LR: 1.89e-04 | 3976.0ms/step | 15.1 samples/s
62
+ Step 29500/50462 | Loss: 2.5302 | LR: 1.89e-04 | 3987.8ms/step | 15.0 samples/s
63
+ Validation at step 29500: Loss = 2.5442, Perplexity = 12.73
64
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29500
65
+ Step 29510/50462 | Loss: 2.5208 | LR: 1.89e-04 | 5907.8ms/step | 10.2 samples/s
66
+ Step 29520/50462 | Loss: 2.5628 | LR: 1.89e-04 | 3998.4ms/step | 15.0 samples/s
67
+ Step 29530/50462 | Loss: 2.5463 | LR: 1.89e-04 | 3990.3ms/step | 15.0 samples/s
68
+ Step 29540/50462 | Loss: 2.5673 | LR: 1.88e-04 | 3980.8ms/step | 15.1 samples/s
69
+ Step 29550/50462 | Loss: 2.5592 | LR: 1.88e-04 | 4002.7ms/step | 15.0 samples/s
70
+ Step 29560/50462 | Loss: 2.5307 | LR: 1.88e-04 | 3995.1ms/step | 15.0 samples/s
71
+ Step 29570/50462 | Loss: 2.5442 | LR: 1.88e-04 | 3986.4ms/step | 15.1 samples/s
72
+ Step 29580/50462 | Loss: 2.5741 | LR: 1.88e-04 | 3976.7ms/step | 15.1 samples/s
73
+ Step 29590/50462 | Loss: 2.5524 | LR: 1.88e-04 | 3977.9ms/step | 15.1 samples/s
74
+ Step 29600/50462 | Loss: 2.5645 | LR: 1.88e-04 | 3982.3ms/step | 15.1 samples/s
75
+ Step 29610/50462 | Loss: 2.5682 | LR: 1.87e-04 | 3983.5ms/step | 15.1 samples/s
76
+ Step 29620/50462 | Loss: 2.5469 | LR: 1.87e-04 | 3993.8ms/step | 15.0 samples/s
77
+ Step 29630/50462 | Loss: 2.5522 | LR: 1.87e-04 | 4000.4ms/step | 15.0 samples/s
78
+ Step 29640/50462 | Loss: 2.5661 | LR: 1.87e-04 | 3990.2ms/step | 15.0 samples/s
79
+ Step 29650/50462 | Loss: 2.5361 | LR: 1.87e-04 | 3980.1ms/step | 15.1 samples/s
80
+ Step 29660/50462 | Loss: 2.5809 | LR: 1.87e-04 | 3993.8ms/step | 15.0 samples/s
81
+ Step 29670/50462 | Loss: 2.5482 | LR: 1.86e-04 | 3981.1ms/step | 15.1 samples/s
82
+ Step 29680/50462 | Loss: 2.5631 | LR: 1.86e-04 | 3982.5ms/step | 15.1 samples/s
83
+ Step 29690/50462 | Loss: 2.5693 | LR: 1.86e-04 | 3984.4ms/step | 15.1 samples/s
84
+ Step 29700/50462 | Loss: 2.5583 | LR: 1.86e-04 | 3979.4ms/step | 15.1 samples/s
85
+ Step 29710/50462 | Loss: 2.5798 | LR: 1.86e-04 | 3980.3ms/step | 15.1 samples/s
86
+ Step 29720/50462 | Loss: 2.5395 | LR: 1.86e-04 | 3991.4ms/step | 15.0 samples/s
87
+ Step 29730/50462 | Loss: 2.5602 | LR: 1.86e-04 | 3976.3ms/step | 15.1 samples/s
88
+ Step 29740/50462 | Loss: 2.5936 | LR: 1.85e-04 | 3986.1ms/step | 15.1 samples/s
89
+ Step 29750/50462 | Loss: 2.5300 | LR: 1.85e-04 | 3984.8ms/step | 15.1 samples/s
90
+ Validation at step 29750: Loss = 2.5448, Perplexity = 12.74
91
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29750
92
+ Step 29760/50462 | Loss: 2.5724 | LR: 1.85e-04 | 5897.3ms/step | 10.2 samples/s
93
+ Step 29770/50462 | Loss: 2.5270 | LR: 1.85e-04 | 3992.1ms/step | 15.0 samples/s
94
+ Step 29780/50462 | Loss: 2.5747 | LR: 1.85e-04 | 3987.3ms/step | 15.0 samples/s
95
+ Step 29790/50462 | Loss: 2.5694 | LR: 1.85e-04 | 3976.7ms/step | 15.1 samples/s
96
+ Step 29800/50462 | Loss: 2.5570 | LR: 1.85e-04 | 3996.4ms/step | 15.0 samples/s
97
+ Step 29810/50462 | Loss: 2.5637 | LR: 1.84e-04 | 3981.2ms/step | 15.1 samples/s
98
+ Step 29820/50462 | Loss: 2.5685 | LR: 1.84e-04 | 3994.3ms/step | 15.0 samples/s
99
+ Step 29830/50462 | Loss: 2.5205 | LR: 1.84e-04 | 3984.2ms/step | 15.1 samples/s
100
+ Step 29840/50462 | Loss: 2.5729 | LR: 1.84e-04 | 3986.0ms/step | 15.1 samples/s
101
+ Step 29850/50462 | Loss: 2.5275 | LR: 1.84e-04 | 3988.5ms/step | 15.0 samples/s
102
+ Step 29860/50462 | Loss: 2.5761 | LR: 1.84e-04 | 3990.2ms/step | 15.0 samples/s
103
+ Step 29870/50462 | Loss: 2.5489 | LR: 1.83e-04 | 3983.9ms/step | 15.1 samples/s
104
+ Step 29880/50462 | Loss: 2.5391 | LR: 1.83e-04 | 3982.4ms/step | 15.1 samples/s
105
+ Step 29890/50462 | Loss: 2.5414 | LR: 1.83e-04 | 3989.4ms/step | 15.0 samples/s
106
+ Step 29900/50462 | Loss: 2.5228 | LR: 1.83e-04 | 3988.9ms/step | 15.0 samples/s
107
+ Step 29910/50462 | Loss: 2.5585 | LR: 1.83e-04 | 3990.3ms/step | 15.0 samples/s
108
+ Step 29920/50462 | Loss: 2.5693 | LR: 1.83e-04 | 3982.0ms/step | 15.1 samples/s
109
+ Step 29930/50462 | Loss: 2.5479 | LR: 1.83e-04 | 3982.7ms/step | 15.1 samples/s
110
+ Step 29940/50462 | Loss: 2.5505 | LR: 1.82e-04 | 3982.8ms/step | 15.1 samples/s
111
+ Step 29950/50462 | Loss: 2.5861 | LR: 1.82e-04 | 3979.0ms/step | 15.1 samples/s
112
+ Step 29960/50462 | Loss: 2.5470 | LR: 1.82e-04 | 4001.0ms/step | 15.0 samples/s
113
+ Step 29970/50462 | Loss: 2.5750 | LR: 1.82e-04 | 3981.0ms/step | 15.1 samples/s
114
+ Step 29980/50462 | Loss: 2.5730 | LR: 1.82e-04 | 3984.8ms/step | 15.1 samples/s
115
+ Step 29990/50462 | Loss: 2.5568 | LR: 1.82e-04 | 3985.4ms/step | 15.1 samples/s
116
+ Step 30000/50462 | Loss: 2.5687 | LR: 1.81e-04 | 3994.3ms/step | 15.0 samples/s
117
+ Validation at step 30000: Loss = 2.5429, Perplexity = 12.72
118
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30000
119
+ Step 30010/50462 | Loss: 2.5326 | LR: 1.81e-04 | 5928.2ms/step | 10.1 samples/s
120
+ Step 30020/50462 | Loss: 2.5467 | LR: 1.81e-04 | 3982.9ms/step | 15.1 samples/s
121
+ Step 30030/50462 | Loss: 2.5323 | LR: 1.81e-04 | 3986.1ms/step | 15.1 samples/s
122
+ Step 30040/50462 | Loss: 2.5470 | LR: 1.81e-04 | 3983.4ms/step | 15.1 samples/s
123
+ Step 30050/50462 | Loss: 2.5496 | LR: 1.81e-04 | 3979.3ms/step | 15.1 samples/s
124
+ Step 30060/50462 | Loss: 2.5366 | LR: 1.81e-04 | 3980.6ms/step | 15.1 samples/s
125
+ Step 30070/50462 | Loss: 2.5250 | LR: 1.80e-04 | 3992.2ms/step | 15.0 samples/s
126
+ Step 30080/50462 | Loss: 2.5219 | LR: 1.80e-04 | 3984.7ms/step | 15.1 samples/s
127
+ Step 30090/50462 | Loss: 2.5584 | LR: 1.80e-04 | 3989.0ms/step | 15.0 samples/s
128
+ Step 30100/50462 | Loss: 2.5619 | LR: 1.80e-04 | 3983.3ms/step | 15.1 samples/s
129
+ Step 30110/50462 | Loss: 2.5072 | LR: 1.80e-04 | 3988.1ms/step | 15.0 samples/s
130
+ Step 30120/50462 | Loss: 2.5425 | LR: 1.80e-04 | 3985.9ms/step | 15.1 samples/s
131
+ Step 30130/50462 | Loss: 2.5363 | LR: 1.79e-04 | 3986.2ms/step | 15.1 samples/s
132
+ Step 30140/50462 | Loss: 2.5634 | LR: 1.79e-04 | 3998.5ms/step | 15.0 samples/s
133
+ Step 30150/50462 | Loss: 2.5305 | LR: 1.79e-04 | 3986.6ms/step | 15.1 samples/s
134
+ Step 30160/50462 | Loss: 2.5274 | LR: 1.79e-04 | 3977.9ms/step | 15.1 samples/s
135
+ Step 30170/50462 | Loss: 2.5864 | LR: 1.79e-04 | 3994.8ms/step | 15.0 samples/s
136
+ Step 30180/50462 | Loss: 2.5439 | LR: 1.79e-04 | 3992.6ms/step | 15.0 samples/s
137
+ Step 30190/50462 | Loss: 2.5634 | LR: 1.79e-04 | 3978.2ms/step | 15.1 samples/s
138
+ Step 30200/50462 | Loss: 2.5368 | LR: 1.78e-04 | 3983.8ms/step | 15.1 samples/s
139
+ Step 30210/50462 | Loss: 2.5514 | LR: 1.78e-04 | 4585.9ms/step | 13.1 samples/s
140
+ Step 30220/50462 | Loss: 2.5701 | LR: 1.78e-04 | 3988.9ms/step | 15.0 samples/s
141
+ Step 30230/50462 | Loss: 2.5407 | LR: 1.78e-04 | 3983.5ms/step | 15.1 samples/s
142
+ Step 30240/50462 | Loss: 2.5461 | LR: 1.78e-04 | 3989.4ms/step | 15.0 samples/s
143
+ Step 30250/50462 | Loss: 2.5235 | LR: 1.78e-04 | 3974.7ms/step | 15.1 samples/s
144
+ Validation at step 30250: Loss = 2.5413, Perplexity = 12.70
145
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30250
146
+ Step 30260/50462 | Loss: 2.5831 | LR: 1.78e-04 | 5897.1ms/step | 10.2 samples/s
147
+ Step 30270/50462 | Loss: 2.5569 | LR: 1.77e-04 | 3995.8ms/step | 15.0 samples/s
148
+ Step 30280/50462 | Loss: 2.5572 | LR: 1.77e-04 | 3979.3ms/step | 15.1 samples/s
149
+ Step 30290/50462 | Loss: 2.5618 | LR: 1.77e-04 | 3984.1ms/step | 15.1 samples/s
150
+ Step 30300/50462 | Loss: 2.5267 | LR: 1.77e-04 | 3998.0ms/step | 15.0 samples/s
151
+ Step 30310/50462 | Loss: 2.5489 | LR: 1.77e-04 | 3985.0ms/step | 15.1 samples/s
152
+ Step 30320/50462 | Loss: 2.5487 | LR: 1.77e-04 | 3989.3ms/step | 15.0 samples/s
153
+ Step 30330/50462 | Loss: 2.5439 | LR: 1.76e-04 | 3989.2ms/step | 15.0 samples/s
154
+ Step 30340/50462 | Loss: 2.5304 | LR: 1.76e-04 | 3982.6ms/step | 15.1 samples/s
155
+ Step 30350/50462 | Loss: 2.5489 | LR: 1.76e-04 | 3976.5ms/step | 15.1 samples/s
156
+ Step 30360/50462 | Loss: 2.5540 | LR: 1.76e-04 | 3992.5ms/step | 15.0 samples/s
157
+ Step 30370/50462 | Loss: 2.5561 | LR: 1.76e-04 | 4583.4ms/step | 13.1 samples/s
158
+ Step 30380/50462 | Loss: 2.5716 | LR: 1.76e-04 | 3985.3ms/step | 15.1 samples/s
159
+ Step 30390/50462 | Loss: 2.5661 | LR: 1.76e-04 | 3992.1ms/step | 15.0 samples/s
160
+ Step 30400/50462 | Loss: 2.5275 | LR: 1.75e-04 | 3980.7ms/step | 15.1 samples/s
161
+ Step 30410/50462 | Loss: 2.5692 | LR: 1.75e-04 | 3991.2ms/step | 15.0 samples/s
162
+ Step 30420/50462 | Loss: 2.5696 | LR: 1.75e-04 | 3996.4ms/step | 15.0 samples/s
163
+ Step 30430/50462 | Loss: 2.5585 | LR: 1.75e-04 | 3984.2ms/step | 15.1 samples/s
164
+ Step 30440/50462 | Loss: 2.5123 | LR: 1.75e-04 | 3978.7ms/step | 15.1 samples/s
165
+ Step 30450/50462 | Loss: 2.5462 | LR: 1.75e-04 | 3994.8ms/step | 15.0 samples/s
166
+ Step 30460/50462 | Loss: 2.5577 | LR: 1.75e-04 | 3974.3ms/step | 15.1 samples/s
167
+ Step 30470/50462 | Loss: 2.5401 | LR: 1.74e-04 | 3993.2ms/step | 15.0 samples/s
168
+ Step 30480/50462 | Loss: 2.5399 | LR: 1.74e-04 | 3986.8ms/step | 15.0 samples/s
169
+ Step 30490/50462 | Loss: 2.5532 | LR: 1.74e-04 | 3983.3ms/step | 15.1 samples/s
170
+ Step 30500/50462 | Loss: 2.5494 | LR: 1.74e-04 | 3985.2ms/step | 15.1 samples/s
171
+ Validation at step 30500: Loss = 2.5397, Perplexity = 12.68
172
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30500
173
+ Step 30510/50462 | Loss: 2.5273 | LR: 1.74e-04 | 5886.6ms/step | 10.2 samples/s
174
+ Step 30520/50462 | Loss: 2.5863 | LR: 1.74e-04 | 3993.9ms/step | 15.0 samples/s
175
+ Step 30530/50462 | Loss: 2.5592 | LR: 1.73e-04 | 3998.5ms/step | 15.0 samples/s
176
+ Step 30540/50462 | Loss: 2.5506 | LR: 1.73e-04 | 3984.7ms/step | 15.1 samples/s
177
+ Step 30550/50462 | Loss: 2.5670 | LR: 1.73e-04 | 3984.9ms/step | 15.1 samples/s
178
+ Step 30560/50462 | Loss: 2.5413 | LR: 1.73e-04 | 3977.7ms/step | 15.1 samples/s
179
+ Step 30570/50462 | Loss: 2.5556 | LR: 1.73e-04 | 3992.9ms/step | 15.0 samples/s
180
+ Step 30580/50462 | Loss: 2.5546 | LR: 1.73e-04 | 3986.2ms/step | 15.1 samples/s
181
+ Step 30590/50462 | Loss: 2.5587 | LR: 1.73e-04 | 3991.3ms/step | 15.0 samples/s
182
+ Step 30600/50462 | Loss: 2.5823 | LR: 1.72e-04 | 3986.6ms/step | 15.1 samples/s
183
+ Step 30610/50462 | Loss: 2.5677 | LR: 1.72e-04 | 3993.1ms/step | 15.0 samples/s
184
+ Step 30620/50462 | Loss: 2.5349 | LR: 1.72e-04 | 3994.7ms/step | 15.0 samples/s
185
+ Step 30630/50462 | Loss: 2.5687 | LR: 1.72e-04 | 4017.6ms/step | 14.9 samples/s
186
+ Step 30640/50462 | Loss: 2.5348 | LR: 1.72e-04 | 4018.2ms/step | 14.9 samples/s
187
+ Step 30650/50462 | Loss: 2.5321 | LR: 1.72e-04 | 4010.4ms/step | 15.0 samples/s
188
+ Step 30660/50462 | Loss: 2.5199 | LR: 1.72e-04 | 4010.9ms/step | 15.0 samples/s
189
+ Step 30670/50462 | Loss: 2.5480 | LR: 1.71e-04 | 3995.0ms/step | 15.0 samples/s
190
+ Step 30680/50462 | Loss: 2.5683 | LR: 1.71e-04 | 3997.1ms/step | 15.0 samples/s
191
+ Step 30690/50462 | Loss: 2.5588 | LR: 1.71e-04 | 4002.1ms/step | 15.0 samples/s
192
+ Step 30700/50462 | Loss: 2.5392 | LR: 1.71e-04 | 4573.7ms/step | 13.1 samples/s
193
+ Step 30710/50462 | Loss: 2.5489 | LR: 1.71e-04 | 3993.7ms/step | 15.0 samples/s
194
+ Step 30720/50462 | Loss: 2.5524 | LR: 1.71e-04 | 4005.1ms/step | 15.0 samples/s
195
+ Step 30730/50462 | Loss: 2.5375 | LR: 1.70e-04 | 4585.5ms/step | 13.1 samples/s
196
+ Step 30740/50462 | Loss: 2.5377 | LR: 1.70e-04 | 3995.9ms/step | 15.0 samples/s
197
+ Step 30750/50462 | Loss: 2.5375 | LR: 1.70e-04 | 4000.8ms/step | 15.0 samples/s
198
+ Validation at step 30750: Loss = 2.5388, Perplexity = 12.66
199
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30750
200
+ Step 30760/50462 | Loss: 2.5459 | LR: 1.70e-04 | 5909.0ms/step | 10.2 samples/s
201
+ Step 30770/50462 | Loss: 2.5489 | LR: 1.70e-04 | 3997.7ms/step | 15.0 samples/s
202
+ Step 30780/50462 | Loss: 2.5801 | LR: 1.70e-04 | 4002.4ms/step | 15.0 samples/s
203
+ Step 30790/50462 | Loss: 2.5618 | LR: 1.70e-04 | 3993.9ms/step | 15.0 samples/s
204
+ Step 30800/50462 | Loss: 2.5627 | LR: 1.69e-04 | 3996.9ms/step | 15.0 samples/s
205
+ Step 30810/50462 | Loss: 2.5269 | LR: 1.69e-04 | 3988.8ms/step | 15.0 samples/s
206
+ Step 30820/50462 | Loss: 2.5471 | LR: 1.69e-04 | 3998.5ms/step | 15.0 samples/s
207
+ Step 30830/50462 | Loss: 2.5710 | LR: 1.69e-04 | 3994.2ms/step | 15.0 samples/s
208
+ Step 30840/50462 | Loss: 2.5394 | LR: 1.69e-04 | 3983.7ms/step | 15.1 samples/s
209
+ Step 30850/50462 | Loss: 2.5713 | LR: 1.69e-04 | 3992.7ms/step | 15.0 samples/s
210
+ Step 30860/50462 | Loss: 2.5933 | LR: 1.69e-04 | 4005.3ms/step | 15.0 samples/s
211
+ Step 30870/50462 | Loss: 2.5611 | LR: 1.68e-04 | 3998.3ms/step | 15.0 samples/s
212
+ Step 30880/50462 | Loss: 2.5608 | LR: 1.68e-04 | 3995.9ms/step | 15.0 samples/s
213
+ Step 30890/50462 | Loss: 2.5533 | LR: 1.68e-04 | 3996.9ms/step | 15.0 samples/s
214
+ Step 30900/50462 | Loss: 2.5587 | LR: 1.68e-04 | 3994.1ms/step | 15.0 samples/s
215
+ Step 30910/50462 | Loss: 2.5660 | LR: 1.68e-04 | 3998.7ms/step | 15.0 samples/s
216
+ Step 30920/50462 | Loss: 2.5657 | LR: 1.68e-04 | 4000.1ms/step | 15.0 samples/s
217
+ Step 30930/50462 | Loss: 2.5450 | LR: 1.67e-04 | 4003.1ms/step | 15.0 samples/s
218
+ Step 30940/50462 | Loss: 2.5559 | LR: 1.67e-04 | 3990.7ms/step | 15.0 samples/s
219
+ Step 30950/50462 | Loss: 2.5527 | LR: 1.67e-04 | 3992.3ms/step | 15.0 samples/s
220
+ Step 30960/50462 | Loss: 2.5343 | LR: 1.67e-04 | 3991.3ms/step | 15.0 samples/s
221
+ Step 30970/50462 | Loss: 2.5656 | LR: 1.67e-04 | 3994.5ms/step | 15.0 samples/s
222
+ Step 30980/50462 | Loss: 2.5456 | LR: 1.67e-04 | 4001.7ms/step | 15.0 samples/s
223
+ Step 30990/50462 | Loss: 2.5560 | LR: 1.67e-04 | 4011.8ms/step | 15.0 samples/s
224
+ Step 31000/50462 | Loss: 2.4963 | LR: 1.66e-04 | 4019.0ms/step | 14.9 samples/s
225
+ Validation at step 31000: Loss = 2.5375, Perplexity = 12.65
226
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-31000
227
+ Step 31010/50462 | Loss: 2.5399 | LR: 1.66e-04 | 5965.3ms/step | 10.1 samples/s
228
+ Step 31020/50462 | Loss: 2.5483 | LR: 1.66e-04 | 4024.6ms/step | 14.9 samples/s
229
+ Step 31030/50462 | Loss: 2.5314 | LR: 1.66e-04 | 4015.7ms/step | 14.9 samples/s
230
+ Step 31040/50462 | Loss: 2.5474 | LR: 1.66e-04 | 4009.3ms/step | 15.0 samples/s
231
+ Step 31050/50462 | Loss: 2.5361 | LR: 1.66e-04 | 4018.4ms/step | 14.9 samples/s
232
+ Step 31060/50462 | Loss: 2.5478 | LR: 1.66e-04 | 4009.5ms/step | 15.0 samples/s
233
+ Step 31070/50462 | Loss: 2.5506 | LR: 1.65e-04 | 4016.6ms/step | 14.9 samples/s
234
+ Step 31080/50462 | Loss: 2.5307 | LR: 1.65e-04 | 4019.7ms/step | 14.9 samples/s
235
+ Step 31090/50462 | Loss: 2.5569 | LR: 1.65e-04 | 4024.1ms/step | 14.9 samples/s
236
+ Step 31100/50462 | Loss: 2.5829 | LR: 1.65e-04 | 4021.7ms/step | 14.9 samples/s
237
+ Step 31110/50462 | Loss: 2.5786 | LR: 1.65e-04 | 4013.8ms/step | 14.9 samples/s
238
+ Step 31120/50462 | Loss: 2.5577 | LR: 1.65e-04 | 4013.2ms/step | 15.0 samples/s
239
+ Step 31130/50462 | Loss: 2.5301 | LR: 1.64e-04 | 4016.7ms/step | 14.9 samples/s
240
+ Step 31140/50462 | Loss: 2.5472 | LR: 1.64e-04 | 4023.6ms/step | 14.9 samples/s
241
+ Step 31150/50462 | Loss: 2.5368 | LR: 1.64e-04 | 4014.8ms/step | 14.9 samples/s
242
+ Step 31160/50462 | Loss: 2.5665 | LR: 1.64e-04 | 4009.1ms/step | 15.0 samples/s
243
+ Step 31170/50462 | Loss: 2.5616 | LR: 1.64e-04 | 4008.6ms/step | 15.0 samples/s
244
+ Step 31180/50462 | Loss: 2.5573 | LR: 1.64e-04 | 4026.7ms/step | 14.9 samples/s
245
+ Step 31190/50462 | Loss: 2.5453 | LR: 1.64e-04 | 4009.0ms/step | 15.0 samples/s
246
+ Step 31200/50462 | Loss: 2.5745 | LR: 1.63e-04 | 4026.0ms/step | 14.9 samples/s
247
+ Step 31210/50462 | Loss: 2.5811 | LR: 1.63e-04 | 4024.5ms/step | 14.9 samples/s
248
+ Step 31220/50462 | Loss: 2.5491 | LR: 1.63e-04 | 4030.0ms/step | 14.9 samples/s
249
+ Step 31230/50462 | Loss: 2.5344 | LR: 1.63e-04 | 4008.4ms/step | 15.0 samples/s
250
+ Step 31240/50462 | Loss: 2.5518 | LR: 1.63e-04 | 4014.2ms/step | 14.9 samples/s
251
+ Step 31250/50462 | Loss: 2.5773 | LR: 1.63e-04 | 4008.5ms/step | 15.0 samples/s
252
+ Validation at step 31250: Loss = 2.5363, Perplexity = 12.63
253
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-31250
254
+ Step 31260/50462 | Loss: 2.5863 | LR: 1.63e-04 | 5937.4ms/step | 10.1 samples/s
255
+ Step 31270/50462 | Loss: 2.5513 | LR: 1.62e-04 | 4026.0ms/step | 14.9 samples/s
256
+ Step 31280/50462 | Loss: 2.5491 | LR: 1.62e-04 | 4012.9ms/step | 15.0 samples/s
257
+ Step 31290/50462 | Loss: 2.5909 | LR: 1.62e-04 | 4023.2ms/step | 14.9 samples/s
258
+ Step 31300/50462 | Loss: 2.5514 | LR: 1.62e-04 | 4021.0ms/step | 14.9 samples/s
259
+ Step 31310/50462 | Loss: 2.5486 | LR: 1.62e-04 | 4006.4ms/step | 15.0 samples/s
260
+ Step 31320/50462 | Loss: 2.5626 | LR: 1.62e-04 | 4008.6ms/step | 15.0 samples/s
261
+ Step 31330/50462 | Loss: 2.5745 | LR: 1.62e-04 | 4007.9ms/step | 15.0 samples/s
262
+ Step 31340/50462 | Loss: 2.5606 | LR: 1.61e-04 | 4007.8ms/step | 15.0 samples/s
263
+ Step 31350/50462 | Loss: 2.5642 | LR: 1.61e-04 | 4012.3ms/step | 15.0 samples/s
264
+ Step 31360/50462 | Loss: 2.5640 | LR: 1.61e-04 | 4020.7ms/step | 14.9 samples/s
265
+ Step 31370/50462 | Loss: 2.5501 | LR: 1.61e-04 | 4017.1ms/step | 14.9 samples/s
266
+ Step 31380/50462 | Loss: 2.5525 | LR: 1.61e-04 | 4005.5ms/step | 15.0 samples/s
267
+ Step 31390/50462 | Loss: 2.5786 | LR: 1.61e-04 | 4015.4ms/step | 14.9 samples/s
268
+ Step 31400/50462 | Loss: 2.5493 | LR: 1.60e-04 | 4002.6ms/step | 15.0 samples/s
269
+ Step 31410/50462 | Loss: 2.5167 | LR: 1.60e-04 | 4003.3ms/step | 15.0 samples/s
270
+ Step 31420/50462 | Loss: 2.5882 | LR: 1.60e-04 | 3998.9ms/step | 15.0 samples/s
271
+ Step 31430/50462 | Loss: 2.5397 | LR: 1.60e-04 | 4002.4ms/step | 15.0 samples/s
272
+ Step 31440/50462 | Loss: 2.5867 | LR: 1.60e-04 | 4008.3ms/step | 15.0 samples/s
273
+ Step 31450/50462 | Loss: 2.5532 | LR: 1.60e-04 | 4007.2ms/step | 15.0 samples/s
274
+ Step 31460/50462 | Loss: 2.5370 | LR: 1.60e-04 | 4012.6ms/step | 15.0 samples/s
275
+ Step 31470/50462 | Loss: 2.5664 | LR: 1.59e-04 | 4011.0ms/step | 15.0 samples/s
276
+ Step 31480/50462 | Loss: 2.5354 | LR: 1.59e-04 | 4021.0ms/step | 14.9 samples/s
277
+ Step 31490/50462 | Loss: 2.5244 | LR: 1.59e-04 | 4019.4ms/step | 14.9 samples/s
278
+ Step 31500/50462 | Loss: 2.5909 | LR: 1.59e-04 | 4014.9ms/step | 14.9 samples/s
279
+ Validation at step 31500: Loss = 2.5352, Perplexity = 12.62
280
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-31500
281
+ Step 31510/50462 | Loss: 2.5504 | LR: 1.59e-04 | 5917.3ms/step | 10.1 samples/s
282
+ Step 31520/50462 | Loss: 2.5485 | LR: 1.59e-04 | 4003.2ms/step | 15.0 samples/s
283
+ Step 31530/50462 | Loss: 2.5791 | LR: 1.59e-04 | 4006.0ms/step | 15.0 samples/s
284
+ Step 31540/50462 | Loss: 2.5473 | LR: 1.58e-04 | 4007.4ms/step | 15.0 samples/s
285
+ Step 31550/50462 | Loss: 2.5473 | LR: 1.58e-04 | 4006.6ms/step | 15.0 samples/s
286
+ Step 31560/50462 | Loss: 2.5479 | LR: 1.58e-04 | 3996.5ms/step | 15.0 samples/s
287
+ Step 31570/50462 | Loss: 2.5394 | LR: 1.58e-04 | 4009.2ms/step | 15.0 samples/s
288
+ Step 31580/50462 | Loss: 2.5267 | LR: 1.58e-04 | 4006.2ms/step | 15.0 samples/s
289
+ Step 31590/50462 | Loss: 2.5394 | LR: 1.58e-04 | 4004.9ms/step | 15.0 samples/s
290
+ Step 31600/50462 | Loss: 2.5777 | LR: 1.58e-04 | 3998.6ms/step | 15.0 samples/s
291
+ Step 31610/50462 | Loss: 2.5267 | LR: 1.57e-04 | 3991.1ms/step | 15.0 samples/s
292
+ Step 31620/50462 | Loss: 2.5502 | LR: 1.57e-04 | 3996.2ms/step | 15.0 samples/s
293
+ Step 31630/50462 | Loss: 2.5190 | LR: 1.57e-04 | 4000.0ms/step | 15.0 samples/s
294
+ Step 31640/50462 | Loss: 2.5518 | LR: 1.57e-04 | 4005.4ms/step | 15.0 samples/s
295
+ Step 31650/50462 | Loss: 2.5342 | LR: 1.57e-04 | 4004.9ms/step | 15.0 samples/s
296
+ Step 31660/50462 | Loss: 2.5939 | LR: 1.57e-04 | 3998.6ms/step | 15.0 samples/s
297
+ Step 31670/50462 | Loss: 2.5803 | LR: 1.57e-04 | 4011.0ms/step | 15.0 samples/s
298
+ Step 31680/50462 | Loss: 2.5215 | LR: 1.56e-04 | 4011.0ms/step | 15.0 samples/s
299
+ Step 31690/50462 | Loss: 2.5518 | LR: 1.56e-04 | 4007.5ms/step | 15.0 samples/s
300
+ Step 31700/50462 | Loss: 2.5528 | LR: 1.56e-04 | 4000.9ms/step | 15.0 samples/s
301
+ Step 31710/50462 | Loss: 2.5412 | LR: 1.56e-04 | 3998.4ms/step | 15.0 samples/s
302
+ Step 31720/50462 | Loss: 2.5706 | LR: 1.56e-04 | 4004.2ms/step | 15.0 samples/s
303
+ Step 31730/50462 | Loss: 2.5727 | LR: 1.56e-04 | 3996.7ms/step | 15.0 samples/s
304
+ Step 31740/50462 | Loss: 2.5718 | LR: 1.55e-04 | 3997.7ms/step | 15.0 samples/s
305
+ Step 31750/50462 | Loss: 2.5136 | LR: 1.55e-04 | 4012.2ms/step | 15.0 samples/s
306
+ Validation at step 31750: Loss = 2.5347, Perplexity = 12.61
307
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-31750
308
+ Step 31760/50462 | Loss: 2.5616 | LR: 1.55e-04 | 5919.8ms/step | 10.1 samples/s
309
+ Step 31770/50462 | Loss: 2.5832 | LR: 1.55e-04 | 4003.3ms/step | 15.0 samples/s
310
+ Step 31780/50462 | Loss: 2.5087 | LR: 1.55e-04 | 4010.1ms/step | 15.0 samples/s
311
+ Step 31790/50462 | Loss: 2.5240 | LR: 1.55e-04 | 4015.0ms/step | 14.9 samples/s
312
+ Step 31800/50462 | Loss: 2.5576 | LR: 1.55e-04 | 3992.9ms/step | 15.0 samples/s
313
+ Step 31810/50462 | Loss: 2.5349 | LR: 1.54e-04 | 4003.6ms/step | 15.0 samples/s
314
+ Step 31820/50462 | Loss: 2.5401 | LR: 1.54e-04 | 4001.6ms/step | 15.0 samples/s
315
+ Step 31830/50462 | Loss: 2.5467 | LR: 1.54e-04 | 3992.7ms/step | 15.0 samples/s
316
+ Step 31840/50462 | Loss: 2.5450 | LR: 1.54e-04 | 3990.7ms/step | 15.0 samples/s
317
+ Step 31850/50462 | Loss: 2.5814 | LR: 1.54e-04 | 4001.8ms/step | 15.0 samples/s
318
+ Step 31860/50462 | Loss: 2.5581 | LR: 1.54e-04 | 3992.6ms/step | 15.0 samples/s
319
+ Step 31870/50462 | Loss: 2.6159 | LR: 1.54e-04 | 4007.7ms/step | 15.0 samples/s
320
+ Step 31880/50462 | Loss: 2.5704 | LR: 1.53e-04 | 4004.8ms/step | 15.0 samples/s
321
+ Step 31890/50462 | Loss: 2.5370 | LR: 1.53e-04 | 4005.4ms/step | 15.0 samples/s
322
+ Step 31900/50462 | Loss: 2.5506 | LR: 1.53e-04 | 4000.7ms/step | 15.0 samples/s
323
+ Step 31910/50462 | Loss: 2.5195 | LR: 1.53e-04 | 4009.0ms/step | 15.0 samples/s
324
+ Step 31920/50462 | Loss: 2.5121 | LR: 1.53e-04 | 4003.4ms/step | 15.0 samples/s
325
+ Step 31930/50462 | Loss: 2.5359 | LR: 1.53e-04 | 3994.4ms/step | 15.0 samples/s
326
+ Step 31940/50462 | Loss: 2.5212 | LR: 1.53e-04 | 4000.1ms/step | 15.0 samples/s
327
+ Step 31950/50462 | Loss: 2.5410 | LR: 1.52e-04 | 3999.9ms/step | 15.0 samples/s
328
+ Step 31960/50462 | Loss: 2.5588 | LR: 1.52e-04 | 3991.5ms/step | 15.0 samples/s
329
+ Step 31970/50462 | Loss: 2.5588 | LR: 1.52e-04 | 4012.3ms/step | 15.0 samples/s
330
+ Step 31980/50462 | Loss: 2.5522 | LR: 1.52e-04 | 3995.2ms/step | 15.0 samples/s
331
+ Step 31990/50462 | Loss: 2.5022 | LR: 1.52e-04 | 3992.8ms/step | 15.0 samples/s
332
+ Step 32000/50462 | Loss: 2.5552 | LR: 1.52e-04 | 4003.1ms/step | 15.0 samples/s
333
+ Validation at step 32000: Loss = 2.5339, Perplexity = 12.60
334
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-32000
335
+ Step 32010/50462 | Loss: 2.5440 | LR: 1.52e-04 | 5929.5ms/step | 10.1 samples/s
336
+ Step 32020/50462 | Loss: 2.5226 | LR: 1.51e-04 | 4005.0ms/step | 15.0 samples/s
337
+ Step 32030/50462 | Loss: 2.5572 | LR: 1.51e-04 | 4000.9ms/step | 15.0 samples/s
338
+ Step 32040/50462 | Loss: 2.5329 | LR: 1.51e-04 | 3989.6ms/step | 15.0 samples/s
339
+ Step 32050/50462 | Loss: 2.5465 | LR: 1.51e-04 | 4009.7ms/step | 15.0 samples/s
340
+ Step 32060/50462 | Loss: 2.5826 | LR: 1.51e-04 | 4001.3ms/step | 15.0 samples/s
341
+ Step 32070/50462 | Loss: 2.5750 | LR: 1.51e-04 | 3985.6ms/step | 15.1 samples/s
342
+ Step 32080/50462 | Loss: 2.5734 | LR: 1.51e-04 | 3998.9ms/step | 15.0 samples/s
343
+ Step 32090/50462 | Loss: 2.5172 | LR: 1.50e-04 | 3996.0ms/step | 15.0 samples/s
344
+ Step 32100/50462 | Loss: 2.5301 | LR: 1.50e-04 | 3999.9ms/step | 15.0 samples/s
345
+ Step 32110/50462 | Loss: 2.5331 | LR: 1.50e-04 | 3985.5ms/step | 15.1 samples/s
346
+ Step 32120/50462 | Loss: 2.5640 | LR: 1.50e-04 | 3988.3ms/step | 15.0 samples/s
347
+ Step 32130/50462 | Loss: 2.5198 | LR: 1.50e-04 | 4010.1ms/step | 15.0 samples/s
348
+ Step 32140/50462 | Loss: 2.5624 | LR: 1.50e-04 | 4000.4ms/step | 15.0 samples/s
349
+ Step 32150/50462 | Loss: 2.5512 | LR: 1.50e-04 | 4005.9ms/step | 15.0 samples/s
350
+ Step 32160/50462 | Loss: 2.5277 | LR: 1.49e-04 | 4001.5ms/step | 15.0 samples/s
351
+ Step 32170/50462 | Loss: 2.5752 | LR: 1.49e-04 | 4002.6ms/step | 15.0 samples/s
352
+ Step 32180/50462 | Loss: 2.5022 | LR: 1.49e-04 | 3992.3ms/step | 15.0 samples/s
353
+ Step 32190/50462 | Loss: 2.5572 | LR: 1.49e-04 | 3991.8ms/step | 15.0 samples/s
354
+ Step 32200/50462 | Loss: 2.5409 | LR: 1.49e-04 | 3996.4ms/step | 15.0 samples/s
355
+ Step 32210/50462 | Loss: 2.5555 | LR: 1.49e-04 | 3996.5ms/step | 15.0 samples/s
356
+ Step 32220/50462 | Loss: 2.5448 | LR: 1.49e-04 | 3997.5ms/step | 15.0 samples/s
357
+ Step 32230/50462 | Loss: 2.5546 | LR: 1.48e-04 | 3990.6ms/step | 15.0 samples/s
358
+ Step 32240/50462 | Loss: 2.5533 | LR: 1.48e-04 | 3999.1ms/step | 15.0 samples/s
359
+ Step 32250/50462 | Loss: 2.5644 | LR: 1.48e-04 | 4010.7ms/step | 15.0 samples/s
360
+ Validation at step 32250: Loss = 2.5319, Perplexity = 12.58
361
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-32250
362
+ Step 32260/50462 | Loss: 2.5284 | LR: 1.48e-04 | 5913.0ms/step | 10.1 samples/s
363
+ Step 32270/50462 | Loss: 2.5472 | LR: 1.48e-04 | 3997.5ms/step | 15.0 samples/s
364
+ Step 32280/50462 | Loss: 2.5353 | LR: 1.48e-04 | 3993.5ms/step | 15.0 samples/s
365
+ Step 32290/50462 | Loss: 2.5247 | LR: 1.48e-04 | 4002.8ms/step | 15.0 samples/s
366
+ Step 32300/50462 | Loss: 2.5432 | LR: 1.47e-04 | 3987.0ms/step | 15.0 samples/s
367
+ Step 32310/50462 | Loss: 2.5293 | LR: 1.47e-04 | 3987.1ms/step | 15.0 samples/s
368
+ Step 32320/50462 | Loss: 2.5346 | LR: 1.47e-04 | 3986.1ms/step | 15.1 samples/s
369
+ Step 32330/50462 | Loss: 2.5362 | LR: 1.47e-04 | 3994.2ms/step | 15.0 samples/s
370
+ Step 32340/50462 | Loss: 2.5666 | LR: 1.47e-04 | 4001.0ms/step | 15.0 samples/s
371
+ Step 32350/50462 | Loss: 2.5409 | LR: 1.47e-04 | 3996.1ms/step | 15.0 samples/s
372
+ Step 32360/50462 | Loss: 2.5228 | LR: 1.47e-04 | 3999.0ms/step | 15.0 samples/s
373
+ Step 32370/50462 | Loss: 2.5348 | LR: 1.46e-04 | 4000.5ms/step | 15.0 samples/s
374
+ Step 32380/50462 | Loss: 2.4890 | LR: 1.46e-04 | 4008.0ms/step | 15.0 samples/s
375
+ Step 32390/50462 | Loss: 2.5361 | LR: 1.46e-04 | 4003.5ms/step | 15.0 samples/s
376
+ Step 32400/50462 | Loss: 2.5536 | LR: 1.46e-04 | 3995.3ms/step | 15.0 samples/s
377
+ Step 32410/50462 | Loss: 2.5567 | LR: 1.46e-04 | 3993.3ms/step | 15.0 samples/s
378
+ Step 32420/50462 | Loss: 2.5547 | LR: 1.46e-04 | 4002.6ms/step | 15.0 samples/s
379
+ Step 32430/50462 | Loss: 2.5301 | LR: 1.45e-04 | 4011.6ms/step | 15.0 samples/s
380
+ Step 32440/50462 | Loss: 2.5352 | LR: 1.45e-04 | 3998.8ms/step | 15.0 samples/s
381
+ Step 32450/50462 | Loss: 2.5188 | LR: 1.45e-04 | 4006.0ms/step | 15.0 samples/s
382
+ Step 32460/50462 | Loss: 2.5557 | LR: 1.45e-04 | 3988.6ms/step | 15.0 samples/s
383
+ Step 32470/50462 | Loss: 2.5433 | LR: 1.45e-04 | 4007.9ms/step | 15.0 samples/s
384
+ Step 32480/50462 | Loss: 2.5508 | LR: 1.45e-04 | 3999.3ms/step | 15.0 samples/s
385
+ Step 32490/50462 | Loss: 2.4919 | LR: 1.45e-04 | 3991.8ms/step | 15.0 samples/s
386
+ Step 32500/50462 | Loss: 2.5285 | LR: 1.44e-04 | 3998.0ms/step | 15.0 samples/s
387
+ Validation at step 32500: Loss = 2.5312, Perplexity = 12.57
388
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-32500
389
+ Step 32510/50462 | Loss: 2.5552 | LR: 1.44e-04 | 5912.4ms/step | 10.1 samples/s
390
+ Step 32520/50462 | Loss: 2.5924 | LR: 1.44e-04 | 4000.5ms/step | 15.0 samples/s
391
+ Step 32530/50462 | Loss: 2.5570 | LR: 1.44e-04 | 3994.9ms/step | 15.0 samples/s
392
+ Step 32540/50462 | Loss: 2.5616 | LR: 1.44e-04 | 3996.0ms/step | 15.0 samples/s
393
+ Step 32550/50462 | Loss: 2.5670 | LR: 1.44e-04 | 3999.5ms/step | 15.0 samples/s
394
+ Step 32560/50462 | Loss: 2.5546 | LR: 1.44e-04 | 4004.9ms/step | 15.0 samples/s
395
+ Step 32570/50462 | Loss: 2.5680 | LR: 1.43e-04 | 3995.8ms/step | 15.0 samples/s
396
+ Step 32580/50462 | Loss: 2.5565 | LR: 1.43e-04 | 4005.3ms/step | 15.0 samples/s
397
+ Step 32590/50462 | Loss: 2.5416 | LR: 1.43e-04 | 3993.8ms/step | 15.0 samples/s
398
+ Step 32600/50462 | Loss: 2.5362 | LR: 1.43e-04 | 3999.2ms/step | 15.0 samples/s
399
+ Step 32610/50462 | Loss: 2.5364 | LR: 1.43e-04 | 3998.6ms/step | 15.0 samples/s
400
+ Step 32620/50462 | Loss: 2.5368 | LR: 1.43e-04 | 4001.9ms/step | 15.0 samples/s
401
+ Step 32630/50462 | Loss: 2.5533 | LR: 1.43e-04 | 4003.5ms/step | 15.0 samples/s
402
+ Step 32640/50462 | Loss: 2.5462 | LR: 1.42e-04 | 3996.1ms/step | 15.0 samples/s
403
+ Step 32650/50462 | Loss: 2.5922 | LR: 1.42e-04 | 4000.4ms/step | 15.0 samples/s
404
+ Step 32660/50462 | Loss: 2.5491 | LR: 1.42e-04 | 3994.3ms/step | 15.0 samples/s
405
+ Step 32670/50462 | Loss: 2.5630 | LR: 1.42e-04 | 4010.8ms/step | 15.0 samples/s
406
+ Step 32680/50462 | Loss: 2.5407 | LR: 1.42e-04 | 3996.9ms/step | 15.0 samples/s
407
+ Step 32690/50462 | Loss: 2.5797 | LR: 1.42e-04 | 4002.4ms/step | 15.0 samples/s
408
+ Step 32700/50462 | Loss: 2.5387 | LR: 1.42e-04 | 4001.6ms/step | 15.0 samples/s
409
+ Step 32710/50462 | Loss: 2.5425 | LR: 1.41e-04 | 3995.6ms/step | 15.0 samples/s
410
+ Step 32720/50462 | Loss: 2.5390 | LR: 1.41e-04 | 3998.1ms/step | 15.0 samples/s
411
+ Step 32730/50462 | Loss: 2.5711 | LR: 1.41e-04 | 3998.5ms/step | 15.0 samples/s
412
+ Step 32740/50462 | Loss: 2.5186 | LR: 1.41e-04 | 4000.4ms/step | 15.0 samples/s
413
+ Step 32750/50462 | Loss: 2.5370 | LR: 1.41e-04 | 4006.8ms/step | 15.0 samples/s
414
+ Validation at step 32750: Loss = 2.5288, Perplexity = 12.54
415
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-32750
416
+ Step 32760/50462 | Loss: 2.5310 | LR: 1.41e-04 | 5914.5ms/step | 10.1 samples/s
417
+ Step 32770/50462 | Loss: 2.5497 | LR: 1.41e-04 | 4015.5ms/step | 14.9 samples/s
418
+ Step 32780/50462 | Loss: 2.5638 | LR: 1.41e-04 | 3999.9ms/step | 15.0 samples/s
419
+ Step 32790/50462 | Loss: 2.5550 | LR: 1.40e-04 | 4007.3ms/step | 15.0 samples/s
420
+ Step 32800/50462 | Loss: 2.5459 | LR: 1.40e-04 | 4003.0ms/step | 15.0 samples/s
421
+ Step 32810/50462 | Loss: 2.5576 | LR: 1.40e-04 | 3993.3ms/step | 15.0 samples/s
422
+ Step 32820/50462 | Loss: 2.5105 | LR: 1.40e-04 | 4001.5ms/step | 15.0 samples/s
423
+ Step 32830/50462 | Loss: 2.5560 | LR: 1.40e-04 | 3998.5ms/step | 15.0 samples/s
424
+ Step 32840/50462 | Loss: 2.5490 | LR: 1.40e-04 | 4007.3ms/step | 15.0 samples/s
425
+ Step 32850/50462 | Loss: 2.5578 | LR: 1.40e-04 | 3997.7ms/step | 15.0 samples/s
426
+ Step 32860/50462 | Loss: 2.5323 | LR: 1.39e-04 | 3985.9ms/step | 15.1 samples/s
427
+ Step 32870/50462 | Loss: 2.5270 | LR: 1.39e-04 | 3984.9ms/step | 15.1 samples/s
428
+ Step 32880/50462 | Loss: 2.5383 | LR: 1.39e-04 | 3996.1ms/step | 15.0 samples/s
429
+ Step 32890/50462 | Loss: 2.5557 | LR: 1.39e-04 | 4002.9ms/step | 15.0 samples/s
430
+ Step 32900/50462 | Loss: 2.5480 | LR: 1.39e-04 | 4004.4ms/step | 15.0 samples/s
431
+ Step 32910/50462 | Loss: 2.5626 | LR: 1.39e-04 | 4014.0ms/step | 14.9 samples/s
432
+ Step 32920/50462 | Loss: 2.5148 | LR: 1.39e-04 | 4004.1ms/step | 15.0 samples/s
433
+ Step 32930/50462 | Loss: 2.5294 | LR: 1.38e-04 | 4004.5ms/step | 15.0 samples/s
434
+ Step 32940/50462 | Loss: 2.5351 | LR: 1.38e-04 | 3990.9ms/step | 15.0 samples/s
435
+ Step 32950/50462 | Loss: 2.5453 | LR: 1.38e-04 | 4003.4ms/step | 15.0 samples/s
436
+ Step 32960/50462 | Loss: 2.5551 | LR: 1.38e-04 | 3997.9ms/step | 15.0 samples/s
437
+ Step 32970/50462 | Loss: 2.5432 | LR: 1.38e-04 | 4001.7ms/step | 15.0 samples/s
438
+ Step 32980/50462 | Loss: 2.5420 | LR: 1.38e-04 | 4003.8ms/step | 15.0 samples/s
439
+ Step 32990/50462 | Loss: 2.5328 | LR: 1.38e-04 | 3990.5ms/step | 15.0 samples/s
440
+ Step 33000/50462 | Loss: 2.5538 | LR: 1.37e-04 | 3999.8ms/step | 15.0 samples/s
441
+ Validation at step 33000: Loss = 2.5285, Perplexity = 12.53
442
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-33000
443
+ Step 33010/50462 | Loss: 2.5227 | LR: 1.37e-04 | 5974.9ms/step | 10.0 samples/s
444
+ Step 33020/50462 | Loss: 2.5038 | LR: 1.37e-04 | 4010.2ms/step | 15.0 samples/s
445
+ Step 33030/50462 | Loss: 2.5191 | LR: 1.37e-04 | 3997.6ms/step | 15.0 samples/s
446
+ Step 33040/50462 | Loss: 2.5391 | LR: 1.37e-04 | 4008.3ms/step | 15.0 samples/s
447
+ Step 33050/50462 | Loss: 2.5638 | LR: 1.37e-04 | 4013.8ms/step | 14.9 samples/s
448
+ Step 33060/50462 | Loss: 2.5283 | LR: 1.37e-04 | 4002.9ms/step | 15.0 samples/s
449
+ Step 33070/50462 | Loss: 2.5504 | LR: 1.36e-04 | 4004.9ms/step | 15.0 samples/s
450
+ Step 33080/50462 | Loss: 2.5422 | LR: 1.36e-04 | 4013.4ms/step | 14.9 samples/s
451
+ Step 33090/50462 | Loss: 2.5337 | LR: 1.36e-04 | 3990.7ms/step | 15.0 samples/s
452
+ Step 33100/50462 | Loss: 2.5267 | LR: 1.36e-04 | 3997.5ms/step | 15.0 samples/s
453
+ Step 33110/50462 | Loss: 2.5416 | LR: 1.36e-04 | 4006.2ms/step | 15.0 samples/s
454
+ Step 33120/50462 | Loss: 2.5168 | LR: 1.36e-04 | 3997.7ms/step | 15.0 samples/s
455
+ Step 33130/50462 | Loss: 2.5398 | LR: 1.36e-04 | 3991.7ms/step | 15.0 samples/s
456
+ Step 33140/50462 | Loss: 2.5436 | LR: 1.35e-04 | 4007.0ms/step | 15.0 samples/s
457
+ Step 33150/50462 | Loss: 2.5607 | LR: 1.35e-04 | 4009.8ms/step | 15.0 samples/s
458
+ Step 33160/50462 | Loss: 2.5532 | LR: 1.35e-04 | 4002.2ms/step | 15.0 samples/s
459
+ Step 33170/50462 | Loss: 2.5251 | LR: 1.35e-04 | 4012.1ms/step | 15.0 samples/s
460
+ Step 33180/50462 | Loss: 2.5563 | LR: 1.35e-04 | 4002.3ms/step | 15.0 samples/s
461
+ Step 33190/50462 | Loss: 2.5745 | LR: 1.35e-04 | 4014.5ms/step | 14.9 samples/s
462
+ Step 33200/50462 | Loss: 2.5310 | LR: 1.35e-04 | 3997.7ms/step | 15.0 samples/s
463
+ Step 33210/50462 | Loss: 2.5302 | LR: 1.34e-04 | 4004.6ms/step | 15.0 samples/s
464
+ Step 33220/50462 | Loss: 2.5650 | LR: 1.34e-04 | 3996.7ms/step | 15.0 samples/s
465
+ Step 33230/50462 | Loss: 2.5651 | LR: 1.34e-04 | 4005.1ms/step | 15.0 samples/s
466
+ Step 33240/50462 | Loss: 2.5637 | LR: 1.34e-04 | 4000.0ms/step | 15.0 samples/s
467
+ Step 33250/50462 | Loss: 2.5328 | LR: 1.34e-04 | 4009.8ms/step | 15.0 samples/s
468
+ Validation at step 33250: Loss = 2.5277, Perplexity = 12.53
469
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-33250
470
+ Step 33260/50462 | Loss: 2.5196 | LR: 1.34e-04 | 5943.6ms/step | 10.1 samples/s
471
+ Step 33270/50462 | Loss: 2.5541 | LR: 1.34e-04 | 4003.7ms/step | 15.0 samples/s
472
+ Step 33280/50462 | Loss: 2.5416 | LR: 1.33e-04 | 4000.1ms/step | 15.0 samples/s
473
+ Step 33290/50462 | Loss: 2.5499 | LR: 1.33e-04 | 4009.1ms/step | 15.0 samples/s
474
+ Step 33300/50462 | Loss: 2.5548 | LR: 1.33e-04 | 4002.4ms/step | 15.0 samples/s
475
+ Step 33310/50462 | Loss: 2.5393 | LR: 1.33e-04 | 3988.0ms/step | 15.0 samples/s
476
+ Step 33320/50462 | Loss: 2.5324 | LR: 1.33e-04 | 3999.7ms/step | 15.0 samples/s
477
+ Step 33330/50462 | Loss: 2.5408 | LR: 1.33e-04 | 4012.2ms/step | 15.0 samples/s
478
+ Step 33340/50462 | Loss: 2.5421 | LR: 1.33e-04 | 4005.8ms/step | 15.0 samples/s
479
+ Step 33350/50462 | Loss: 2.5296 | LR: 1.32e-04 | 4003.1ms/step | 15.0 samples/s
480
+ Step 33360/50462 | Loss: 2.5457 | LR: 1.32e-04 | 3992.2ms/step | 15.0 samples/s
481
+ Step 33370/50462 | Loss: 2.5395 | LR: 1.32e-04 | 4007.9ms/step | 15.0 samples/s
482
+ Step 33380/50462 | Loss: 2.5402 | LR: 1.32e-04 | 3993.7ms/step | 15.0 samples/s
483
+ Step 33390/50462 | Loss: 2.5283 | LR: 1.32e-04 | 3997.9ms/step | 15.0 samples/s
484
+ Step 33400/50462 | Loss: 2.5477 | LR: 1.32e-04 | 4001.3ms/step | 15.0 samples/s
485
+ Step 33410/50462 | Loss: 2.5464 | LR: 1.32e-04 | 3990.9ms/step | 15.0 samples/s
486
+ Step 33420/50462 | Loss: 2.5422 | LR: 1.32e-04 | 3992.8ms/step | 15.0 samples/s
487
+ Step 33430/50462 | Loss: 2.5738 | LR: 1.31e-04 | 4004.7ms/step | 15.0 samples/s
488
+ Step 33440/50462 | Loss: 2.5359 | LR: 1.31e-04 | 3997.8ms/step | 15.0 samples/s
489
+ Step 33450/50462 | Loss: 2.5318 | LR: 1.31e-04 | 4002.0ms/step | 15.0 samples/s
490
+ Step 33460/50462 | Loss: 2.5651 | LR: 1.31e-04 | 3989.9ms/step | 15.0 samples/s
491
+ Step 33470/50462 | Loss: 2.5774 | LR: 1.31e-04 | 3993.9ms/step | 15.0 samples/s
492
+ Step 33480/50462 | Loss: 2.5308 | LR: 1.31e-04 | 4001.3ms/step | 15.0 samples/s
493
+ Step 33490/50462 | Loss: 2.5301 | LR: 1.31e-04 | 3996.6ms/step | 15.0 samples/s
494
+ Step 33500/50462 | Loss: 2.5022 | LR: 1.30e-04 | 3989.7ms/step | 15.0 samples/s
495
+ Validation at step 33500: Loss = 2.5274, Perplexity = 12.52
496
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-33500
497
+ Step 33510/50462 | Loss: 2.5683 | LR: 1.30e-04 | 5891.9ms/step | 10.2 samples/s
498
+ Step 33520/50462 | Loss: 2.5509 | LR: 1.30e-04 | 4004.6ms/step | 15.0 samples/s
499
+ Step 33530/50462 | Loss: 2.5228 | LR: 1.30e-04 | 3995.4ms/step | 15.0 samples/s
500
+ Step 33540/50462 | Loss: 2.5472 | LR: 1.30e-04 | 4001.6ms/step | 15.0 samples/s
501
+ Step 33550/50462 | Loss: 2.5511 | LR: 1.30e-04 | 3996.5ms/step | 15.0 samples/s
502
+ Step 33560/50462 | Loss: 2.5294 | LR: 1.30e-04 | 3996.8ms/step | 15.0 samples/s
503
+ Step 33570/50462 | Loss: 2.5269 | LR: 1.29e-04 | 4005.3ms/step | 15.0 samples/s
504
+ Step 33580/50462 | Loss: 2.5751 | LR: 1.29e-04 | 4016.4ms/step | 14.9 samples/s
505
+ Step 33590/50462 | Loss: 2.5351 | LR: 1.29e-04 | 4005.0ms/step | 15.0 samples/s
506
+ Step 33600/50462 | Loss: 2.5408 | LR: 1.29e-04 | 3999.1ms/step | 15.0 samples/s
507
+ Step 33610/50462 | Loss: 2.5099 | LR: 1.29e-04 | 4000.8ms/step | 15.0 samples/s
508
+ Step 33620/50462 | Loss: 2.5174 | LR: 1.29e-04 | 3995.1ms/step | 15.0 samples/s
509
+ Step 33630/50462 | Loss: 2.5299 | LR: 1.29e-04 | 3990.9ms/step | 15.0 samples/s
510
+ Step 33640/50462 | Loss: 2.5530 | LR: 1.28e-04 | 4005.6ms/step | 15.0 samples/s
511
+ Step 33650/50462 | Loss: 2.5466 | LR: 1.28e-04 | 3999.2ms/step | 15.0 samples/s
512
+ Step 33660/50462 | Loss: 2.5458 | LR: 1.28e-04 | 4008.9ms/step | 15.0 samples/s
513
+ Step 33670/50462 | Loss: 2.5438 | LR: 1.28e-04 | 3992.8ms/step | 15.0 samples/s
514
+ Step 33680/50462 | Loss: 2.4912 | LR: 1.28e-04 | 3995.2ms/step | 15.0 samples/s
515
+ Step 33690/50462 | Loss: 2.5471 | LR: 1.28e-04 | 3986.9ms/step | 15.0 samples/s
516
+ Step 33700/50462 | Loss: 2.5055 | LR: 1.28e-04 | 3994.1ms/step | 15.0 samples/s
517
+ Step 33710/50462 | Loss: 2.5489 | LR: 1.27e-04 | 3995.0ms/step | 15.0 samples/s
518
+ Step 33720/50462 | Loss: 2.5623 | LR: 1.27e-04 | 3993.1ms/step | 15.0 samples/s
519
+ Step 33730/50462 | Loss: 2.5244 | LR: 1.27e-04 | 3993.8ms/step | 15.0 samples/s
520
+ Step 33740/50462 | Loss: 2.5425 | LR: 1.27e-04 | 3992.9ms/step | 15.0 samples/s
521
+ Step 33750/50462 | Loss: 2.5329 | LR: 1.27e-04 | 3992.5ms/step | 15.0 samples/s
522
+ Validation at step 33750: Loss = 2.5255, Perplexity = 12.50
523
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-33750
524
+ Step 33760/50462 | Loss: 2.5262 | LR: 1.27e-04 | 5897.0ms/step | 10.2 samples/s
525
+ Step 33770/50462 | Loss: 2.5317 | LR: 1.27e-04 | 3993.7ms/step | 15.0 samples/s
526
+ Step 33780/50462 | Loss: 2.5610 | LR: 1.27e-04 | 3995.9ms/step | 15.0 samples/s
527
+ Step 33790/50462 | Loss: 2.5851 | LR: 1.26e-04 | 3982.2ms/step | 15.1 samples/s
528
+ Step 33800/50462 | Loss: 2.5204 | LR: 1.26e-04 | 3995.7ms/step | 15.0 samples/s
529
+ Step 33810/50462 | Loss: 2.5283 | LR: 1.26e-04 | 3992.2ms/step | 15.0 samples/s
530
+ Step 33820/50462 | Loss: 2.5168 | LR: 1.26e-04 | 3986.6ms/step | 15.1 samples/s
531
+ Step 33830/50462 | Loss: 2.5435 | LR: 1.26e-04 | 3998.1ms/step | 15.0 samples/s
532
+ Step 33840/50462 | Loss: 2.5605 | LR: 1.26e-04 | 3999.6ms/step | 15.0 samples/s
533
+ Step 33850/50462 | Loss: 2.5547 | LR: 1.26e-04 | 3994.2ms/step | 15.0 samples/s
534
+ Step 33860/50462 | Loss: 2.5653 | LR: 1.25e-04 | 3998.4ms/step | 15.0 samples/s
535
+ Step 33870/50462 | Loss: 2.5200 | LR: 1.25e-04 | 3997.5ms/step | 15.0 samples/s
536
+ Step 33880/50462 | Loss: 2.5544 | LR: 1.25e-04 | 4000.9ms/step | 15.0 samples/s
537
+ Step 33890/50462 | Loss: 2.5740 | LR: 1.25e-04 | 3994.1ms/step | 15.0 samples/s
538
+ Step 33900/50462 | Loss: 2.5620 | LR: 1.25e-04 | 4001.0ms/step | 15.0 samples/s
539
+ Step 33910/50462 | Loss: 2.5652 | LR: 1.25e-04 | 3985.8ms/step | 15.1 samples/s
540
+ Step 33920/50462 | Loss: 2.5058 | LR: 1.25e-04 | 3985.4ms/step | 15.1 samples/s
541
+ Step 33930/50462 | Loss: 2.5710 | LR: 1.24e-04 | 4000.5ms/step | 15.0 samples/s
542
+ Step 33940/50462 | Loss: 2.5517 | LR: 1.24e-04 | 4001.6ms/step | 15.0 samples/s
543
+ Step 33950/50462 | Loss: 2.5431 | LR: 1.24e-04 | 3991.1ms/step | 15.0 samples/s
544
+ Step 33960/50462 | Loss: 2.5400 | LR: 1.24e-04 | 3992.1ms/step | 15.0 samples/s
545
+ Step 33970/50462 | Loss: 2.5317 | LR: 1.24e-04 | 4001.0ms/step | 15.0 samples/s
546
+ Step 33980/50462 | Loss: 2.5551 | LR: 1.24e-04 | 3993.5ms/step | 15.0 samples/s
547
+ Step 33990/50462 | Loss: 2.5302 | LR: 1.24e-04 | 3996.2ms/step | 15.0 samples/s
548
+ Step 34000/50462 | Loss: 2.5320 | LR: 1.24e-04 | 3995.4ms/step | 15.0 samples/s
549
+ Validation at step 34000: Loss = 2.5257, Perplexity = 12.50
550
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34000
551
+ Step 34010/50462 | Loss: 2.5450 | LR: 1.23e-04 | 5884.7ms/step | 10.2 samples/s
552
+ Step 34020/50462 | Loss: 2.5156 | LR: 1.23e-04 | 3996.6ms/step | 15.0 samples/s
553
+ Step 34030/50462 | Loss: 2.5530 | LR: 1.23e-04 | 3999.3ms/step | 15.0 samples/s
554
+ Step 34040/50462 | Loss: 2.5551 | LR: 1.23e-04 | 3989.2ms/step | 15.0 samples/s
555
+ Step 34050/50462 | Loss: 2.5222 | LR: 1.23e-04 | 3988.7ms/step | 15.0 samples/s
556
+ Step 34060/50462 | Loss: 2.5546 | LR: 1.23e-04 | 4002.3ms/step | 15.0 samples/s
557
+ Step 34070/50462 | Loss: 2.5347 | LR: 1.23e-04 | 3997.9ms/step | 15.0 samples/s
558
+ Step 34080/50462 | Loss: 2.5056 | LR: 1.22e-04 | 3989.8ms/step | 15.0 samples/s
559
+ Step 34090/50462 | Loss: 2.5670 | LR: 1.22e-04 | 3978.4ms/step | 15.1 samples/s
560
+ Step 34100/50462 | Loss: 2.5446 | LR: 1.22e-04 | 3987.9ms/step | 15.0 samples/s
561
+ Step 34110/50462 | Loss: 2.5539 | LR: 1.22e-04 | 3994.8ms/step | 15.0 samples/s
562
+ Step 34120/50462 | Loss: 2.5154 | LR: 1.22e-04 | 3993.1ms/step | 15.0 samples/s
563
+ Step 34130/50462 | Loss: 2.5160 | LR: 1.22e-04 | 4006.1ms/step | 15.0 samples/s
564
+ Step 34140/50462 | Loss: 2.5086 | LR: 1.22e-04 | 3990.4ms/step | 15.0 samples/s
565
+ Step 34150/50462 | Loss: 2.5210 | LR: 1.21e-04 | 3975.4ms/step | 15.1 samples/s
566
+ Step 34160/50462 | Loss: 2.5732 | LR: 1.21e-04 | 4000.7ms/step | 15.0 samples/s
567
+ Step 34170/50462 | Loss: 2.5560 | LR: 1.21e-04 | 4011.2ms/step | 15.0 samples/s
568
+ Step 34180/50462 | Loss: 2.5096 | LR: 1.21e-04 | 4001.5ms/step | 15.0 samples/s
569
+ Step 34190/50462 | Loss: 2.4965 | LR: 1.21e-04 | 3995.3ms/step | 15.0 samples/s
570
+ Step 34200/50462 | Loss: 2.5670 | LR: 1.21e-04 | 3998.6ms/step | 15.0 samples/s
571
+ Step 34210/50462 | Loss: 2.5240 | LR: 1.21e-04 | 3995.4ms/step | 15.0 samples/s
572
+ Step 34220/50462 | Loss: 2.5623 | LR: 1.21e-04 | 3993.9ms/step | 15.0 samples/s
573
+ Step 34230/50462 | Loss: 2.5258 | LR: 1.20e-04 | 4000.1ms/step | 15.0 samples/s
574
+ Step 34240/50462 | Loss: 2.5736 | LR: 1.20e-04 | 3986.8ms/step | 15.0 samples/s
575
+ Step 34250/50462 | Loss: 2.5252 | LR: 1.20e-04 | 3993.0ms/step | 15.0 samples/s
576
+ Validation at step 34250: Loss = 2.5244, Perplexity = 12.48
577
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-34250
578
+ Step 34260/50462 | Loss: 2.5287 | LR: 1.20e-04 | 5872.7ms/step | 10.2 samples/s
579
+ Step 34270/50462 | Loss: 2.5308 | LR: 1.20e-04 | 3989.2ms/step | 15.0 samples/s
580
+ Step 34280/50462 | Loss: 2.5218 | LR: 1.20e-04 | 3988.6ms/step | 15.0 samples/s
581
+ Step 34290/50462 | Loss: 2.5161 | LR: 1.20e-04 | 3994.7ms/step | 15.0 samples/s
582
+ Step 34300/50462 | Loss: 2.5562 | LR: 1.19e-04 | 3989.2ms/step | 15.0 samples/s
583
+ Step 34310/50462 | Loss: 2.5269 | LR: 1.19e-04 | 3996.9ms/step | 15.0 samples/s
584
+ Step 34320/50462 | Loss: 2.5511 | LR: 1.19e-04 | 3989.2ms/step | 15.0 samples/s
585
+ Step 34330/50462 | Loss: 2.5325 | LR: 1.19e-04 | 3987.2ms/step | 15.0 samples/s
586
+ Step 34340/50462 | Loss: 2.5275 | LR: 1.19e-04 | 3989.2ms/step | 15.0 samples/s
587
+ Step 34350/50462 | Loss: 2.5452 | LR: 1.19e-04 | 3983.9ms/step | 15.1 samples/s
epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/files/wandb-metadata.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-28T21:46:32.321092Z",
5
+ "args": [
6
+ "--config",
7
+ "configs/p2_1p2b_mixed_seed71_4gpu_resume28500.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/SparseAtt.git",
14
+ "commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-train-4-gpu-0-29",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 4,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "6144316817408"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-c65885f7-52dc-6f23-a3dd-6addf8b518a2"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-7e349d73-ad76-d62d-635c-266997fedcc1"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-4e4c2343-4f5b-a2ff-312e-03d402a27803"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-3a3f0ac7-1596-1797-bf0d-31f3418789cd"
61
+ }
62
+ ],
63
+ "cudaVersion": "13.0",
64
+ "writerId": "wmgo8ahwbgaod8ure9s0rzrnrutdk34n"
65
+ }
epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-28 21:46:32,411 INFO MainThread:472 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-28 21:46:32,412 INFO MainThread:472 [wandb_setup.py:_flush():81] Configure stats pid to 472
3
+ 2026-08-28 21:46:32,412 INFO MainThread:472 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-28 21:46:32,412 INFO MainThread:472 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_214632-8ynn1we6/logs/debug.log
5
+ 2026-08-28 21:46:32,412 INFO MainThread:472 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_214632-8ynn1we6/logs/debug-internal.log
6
+ 2026-08-28 21:46:32,412 INFO MainThread:472 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-28 21:46:32,412 INFO MainThread:472 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 71, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed71 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 71, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-28 21:46:32,412 INFO MainThread:472 [wandb_init.py:init():898] starting backend
10
+ 2026-08-28 21:46:33,543 INFO MainThread:472 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-28 21:46:34,145 INFO MainThread:472 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-28 21:46:34,151 INFO MainThread:472 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-28 21:46:34,188 INFO MainThread:472 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-28 21:46:34,819 INFO MainThread:472 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-28 21:46:34,822 INFO MainThread:472 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-28 21:46:35,431 INFO MainThread:472 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-28 21:46:35,432 INFO MainThread:472 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-28 21:46:35,432 INFO MainThread:472 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-28 21:46:35,432 INFO MainThread:472 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-28 21:46:35,484 INFO MainThread:472 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260828_214632-8ynn1we6/run-8ynn1we6.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0c08baac64b6026ea561029cb0c36546d3b12f8a94be398903ed77a8799a443
3
+ size 7471104
epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/files/output.log ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 29000: Loss = 2.5472, Perplexity = 12.77
3
+ wandb: WARNING Tried to log to step 29000 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 29001: sparse_stochastic (configured sparse attention)
5
+ Step 29010/50462 | Loss: 2.5465 | LR: 1.97e-04 | 105113.9ms/step | 0.6 samples/s
6
+ wandb: WARNING Tried to log to step 29010 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 29020/50462 | Loss: 2.5168 | LR: 1.96e-04 | 12079.1ms/step | 5.0 samples/s
8
+ wandb: WARNING Tried to log to step 29020 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 29030/50462 | Loss: 2.5452 | LR: 1.96e-04 | 12036.5ms/step | 5.0 samples/s
10
+ wandb: WARNING Tried to log to step 29030 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 29040/50462 | Loss: 2.5690 | LR: 1.96e-04 | 11882.0ms/step | 5.0 samples/s
12
+ wandb: WARNING Tried to log to step 29040 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 29050/50462 | Loss: 2.5354 | LR: 1.96e-04 | 11867.8ms/step | 5.1 samples/s
14
+ wandb: WARNING Tried to log to step 29050 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 29060/50462 | Loss: 2.5873 | LR: 1.96e-04 | 12090.1ms/step | 5.0 samples/s
16
+ wandb: WARNING Tried to log to step 29060 that is less than the current step 29061. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 29070/50462 | Loss: 2.5506 | LR: 1.96e-04 | 11901.4ms/step | 5.0 samples/s
18
+ Step 29080/50462 | Loss: 2.5596 | LR: 1.96e-04 | 11892.5ms/step | 5.0 samples/s
19
+ Step 29090/50462 | Loss: 2.5504 | LR: 1.95e-04 | 11951.9ms/step | 5.0 samples/s
20
+ Step 29100/50462 | Loss: 2.5628 | LR: 1.95e-04 | 11959.7ms/step | 5.0 samples/s
21
+ Step 29110/50462 | Loss: 2.5619 | LR: 1.95e-04 | 12246.3ms/step | 4.9 samples/s
22
+ Step 29120/50462 | Loss: 2.5659 | LR: 1.95e-04 | 12002.1ms/step | 5.0 samples/s
23
+ Step 29130/50462 | Loss: 2.5645 | LR: 1.95e-04 | 12130.9ms/step | 4.9 samples/s
24
+ Step 29140/50462 | Loss: 2.5994 | LR: 1.95e-04 | 11504.1ms/step | 5.2 samples/s
25
+ Step 29150/50462 | Loss: 2.5235 | LR: 1.94e-04 | 10107.7ms/step | 5.9 samples/s
26
+ Step 29160/50462 | Loss: 2.5563 | LR: 1.94e-04 | 12134.2ms/step | 4.9 samples/s
27
+ Step 29170/50462 | Loss: 2.5235 | LR: 1.94e-04 | 12145.4ms/step | 4.9 samples/s
28
+ Step 29180/50462 | Loss: 2.5304 | LR: 1.94e-04 | 12145.4ms/step | 4.9 samples/s
29
+ Step 29190/50462 | Loss: 2.5503 | LR: 1.94e-04 | 12168.1ms/step | 4.9 samples/s
30
+ Step 29200/50462 | Loss: 2.5488 | LR: 1.94e-04 | 12164.8ms/step | 4.9 samples/s
31
+ Step 29210/50462 | Loss: 2.5682 | LR: 1.94e-04 | 11890.7ms/step | 5.0 samples/s
32
+ Step 29220/50462 | Loss: 2.5340 | LR: 1.93e-04 | 12108.2ms/step | 5.0 samples/s
33
+ Step 29230/50462 | Loss: 2.5666 | LR: 1.93e-04 | 11883.1ms/step | 5.0 samples/s
34
+ Step 29240/50462 | Loss: 2.5868 | LR: 1.93e-04 | 11905.5ms/step | 5.0 samples/s
35
+ Step 29250/50462 | Loss: 2.5577 | LR: 1.93e-04 | 12011.6ms/step | 5.0 samples/s
36
+ Validation at step 29250: Loss = 2.5460, Perplexity = 12.76
37
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29250
38
+ Step 29260/50462 | Loss: 2.5749 | LR: 1.93e-04 | 17422.6ms/step | 3.4 samples/s
39
+ Step 29270/50462 | Loss: 2.5511 | LR: 1.93e-04 | 11929.3ms/step | 5.0 samples/s
40
+ Step 29280/50462 | Loss: 2.5505 | LR: 1.92e-04 | 11987.7ms/step | 5.0 samples/s
41
+ Step 29290/50462 | Loss: 2.5447 | LR: 1.92e-04 | 12069.4ms/step | 5.0 samples/s
42
+ Step 29300/50462 | Loss: 2.5608 | LR: 1.92e-04 | 12088.7ms/step | 5.0 samples/s
43
+ Step 29310/50462 | Loss: 2.5273 | LR: 1.92e-04 | 11856.9ms/step | 5.1 samples/s
44
+ Step 29320/50462 | Loss: 2.5709 | LR: 1.92e-04 | 11931.1ms/step | 5.0 samples/s
45
+ Step 29330/50462 | Loss: 2.5504 | LR: 1.92e-04 | 12268.7ms/step | 4.9 samples/s
46
+ Step 29340/50462 | Loss: 2.5607 | LR: 1.92e-04 | 11681.2ms/step | 5.1 samples/s
47
+ Step 29350/50462 | Loss: 2.5866 | LR: 1.91e-04 | 12018.1ms/step | 5.0 samples/s
48
+ Step 29360/50462 | Loss: 2.5565 | LR: 1.91e-04 | 12117.6ms/step | 5.0 samples/s
49
+ Step 29370/50462 | Loss: 2.5664 | LR: 1.91e-04 | 11901.0ms/step | 5.0 samples/s
50
+ Step 29380/50462 | Loss: 2.5507 | LR: 1.91e-04 | 12056.6ms/step | 5.0 samples/s
51
+ Step 29390/50462 | Loss: 2.5611 | LR: 1.91e-04 | 11629.5ms/step | 5.2 samples/s
52
+ Step 29400/50462 | Loss: 2.5455 | LR: 1.91e-04 | 10053.3ms/step | 6.0 samples/s
53
+ Step 29410/50462 | Loss: 2.5759 | LR: 1.90e-04 | 11928.4ms/step | 5.0 samples/s
54
+ Step 29420/50462 | Loss: 2.5709 | LR: 1.90e-04 | 12079.3ms/step | 5.0 samples/s
55
+ Step 29430/50462 | Loss: 2.5373 | LR: 1.90e-04 | 12110.5ms/step | 5.0 samples/s
56
+ Step 29440/50462 | Loss: 2.5517 | LR: 1.90e-04 | 11866.1ms/step | 5.1 samples/s
57
+ Step 29450/50462 | Loss: 2.5325 | LR: 1.90e-04 | 11997.2ms/step | 5.0 samples/s
58
+ Step 29460/50462 | Loss: 2.5588 | LR: 1.90e-04 | 12094.7ms/step | 5.0 samples/s
59
+ Step 29470/50462 | Loss: 2.5466 | LR: 1.90e-04 | 11908.0ms/step | 5.0 samples/s
60
+ Step 29480/50462 | Loss: 2.5578 | LR: 1.89e-04 | 11859.6ms/step | 5.1 samples/s
61
+ Step 29490/50462 | Loss: 2.5761 | LR: 1.89e-04 | 11837.1ms/step | 5.1 samples/s
62
+ Step 29500/50462 | Loss: 2.5303 | LR: 1.89e-04 | 11847.8ms/step | 5.1 samples/s
63
+ Validation at step 29500: Loss = 2.5443, Perplexity = 12.73
64
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29500
65
+ Step 29510/50462 | Loss: 2.5208 | LR: 1.89e-04 | 17079.8ms/step | 3.5 samples/s
66
+ Step 29520/50462 | Loss: 2.5629 | LR: 1.89e-04 | 12012.4ms/step | 5.0 samples/s
67
+ Step 29530/50462 | Loss: 2.5463 | LR: 1.89e-04 | 12009.8ms/step | 5.0 samples/s
68
+ Step 29540/50462 | Loss: 2.5673 | LR: 1.88e-04 | 11927.5ms/step | 5.0 samples/s
69
+ Step 29550/50462 | Loss: 2.5592 | LR: 1.88e-04 | 11898.4ms/step | 5.0 samples/s
70
+ Step 29560/50462 | Loss: 2.5307 | LR: 1.88e-04 | 11860.9ms/step | 5.1 samples/s
71
+ Step 29570/50462 | Loss: 2.5442 | LR: 1.88e-04 | 12234.2ms/step | 4.9 samples/s
72
+ Step 29580/50462 | Loss: 2.5741 | LR: 1.88e-04 | 11775.7ms/step | 5.1 samples/s
73
+ Step 29590/50462 | Loss: 2.5524 | LR: 1.88e-04 | 11898.8ms/step | 5.0 samples/s
74
+ Step 29600/50462 | Loss: 2.5645 | LR: 1.88e-04 | 11973.1ms/step | 5.0 samples/s
75
+ Step 29610/50462 | Loss: 2.5682 | LR: 1.87e-04 | 11787.7ms/step | 5.1 samples/s
76
+ Step 29620/50462 | Loss: 2.5469 | LR: 1.87e-04 | 12175.9ms/step | 4.9 samples/s
77
+ Step 29630/50462 | Loss: 2.5522 | LR: 1.87e-04 | 11873.4ms/step | 5.1 samples/s
78
+ Step 29640/50462 | Loss: 2.5661 | LR: 1.87e-04 | 11316.4ms/step | 5.3 samples/s
79
+ Step 29650/50462 | Loss: 2.5361 | LR: 1.87e-04 | 10391.1ms/step | 5.8 samples/s
80
+ Step 29660/50462 | Loss: 2.5809 | LR: 1.87e-04 | 11861.9ms/step | 5.1 samples/s
81
+ Step 29670/50462 | Loss: 2.5482 | LR: 1.86e-04 | 12006.0ms/step | 5.0 samples/s
82
+ Step 29680/50462 | Loss: 2.5631 | LR: 1.86e-04 | 11865.3ms/step | 5.1 samples/s
83
+ Step 29690/50462 | Loss: 2.5693 | LR: 1.86e-04 | 12021.2ms/step | 5.0 samples/s
84
+ Step 29700/50462 | Loss: 2.5583 | LR: 1.86e-04 | 12024.8ms/step | 5.0 samples/s
85
+ Step 29710/50462 | Loss: 2.5798 | LR: 1.86e-04 | 12133.6ms/step | 4.9 samples/s
86
+ Step 29720/50462 | Loss: 2.5395 | LR: 1.86e-04 | 12143.1ms/step | 4.9 samples/s
87
+ Step 29730/50462 | Loss: 2.5603 | LR: 1.86e-04 | 12217.9ms/step | 4.9 samples/s
88
+ Step 29740/50462 | Loss: 2.5936 | LR: 1.85e-04 | 12331.1ms/step | 4.9 samples/s
89
+ Step 29750/50462 | Loss: 2.5300 | LR: 1.85e-04 | 12040.5ms/step | 5.0 samples/s
90
+ Validation at step 29750: Loss = 2.5448, Perplexity = 12.74
91
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29750
92
+ Step 29760/50462 | Loss: 2.5724 | LR: 1.85e-04 | 16992.2ms/step | 3.5 samples/s
93
+ Step 29770/50462 | Loss: 2.5270 | LR: 1.85e-04 | 11696.2ms/step | 5.1 samples/s
94
+ Step 29780/50462 | Loss: 2.5747 | LR: 1.85e-04 | 11975.8ms/step | 5.0 samples/s
95
+ Step 29790/50462 | Loss: 2.5694 | LR: 1.85e-04 | 11760.6ms/step | 5.1 samples/s
96
+ Step 29800/50462 | Loss: 2.5570 | LR: 1.85e-04 | 11959.3ms/step | 5.0 samples/s
97
+ Step 29810/50462 | Loss: 2.5637 | LR: 1.84e-04 | 11827.8ms/step | 5.1 samples/s
98
+ Step 29820/50462 | Loss: 2.5684 | LR: 1.84e-04 | 12060.6ms/step | 5.0 samples/s
99
+ Step 29830/50462 | Loss: 2.5205 | LR: 1.84e-04 | 12001.3ms/step | 5.0 samples/s
100
+ Step 29840/50462 | Loss: 2.5729 | LR: 1.84e-04 | 12042.0ms/step | 5.0 samples/s
101
+ Step 29850/50462 | Loss: 2.5275 | LR: 1.84e-04 | 11871.6ms/step | 5.1 samples/s
102
+ Step 29860/50462 | Loss: 2.5761 | LR: 1.84e-04 | 11908.4ms/step | 5.0 samples/s
103
+ Step 29870/50462 | Loss: 2.5489 | LR: 1.83e-04 | 12167.8ms/step | 4.9 samples/s
104
+ Step 29880/50462 | Loss: 2.5391 | LR: 1.83e-04 | 12006.3ms/step | 5.0 samples/s
105
+ Step 29890/50462 | Loss: 2.5413 | LR: 1.83e-04 | 11538.6ms/step | 5.2 samples/s
106
+ Step 29900/50462 | Loss: 2.5227 | LR: 1.83e-04 | 10085.0ms/step | 5.9 samples/s
107
+ Step 29910/50462 | Loss: 2.5585 | LR: 1.83e-04 | 12078.2ms/step | 5.0 samples/s
108
+ Step 29920/50462 | Loss: 2.5693 | LR: 1.83e-04 | 12109.7ms/step | 5.0 samples/s
109
+ Step 29930/50462 | Loss: 2.5479 | LR: 1.83e-04 | 11811.9ms/step | 5.1 samples/s
110
+ Step 29940/50462 | Loss: 2.5505 | LR: 1.82e-04 | 11975.4ms/step | 5.0 samples/s
111
+ Step 29950/50462 | Loss: 2.5861 | LR: 1.82e-04 | 11809.3ms/step | 5.1 samples/s
112
+ Step 29960/50462 | Loss: 2.5470 | LR: 1.82e-04 | 11957.4ms/step | 5.0 samples/s
113
+ Step 29970/50462 | Loss: 2.5749 | LR: 1.82e-04 | 12253.5ms/step | 4.9 samples/s
114
+ Step 29980/50462 | Loss: 2.5730 | LR: 1.82e-04 | 12052.0ms/step | 5.0 samples/s
115
+ Step 29990/50462 | Loss: 2.5568 | LR: 1.82e-04 | 11975.7ms/step | 5.0 samples/s
116
+ Step 30000/50462 | Loss: 2.5687 | LR: 1.81e-04 | 11835.5ms/step | 5.1 samples/s
117
+ Validation at step 30000: Loss = 2.5429, Perplexity = 12.72
118
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30000
119
+ Step 30010/50462 | Loss: 2.5326 | LR: 1.81e-04 | 17220.9ms/step | 3.5 samples/s
120
+ Step 30020/50462 | Loss: 2.5468 | LR: 1.81e-04 | 12197.7ms/step | 4.9 samples/s
121
+ Step 30030/50462 | Loss: 2.5323 | LR: 1.81e-04 | 11843.4ms/step | 5.1 samples/s
122
+ Step 30040/50462 | Loss: 2.5470 | LR: 1.81e-04 | 11926.8ms/step | 5.0 samples/s
123
+ Step 30050/50462 | Loss: 2.5496 | LR: 1.81e-04 | 11902.5ms/step | 5.0 samples/s
124
+ Step 30060/50462 | Loss: 2.5366 | LR: 1.81e-04 | 11780.5ms/step | 5.1 samples/s
125
+ Step 30070/50462 | Loss: 2.5249 | LR: 1.80e-04 | 12087.3ms/step | 5.0 samples/s
126
+ Step 30080/50462 | Loss: 2.5219 | LR: 1.80e-04 | 11951.9ms/step | 5.0 samples/s
127
+ Step 30090/50462 | Loss: 2.5584 | LR: 1.80e-04 | 11944.2ms/step | 5.0 samples/s
128
+ Step 30100/50462 | Loss: 2.5619 | LR: 1.80e-04 | 11819.1ms/step | 5.1 samples/s
129
+ Step 30110/50462 | Loss: 2.5073 | LR: 1.80e-04 | 11932.7ms/step | 5.0 samples/s
130
+ Step 30120/50462 | Loss: 2.5425 | LR: 1.80e-04 | 12126.3ms/step | 4.9 samples/s
131
+ Step 30130/50462 | Loss: 2.5363 | LR: 1.79e-04 | 12047.0ms/step | 5.0 samples/s
132
+ Step 30140/50462 | Loss: 2.5634 | LR: 1.79e-04 | 11740.1ms/step | 5.1 samples/s
133
+ Step 30150/50462 | Loss: 2.5304 | LR: 1.79e-04 | 9937.1ms/step | 6.0 samples/s
134
+ Step 30160/50462 | Loss: 2.5273 | LR: 1.79e-04 | 11837.0ms/step | 5.1 samples/s
135
+ Step 30170/50462 | Loss: 2.5864 | LR: 1.79e-04 | 11712.8ms/step | 5.1 samples/s
136
+ Step 30180/50462 | Loss: 2.5439 | LR: 1.79e-04 | 11808.5ms/step | 5.1 samples/s
137
+ Step 30190/50462 | Loss: 2.5634 | LR: 1.79e-04 | 11855.4ms/step | 5.1 samples/s
138
+ Step 30200/50462 | Loss: 2.5368 | LR: 1.78e-04 | 12035.6ms/step | 5.0 samples/s
139
+ Step 30210/50462 | Loss: 2.5514 | LR: 1.78e-04 | 11974.5ms/step | 5.0 samples/s
140
+ Step 30220/50462 | Loss: 2.5701 | LR: 1.78e-04 | 11786.4ms/step | 5.1 samples/s
141
+ Step 30230/50462 | Loss: 2.5407 | LR: 1.78e-04 | 11755.1ms/step | 5.1 samples/s
142
+ Step 30240/50462 | Loss: 2.5461 | LR: 1.78e-04 | 11943.2ms/step | 5.0 samples/s
143
+ Step 30250/50462 | Loss: 2.5235 | LR: 1.78e-04 | 11808.0ms/step | 5.1 samples/s
144
+ Validation at step 30250: Loss = 2.5413, Perplexity = 12.70
145
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30250
146
+ Step 30260/50462 | Loss: 2.5831 | LR: 1.78e-04 | 17312.3ms/step | 3.5 samples/s
147
+ Step 30270/50462 | Loss: 2.5569 | LR: 1.77e-04 | 12078.7ms/step | 5.0 samples/s
148
+ Step 30280/50462 | Loss: 2.5573 | LR: 1.77e-04 | 11995.0ms/step | 5.0 samples/s
149
+ Step 30290/50462 | Loss: 2.5618 | LR: 1.77e-04 | 11980.1ms/step | 5.0 samples/s
150
+ Step 30300/50462 | Loss: 2.5267 | LR: 1.77e-04 | 11978.5ms/step | 5.0 samples/s
151
+ Step 30310/50462 | Loss: 2.5490 | LR: 1.77e-04 | 11940.3ms/step | 5.0 samples/s
152
+ Step 30320/50462 | Loss: 2.5487 | LR: 1.77e-04 | 11879.0ms/step | 5.1 samples/s
153
+ Step 30330/50462 | Loss: 2.5439 | LR: 1.76e-04 | 12112.0ms/step | 5.0 samples/s
154
+ Step 30340/50462 | Loss: 2.5304 | LR: 1.76e-04 | 12165.9ms/step | 4.9 samples/s
155
+ Step 30350/50462 | Loss: 2.5490 | LR: 1.76e-04 | 11922.8ms/step | 5.0 samples/s
156
+ Step 30360/50462 | Loss: 2.5541 | LR: 1.76e-04 | 11820.3ms/step | 5.1 samples/s
157
+ Step 30370/50462 | Loss: 2.5562 | LR: 1.76e-04 | 12066.8ms/step | 5.0 samples/s
158
+ Step 30380/50462 | Loss: 2.5716 | LR: 1.76e-04 | 12223.0ms/step | 4.9 samples/s
159
+ Step 30390/50462 | Loss: 2.5660 | LR: 1.76e-04 | 11933.0ms/step | 5.0 samples/s
160
+ Step 30400/50462 | Loss: 2.5275 | LR: 1.75e-04 | 9690.5ms/step | 6.2 samples/s
161
+ Step 30410/50462 | Loss: 2.5692 | LR: 1.75e-04 | 12061.1ms/step | 5.0 samples/s
162
+ Step 30420/50462 | Loss: 2.5695 | LR: 1.75e-04 | 11744.3ms/step | 5.1 samples/s
163
+ Step 30430/50462 | Loss: 2.5585 | LR: 1.75e-04 | 11998.8ms/step | 5.0 samples/s
164
+ Step 30440/50462 | Loss: 2.5123 | LR: 1.75e-04 | 11972.4ms/step | 5.0 samples/s
165
+ Step 30450/50462 | Loss: 2.5462 | LR: 1.75e-04 | 11997.8ms/step | 5.0 samples/s
166
+ Step 30460/50462 | Loss: 2.5577 | LR: 1.75e-04 | 11891.5ms/step | 5.0 samples/s
167
+ Step 30470/50462 | Loss: 2.5401 | LR: 1.74e-04 | 11968.3ms/step | 5.0 samples/s
168
+ Step 30480/50462 | Loss: 2.5399 | LR: 1.74e-04 | 12080.9ms/step | 5.0 samples/s
169
+ Step 30490/50462 | Loss: 2.5532 | LR: 1.74e-04 | 12102.5ms/step | 5.0 samples/s
170
+ Step 30500/50462 | Loss: 2.5494 | LR: 1.74e-04 | 11998.7ms/step | 5.0 samples/s
171
+ Validation at step 30500: Loss = 2.5397, Perplexity = 12.68
172
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-30500
173
+ Step 30510/50462 | Loss: 2.5273 | LR: 1.74e-04 | 17115.4ms/step | 3.5 samples/s
174
+ Step 30520/50462 | Loss: 2.5862 | LR: 1.74e-04 | 12183.1ms/step | 4.9 samples/s
175
+ Step 30530/50462 | Loss: 2.5591 | LR: 1.73e-04 | 11849.2ms/step | 5.1 samples/s
176
+ Step 30540/50462 | Loss: 2.5506 | LR: 1.73e-04 | 12147.0ms/step | 4.9 samples/s
177
+ Step 30550/50462 | Loss: 2.5670 | LR: 1.73e-04 | 11823.3ms/step | 5.1 samples/s
178
+ Step 30560/50462 | Loss: 2.5413 | LR: 1.73e-04 | 12073.3ms/step | 5.0 samples/s
179
+ Step 30570/50462 | Loss: 2.5557 | LR: 1.73e-04 | 11849.5ms/step | 5.1 samples/s
180
+ Step 30580/50462 | Loss: 2.5546 | LR: 1.73e-04 | 12021.4ms/step | 5.0 samples/s
181
+ Step 30590/50462 | Loss: 2.5587 | LR: 1.73e-04 | 11929.1ms/step | 5.0 samples/s
182
+ Step 30600/50462 | Loss: 2.5823 | LR: 1.72e-04 | 11910.2ms/step | 5.0 samples/s
183
+ Step 30610/50462 | Loss: 2.5677 | LR: 1.72e-04 | 12256.9ms/step | 4.9 samples/s
184
+ Step 30620/50462 | Loss: 2.5349 | LR: 1.72e-04 | 12304.2ms/step | 4.9 samples/s
185
+ Step 30630/50462 | Loss: 2.5687 | LR: 1.72e-04 | 11793.5ms/step | 5.1 samples/s
186
+ Step 30640/50462 | Loss: 2.5348 | LR: 1.72e-04 | 12379.2ms/step | 4.8 samples/s
187
+ Step 30650/50462 | Loss: 2.5321 | LR: 1.72e-04 | 9512.4ms/step | 6.3 samples/s
188
+ Step 30660/50462 | Loss: 2.5199 | LR: 1.72e-04 | 12045.2ms/step | 5.0 samples/s
189
+ Step 30670/50462 | Loss: 2.5480 | LR: 1.71e-04 | 11896.4ms/step | 5.0 samples/s
190
+ Step 30680/50462 | Loss: 2.5683 | LR: 1.71e-04 | 12075.6ms/step | 5.0 samples/s
191
+ Step 30690/50462 | Loss: 2.5589 | LR: 1.71e-04 | 12092.7ms/step | 5.0 samples/s
epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/files/wandb-metadata.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-28T21:51:02.417273Z",
5
+ "args": [
6
+ "--config",
7
+ "configs/p2_1p2b_mixed_seed71_4gpu_autoresume.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/SparseAtt.git",
14
+ "commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-4-gpu-3-0-14",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 4,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "6072048807936"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-e3358200-f776-7214-8269-c2c219f03c2e"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-788fbe73-59b3-e845-80c4-5eff0b2f64b5"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-123c77ea-9377-5bde-70b6-99ef1c791ff4"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-a3a94461-36d7-5e8a-af9b-6209869fa7f4"
61
+ }
62
+ ],
63
+ "cudaVersion": "13.0",
64
+ "writerId": "7kt6c4b2iuabxdc1z90eja63sqbdohi1"
65
+ }
epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff
 
epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/logs/debug.log ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-08-28 21:51:02,441 INFO MainThread:1755 [wandb_setup.py:_flush():81] Current SDK version is 0.26.1
2
+ 2026-08-28 21:51:02,441 INFO MainThread:1755 [wandb_setup.py:_flush():81] Configure stats pid to 1755
3
+ 2026-08-28 21:51:02,441 INFO MainThread:1755 [wandb_setup.py:_flush():81] Loading settings from environment variables
4
+ 2026-08-28 21:51:02,441 INFO MainThread:1755 [wandb_init.py:setup_run_log_directory():723] Logging user logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_215102-8ynn1we6/logs/debug.log
5
+ 2026-08-28 21:51:02,441 INFO MainThread:1755 [wandb_init.py:setup_run_log_directory():724] Logging internal logs to /mloscratch/homes/navasard/sparse/SparseAtt/wandb/run-20260828_215102-8ynn1we6/logs/debug-internal.log
6
+ 2026-08-28 21:51:02,441 INFO MainThread:1755 [wandb_init.py:init():850] calling init triggers
7
+ 2026-08-28 21:51:02,441 INFO MainThread:1755 [wandb_init.py:init():855] wandb.init called with sweep_config: {}
8
+ config: {'model_path': '/mloscratch/homes/navasard/qwen3_06b_base', 'model_dtype': 'bfloat16', 'init_from_scratch': True, 'train_path': '/mloscratch/homes/navasard/data/fineweb_large/tokenized', 'val_path': None, 'max_seq_length': 8192, 'num_workers': 8, 'output_dir': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205', 'resume_from_checkpoint': '/mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed71/run_20260826_175205/checkpoint-29000', 'force_new_run': False, 'num_epochs': 1, 'max_steps': 50462, 'per_device_batch_size': 1, 'gradient_accumulation_steps': 15, 'learning_rate': 0.0005, 'weight_decay': 0.01, 'warmup_steps': 750, 'lr_scheduler': 'cosine', 'max_grad_norm': 1.0, 'save_steps': 250, 'eval_steps': 250, 'eval_batches': 50, 'logging_steps': 10, 'seed': 71, 'verbose': True, 'checkpoint_activations': True, 'strategy': 'ddp', 'mixed_precision': 'bf16', 'wandb_enabled': True, 'wandb_project': 'stochastic_sparsity', 'wandb_name': '1.2b mixed 8k seed71 4gpu', 'wandb_tags': ['qwen3', '1.2b', '8k', 'p2', 'mixed', 'seed71'], 'hidden_size': 1536, 'intermediate_size': 6144, 'num_attention_heads': 12, 'num_key_value_heads': 4, 'num_hidden_layers': None, 'head_dim': None, 'attention_dropout': None, 'rope_train_position_scale_set': None, 'attention_type': 'sparse_stochastic', 'sparse_block_size': 128, 'sparse_num_random_blocks': 35, 'sparse_mask_pool_size': 10000, 'sparse_val_dense_compare': False, 'sparse_layer_indices': None, 'sparse_mask_cache_dir': '/mloscratch/homes/navasard/sparse/mask_cache_8k_mixed', 'sparse_mask_seed': 71, 'sparse_mixed_nrb_set': [63, 47, 31, 21, 11], 'dense_mode': False, 'dense_step_intervals': None, '_wandb': {}}
9
+ 2026-08-28 21:51:02,442 INFO MainThread:1755 [wandb_init.py:init():898] starting backend
10
+ 2026-08-28 21:51:02,706 INFO MainThread:1755 [wandb_init.py:init():913] sending inform_init request
11
+ 2026-08-28 21:51:03,129 INFO MainThread:1755 [wandb_init.py:init():918] backend started and connected
12
+ 2026-08-28 21:51:03,134 INFO MainThread:1755 [wandb_init.py:init():988] updated telemetry
13
+ 2026-08-28 21:51:03,159 INFO MainThread:1755 [wandb_init.py:init():1011] communicating run to backend with 90.0 second timeout
14
+ 2026-08-28 21:51:03,985 INFO MainThread:1755 [wandb_init.py:init():1051] run resumed
15
+ 2026-08-28 21:51:03,986 INFO MainThread:1755 [wandb_init.py:init():1056] starting run threads in backend
16
+ 2026-08-28 21:51:04,241 INFO MainThread:1755 [wandb_run.py:_console_start():2554] atexit reg
17
+ 2026-08-28 21:51:04,242 INFO MainThread:1755 [wandb_run.py:_redirect():2403] redirect: wrap_raw
18
+ 2026-08-28 21:51:04,242 INFO MainThread:1755 [wandb_run.py:_redirect():2472] Wrapping output streams.
19
+ 2026-08-28 21:51:04,242 INFO MainThread:1755 [wandb_run.py:_redirect():2495] Redirects installed.
20
+ 2026-08-28 21:51:04,253 INFO MainThread:1755 [wandb_init.py:init():1094] run started, returning control to user process
epfl_final/SparseAtt/wandb/run-20260828_215102-8ynn1we6/run-8ynn1we6.wandb ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a894a29a2b1cedb60d3ff9132bc38e5af9c837a03bf3eddfa61d431729048e33
3
+ size 6815744
epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/files/output.log ADDED
@@ -0,0 +1,676 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Starting training...
2
+ Validation at step 25000: Loss = 2.6170, Perplexity = 13.69
3
+ wandb: WARNING Tried to log to step 25000 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
4
+ Switching attention mode for step 25001: sparse_stochastic (configured sparse attention)
5
+ Step 25010/50462 | Loss: 2.6280 | LR: 2.59e-04 | 58173.4ms/step | 1.0 samples/s
6
+ wandb: WARNING Tried to log to step 25010 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
7
+ Step 25020/50462 | Loss: 2.5513 | LR: 2.59e-04 | 4009.6ms/step | 15.0 samples/s
8
+ wandb: WARNING Tried to log to step 25020 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
9
+ Step 25030/50462 | Loss: 2.5974 | LR: 2.59e-04 | 3993.8ms/step | 15.0 samples/s
10
+ wandb: WARNING Tried to log to step 25030 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
11
+ Step 25040/50462 | Loss: 2.5901 | LR: 2.59e-04 | 3996.7ms/step | 15.0 samples/s
12
+ wandb: WARNING Tried to log to step 25040 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
13
+ Step 25050/50462 | Loss: 2.6022 | LR: 2.59e-04 | 4009.1ms/step | 15.0 samples/s
14
+ wandb: WARNING Tried to log to step 25050 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
15
+ Step 25060/50462 | Loss: 2.5829 | LR: 2.59e-04 | 4006.2ms/step | 15.0 samples/s
16
+ wandb: WARNING Tried to log to step 25060 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
17
+ Step 25070/50462 | Loss: 2.6101 | LR: 2.58e-04 | 4003.8ms/step | 15.0 samples/s
18
+ wandb: WARNING Tried to log to step 25070 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
19
+ Step 25080/50462 | Loss: 2.6068 | LR: 2.58e-04 | 4006.0ms/step | 15.0 samples/s
20
+ wandb: WARNING Tried to log to step 25080 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
21
+ Step 25090/50462 | Loss: 2.5944 | LR: 2.58e-04 | 4010.3ms/step | 15.0 samples/s
22
+ wandb: WARNING Tried to log to step 25090 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
23
+ Step 25100/50462 | Loss: 2.5705 | LR: 2.58e-04 | 4008.6ms/step | 15.0 samples/s
24
+ wandb: WARNING Tried to log to step 25100 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
25
+ Step 25110/50462 | Loss: 2.5632 | LR: 2.58e-04 | 4003.6ms/step | 15.0 samples/s
26
+ wandb: WARNING Tried to log to step 25110 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
27
+ Step 25120/50462 | Loss: 2.5900 | LR: 2.58e-04 | 4001.4ms/step | 15.0 samples/s
28
+ wandb: WARNING Tried to log to step 25120 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
29
+ Step 25130/50462 | Loss: 2.6161 | LR: 2.58e-04 | 4004.1ms/step | 15.0 samples/s
30
+ wandb: WARNING Tried to log to step 25130 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
31
+ Step 25140/50462 | Loss: 2.6008 | LR: 2.57e-04 | 4007.2ms/step | 15.0 samples/s
32
+ wandb: WARNING Tried to log to step 25140 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
33
+ Step 25150/50462 | Loss: 2.5963 | LR: 2.57e-04 | 4002.6ms/step | 15.0 samples/s
34
+ wandb: WARNING Tried to log to step 25150 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
35
+ Step 25160/50462 | Loss: 2.5763 | LR: 2.57e-04 | 4002.2ms/step | 15.0 samples/s
36
+ wandb: WARNING Tried to log to step 25160 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
37
+ Step 25170/50462 | Loss: 2.6165 | LR: 2.57e-04 | 4006.9ms/step | 15.0 samples/s
38
+ wandb: WARNING Tried to log to step 25170 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
39
+ Step 25180/50462 | Loss: 2.5574 | LR: 2.57e-04 | 4009.8ms/step | 15.0 samples/s
40
+ wandb: WARNING Tried to log to step 25180 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
41
+ Step 25190/50462 | Loss: 2.5740 | LR: 2.57e-04 | 4005.1ms/step | 15.0 samples/s
42
+ wandb: WARNING Tried to log to step 25190 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
43
+ Step 25200/50462 | Loss: 2.6076 | LR: 2.56e-04 | 4009.1ms/step | 15.0 samples/s
44
+ wandb: WARNING Tried to log to step 25200 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
45
+ Step 25210/50462 | Loss: 2.5915 | LR: 2.56e-04 | 3997.3ms/step | 15.0 samples/s
46
+ wandb: WARNING Tried to log to step 25210 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
47
+ Step 25220/50462 | Loss: 2.5918 | LR: 2.56e-04 | 4000.3ms/step | 15.0 samples/s
48
+ wandb: WARNING Tried to log to step 25220 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
49
+ Step 25230/50462 | Loss: 2.5812 | LR: 2.56e-04 | 4001.9ms/step | 15.0 samples/s
50
+ wandb: WARNING Tried to log to step 25230 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
51
+ Step 25240/50462 | Loss: 2.5987 | LR: 2.56e-04 | 4007.8ms/step | 15.0 samples/s
52
+ wandb: WARNING Tried to log to step 25240 that is less than the current step 25241. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
53
+ Step 25250/50462 | Loss: 2.5879 | LR: 2.56e-04 | 4001.6ms/step | 15.0 samples/s
54
+ Validation at step 25250: Loss = 2.6153, Perplexity = 13.67
55
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25250
56
+ Step 25260/50462 | Loss: 2.5977 | LR: 2.55e-04 | 5877.2ms/step | 10.2 samples/s
57
+ Step 25270/50462 | Loss: 2.5752 | LR: 2.55e-04 | 3997.3ms/step | 15.0 samples/s
58
+ Step 25280/50462 | Loss: 2.6060 | LR: 2.55e-04 | 4012.0ms/step | 15.0 samples/s
59
+ Step 25290/50462 | Loss: 2.5910 | LR: 2.55e-04 | 4006.7ms/step | 15.0 samples/s
60
+ Step 25300/50462 | Loss: 2.6100 | LR: 2.55e-04 | 4007.1ms/step | 15.0 samples/s
61
+ Step 25310/50462 | Loss: 2.6083 | LR: 2.55e-04 | 4003.0ms/step | 15.0 samples/s
62
+ Step 25320/50462 | Loss: 2.5730 | LR: 2.55e-04 | 4017.6ms/step | 14.9 samples/s
63
+ Step 25330/50462 | Loss: 2.6052 | LR: 2.54e-04 | 4011.0ms/step | 15.0 samples/s
64
+ Step 25340/50462 | Loss: 2.6152 | LR: 2.54e-04 | 4012.1ms/step | 15.0 samples/s
65
+ Step 25350/50462 | Loss: 2.5869 | LR: 2.54e-04 | 3998.7ms/step | 15.0 samples/s
66
+ Step 25360/50462 | Loss: 2.5738 | LR: 2.54e-04 | 4011.2ms/step | 15.0 samples/s
67
+ Step 25370/50462 | Loss: 2.6424 | LR: 2.54e-04 | 4005.3ms/step | 15.0 samples/s
68
+ Step 25380/50462 | Loss: 2.6114 | LR: 2.54e-04 | 4004.3ms/step | 15.0 samples/s
69
+ Step 25390/50462 | Loss: 2.5956 | LR: 2.53e-04 | 4010.1ms/step | 15.0 samples/s
70
+ Step 25400/50462 | Loss: 2.5914 | LR: 2.53e-04 | 4000.5ms/step | 15.0 samples/s
71
+ Step 25410/50462 | Loss: 2.6131 | LR: 2.53e-04 | 4006.6ms/step | 15.0 samples/s
72
+ Step 25420/50462 | Loss: 2.6089 | LR: 2.53e-04 | 4009.9ms/step | 15.0 samples/s
73
+ Step 25430/50462 | Loss: 2.5995 | LR: 2.53e-04 | 4003.3ms/step | 15.0 samples/s
74
+ Step 25440/50462 | Loss: 2.6123 | LR: 2.53e-04 | 4013.5ms/step | 14.9 samples/s
75
+ Step 25450/50462 | Loss: 2.5886 | LR: 2.52e-04 | 4000.3ms/step | 15.0 samples/s
76
+ Step 25460/50462 | Loss: 2.5843 | LR: 2.52e-04 | 3999.2ms/step | 15.0 samples/s
77
+ Step 25470/50462 | Loss: 2.5661 | LR: 2.52e-04 | 4014.6ms/step | 14.9 samples/s
78
+ Step 25480/50462 | Loss: 2.5716 | LR: 2.52e-04 | 4007.7ms/step | 15.0 samples/s
79
+ Step 25490/50462 | Loss: 2.5875 | LR: 2.52e-04 | 4008.1ms/step | 15.0 samples/s
80
+ Step 25500/50462 | Loss: 2.5796 | LR: 2.52e-04 | 4008.9ms/step | 15.0 samples/s
81
+ Validation at step 25500: Loss = 2.6125, Perplexity = 13.63
82
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25500
83
+ Step 25510/50462 | Loss: 2.5794 | LR: 2.52e-04 | 5851.1ms/step | 10.3 samples/s
84
+ Step 25520/50462 | Loss: 2.5684 | LR: 2.51e-04 | 4010.6ms/step | 15.0 samples/s
85
+ Step 25530/50462 | Loss: 2.6146 | LR: 2.51e-04 | 4005.3ms/step | 15.0 samples/s
86
+ Step 25540/50462 | Loss: 2.5745 | LR: 2.51e-04 | 4018.4ms/step | 14.9 samples/s
87
+ Step 25550/50462 | Loss: 2.5861 | LR: 2.51e-04 | 4007.7ms/step | 15.0 samples/s
88
+ Step 25560/50462 | Loss: 2.6104 | LR: 2.51e-04 | 4017.7ms/step | 14.9 samples/s
89
+ Step 25570/50462 | Loss: 2.5718 | LR: 2.51e-04 | 4018.8ms/step | 14.9 samples/s
90
+ Step 25580/50462 | Loss: 2.5818 | LR: 2.50e-04 | 4003.7ms/step | 15.0 samples/s
91
+ Step 25590/50462 | Loss: 2.5771 | LR: 2.50e-04 | 4005.5ms/step | 15.0 samples/s
92
+ Step 25600/50462 | Loss: 2.5913 | LR: 2.50e-04 | 4008.1ms/step | 15.0 samples/s
93
+ Step 25610/50462 | Loss: 2.5877 | LR: 2.50e-04 | 4006.7ms/step | 15.0 samples/s
94
+ Step 25620/50462 | Loss: 2.5774 | LR: 2.50e-04 | 4007.5ms/step | 15.0 samples/s
95
+ Step 25630/50462 | Loss: 2.5982 | LR: 2.50e-04 | 4014.2ms/step | 14.9 samples/s
96
+ Step 25640/50462 | Loss: 2.6228 | LR: 2.49e-04 | 4011.7ms/step | 15.0 samples/s
97
+ Step 25650/50462 | Loss: 2.5732 | LR: 2.49e-04 | 4007.7ms/step | 15.0 samples/s
98
+ Step 25660/50462 | Loss: 2.5501 | LR: 2.49e-04 | 4004.7ms/step | 15.0 samples/s
99
+ Step 25670/50462 | Loss: 2.6400 | LR: 2.49e-04 | 4014.1ms/step | 14.9 samples/s
100
+ Step 25680/50462 | Loss: 2.5958 | LR: 2.49e-04 | 4007.3ms/step | 15.0 samples/s
101
+ Step 25690/50462 | Loss: 2.5751 | LR: 2.49e-04 | 4015.3ms/step | 14.9 samples/s
102
+ Step 25700/50462 | Loss: 2.5882 | LR: 2.49e-04 | 4020.7ms/step | 14.9 samples/s
103
+ Step 25710/50462 | Loss: 2.5797 | LR: 2.48e-04 | 3989.1ms/step | 15.0 samples/s
104
+ Step 25720/50462 | Loss: 2.6001 | LR: 2.48e-04 | 4006.1ms/step | 15.0 samples/s
105
+ Step 25730/50462 | Loss: 2.5964 | LR: 2.48e-04 | 4011.8ms/step | 15.0 samples/s
106
+ Step 25740/50462 | Loss: 2.6219 | LR: 2.48e-04 | 4006.0ms/step | 15.0 samples/s
107
+ Step 25750/50462 | Loss: 2.5926 | LR: 2.48e-04 | 4009.1ms/step | 15.0 samples/s
108
+ Validation at step 25750: Loss = 2.6111, Perplexity = 13.61
109
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-25750
110
+ Step 25760/50462 | Loss: 2.6253 | LR: 2.48e-04 | 5851.1ms/step | 10.3 samples/s
111
+ Step 25770/50462 | Loss: 2.6164 | LR: 2.47e-04 | 4013.8ms/step | 14.9 samples/s
112
+ Step 25780/50462 | Loss: 2.5851 | LR: 2.47e-04 | 4008.2ms/step | 15.0 samples/s
113
+ Step 25790/50462 | Loss: 2.6205 | LR: 2.47e-04 | 4016.5ms/step | 14.9 samples/s
114
+ Step 25800/50462 | Loss: 2.5798 | LR: 2.47e-04 | 3998.3ms/step | 15.0 samples/s
115
+ Step 25810/50462 | Loss: 2.5610 | LR: 2.47e-04 | 4017.9ms/step | 14.9 samples/s
116
+ Step 25820/50462 | Loss: 2.6066 | LR: 2.47e-04 | 4001.8ms/step | 15.0 samples/s
117
+ Step 25830/50462 | Loss: 2.6160 | LR: 2.46e-04 | 3995.7ms/step | 15.0 samples/s
118
+ Step 25840/50462 | Loss: 2.6181 | LR: 2.46e-04 | 4001.2ms/step | 15.0 samples/s
119
+ Step 25850/50462 | Loss: 2.6284 | LR: 2.46e-04 | 3997.9ms/step | 15.0 samples/s
120
+ Step 25860/50462 | Loss: 2.6206 | LR: 2.46e-04 | 4009.1ms/step | 15.0 samples/s
121
+ Step 25870/50462 | Loss: 2.5538 | LR: 2.46e-04 | 3995.3ms/step | 15.0 samples/s
122
+ Step 25880/50462 | Loss: 2.5933 | LR: 2.46e-04 | 3998.8ms/step | 15.0 samples/s
123
+ Step 25890/50462 | Loss: 2.5947 | LR: 2.46e-04 | 4009.0ms/step | 15.0 samples/s
124
+ Step 25900/50462 | Loss: 2.5718 | LR: 2.45e-04 | 4007.4ms/step | 15.0 samples/s
125
+ Step 25910/50462 | Loss: 2.5829 | LR: 2.45e-04 | 3994.6ms/step | 15.0 samples/s
126
+ Step 25920/50462 | Loss: 2.5713 | LR: 2.45e-04 | 4015.9ms/step | 14.9 samples/s
127
+ Step 25930/50462 | Loss: 2.5822 | LR: 2.45e-04 | 4021.0ms/step | 14.9 samples/s
128
+ Step 25940/50462 | Loss: 2.5811 | LR: 2.45e-04 | 4012.9ms/step | 15.0 samples/s
129
+ Step 25950/50462 | Loss: 2.5883 | LR: 2.45e-04 | 4004.9ms/step | 15.0 samples/s
130
+ Step 25960/50462 | Loss: 2.5899 | LR: 2.44e-04 | 3999.4ms/step | 15.0 samples/s
131
+ Step 25970/50462 | Loss: 2.5756 | LR: 2.44e-04 | 4004.6ms/step | 15.0 samples/s
132
+ Step 25980/50462 | Loss: 2.6009 | LR: 2.44e-04 | 4000.0ms/step | 15.0 samples/s
133
+ Step 25990/50462 | Loss: 2.6221 | LR: 2.44e-04 | 4004.7ms/step | 15.0 samples/s
134
+ Step 26000/50462 | Loss: 2.6309 | LR: 2.44e-04 | 4006.9ms/step | 15.0 samples/s
135
+ Validation at step 26000: Loss = 2.6093, Perplexity = 13.59
136
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-26000
137
+ Step 26010/50462 | Loss: 2.5757 | LR: 2.44e-04 | 5875.6ms/step | 10.2 samples/s
138
+ Step 26020/50462 | Loss: 2.6059 | LR: 2.43e-04 | 4577.6ms/step | 13.1 samples/s
139
+ Step 26030/50462 | Loss: 2.6089 | LR: 2.43e-04 | 4011.9ms/step | 15.0 samples/s
140
+ Step 26040/50462 | Loss: 2.5799 | LR: 2.43e-04 | 4003.3ms/step | 15.0 samples/s
141
+ Step 26050/50462 | Loss: 2.5462 | LR: 2.43e-04 | 4002.0ms/step | 15.0 samples/s
142
+ Step 26060/50462 | Loss: 2.5989 | LR: 2.43e-04 | 3997.1ms/step | 15.0 samples/s
143
+ Step 26070/50462 | Loss: 2.6032 | LR: 2.43e-04 | 4014.6ms/step | 14.9 samples/s
144
+ Step 26080/50462 | Loss: 2.5678 | LR: 2.43e-04 | 3999.7ms/step | 15.0 samples/s
145
+ Step 26090/50462 | Loss: 2.5768 | LR: 2.42e-04 | 4004.9ms/step | 15.0 samples/s
146
+ Step 26100/50462 | Loss: 2.5668 | LR: 2.42e-04 | 4004.5ms/step | 15.0 samples/s
147
+ Step 26110/50462 | Loss: 2.5622 | LR: 2.42e-04 | 4007.7ms/step | 15.0 samples/s
148
+ Step 26120/50462 | Loss: 2.5803 | LR: 2.42e-04 | 4008.1ms/step | 15.0 samples/s
149
+ Step 26130/50462 | Loss: 2.5797 | LR: 2.42e-04 | 3996.9ms/step | 15.0 samples/s
150
+ Step 26140/50462 | Loss: 2.5677 | LR: 2.42e-04 | 4011.0ms/step | 15.0 samples/s
151
+ Step 26150/50462 | Loss: 2.5590 | LR: 2.41e-04 | 4010.4ms/step | 15.0 samples/s
152
+ Step 26160/50462 | Loss: 2.5888 | LR: 2.41e-04 | 4022.6ms/step | 14.9 samples/s
153
+ Step 26170/50462 | Loss: 2.5621 | LR: 2.41e-04 | 4007.9ms/step | 15.0 samples/s
154
+ Step 26180/50462 | Loss: 2.5942 | LR: 2.41e-04 | 4011.6ms/step | 15.0 samples/s
155
+ Step 26190/50462 | Loss: 2.5780 | LR: 2.41e-04 | 3992.6ms/step | 15.0 samples/s
156
+ Step 26200/50462 | Loss: 2.5669 | LR: 2.41e-04 | 4010.1ms/step | 15.0 samples/s
157
+ Step 26210/50462 | Loss: 2.5655 | LR: 2.40e-04 | 4007.8ms/step | 15.0 samples/s
158
+ Step 26220/50462 | Loss: 2.6029 | LR: 2.40e-04 | 4005.1ms/step | 15.0 samples/s
159
+ Step 26230/50462 | Loss: 2.5883 | LR: 2.40e-04 | 4001.4ms/step | 15.0 samples/s
160
+ Step 26240/50462 | Loss: 2.6144 | LR: 2.40e-04 | 4014.2ms/step | 14.9 samples/s
161
+ Step 26250/50462 | Loss: 2.5718 | LR: 2.40e-04 | 4003.5ms/step | 15.0 samples/s
162
+ Validation at step 26250: Loss = 2.6074, Perplexity = 13.56
163
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-26250
164
+ Step 26260/50462 | Loss: 2.5904 | LR: 2.40e-04 | 5849.1ms/step | 10.3 samples/s
165
+ Step 26270/50462 | Loss: 2.5928 | LR: 2.40e-04 | 3998.3ms/step | 15.0 samples/s
166
+ Step 26280/50462 | Loss: 2.5835 | LR: 2.39e-04 | 4008.7ms/step | 15.0 samples/s
167
+ Step 26290/50462 | Loss: 2.5884 | LR: 2.39e-04 | 4001.1ms/step | 15.0 samples/s
168
+ Step 26300/50462 | Loss: 2.5676 | LR: 2.39e-04 | 3998.7ms/step | 15.0 samples/s
169
+ Step 26310/50462 | Loss: 2.6005 | LR: 2.39e-04 | 4005.8ms/step | 15.0 samples/s
170
+ Step 26320/50462 | Loss: 2.5575 | LR: 2.39e-04 | 4011.5ms/step | 15.0 samples/s
171
+ Step 26330/50462 | Loss: 2.5735 | LR: 2.39e-04 | 4004.5ms/step | 15.0 samples/s
172
+ Step 26340/50462 | Loss: 2.5866 | LR: 2.38e-04 | 4006.8ms/step | 15.0 samples/s
173
+ Step 26350/50462 | Loss: 2.5961 | LR: 2.38e-04 | 3999.2ms/step | 15.0 samples/s
174
+ Step 26360/50462 | Loss: 2.5831 | LR: 2.38e-04 | 4012.3ms/step | 15.0 samples/s
175
+ Step 26370/50462 | Loss: 2.5877 | LR: 2.38e-04 | 4002.2ms/step | 15.0 samples/s
176
+ Step 26380/50462 | Loss: 2.6219 | LR: 2.38e-04 | 4014.0ms/step | 14.9 samples/s
177
+ Step 26390/50462 | Loss: 2.6313 | LR: 2.38e-04 | 4015.9ms/step | 14.9 samples/s
178
+ Step 26400/50462 | Loss: 2.5812 | LR: 2.37e-04 | 4003.1ms/step | 15.0 samples/s
179
+ Step 26410/50462 | Loss: 2.5905 | LR: 2.37e-04 | 4584.8ms/step | 13.1 samples/s
180
+ Step 26420/50462 | Loss: 2.5765 | LR: 2.37e-04 | 4569.2ms/step | 13.1 samples/s
181
+ Step 26430/50462 | Loss: 2.6128 | LR: 2.37e-04 | 4006.3ms/step | 15.0 samples/s
182
+ Step 26440/50462 | Loss: 2.6147 | LR: 2.37e-04 | 4005.5ms/step | 15.0 samples/s
183
+ Step 26450/50462 | Loss: 2.5937 | LR: 2.37e-04 | 3990.9ms/step | 15.0 samples/s
184
+ Step 26460/50462 | Loss: 2.5672 | LR: 2.37e-04 | 3984.6ms/step | 15.1 samples/s
185
+ Step 26470/50462 | Loss: 2.6128 | LR: 2.36e-04 | 3993.2ms/step | 15.0 samples/s
186
+ Step 26480/50462 | Loss: 2.5686 | LR: 2.36e-04 | 4012.1ms/step | 15.0 samples/s
187
+ Step 26490/50462 | Loss: 2.5954 | LR: 2.36e-04 | 3999.3ms/step | 15.0 samples/s
188
+ Step 26500/50462 | Loss: 2.5816 | LR: 2.36e-04 | 4015.5ms/step | 14.9 samples/s
189
+ Validation at step 26500: Loss = 2.6047, Perplexity = 13.53
190
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-26500
191
+ Step 26510/50462 | Loss: 2.5851 | LR: 2.36e-04 | 5867.0ms/step | 10.2 samples/s
192
+ Step 26520/50462 | Loss: 2.5968 | LR: 2.36e-04 | 4000.7ms/step | 15.0 samples/s
193
+ Step 26530/50462 | Loss: 2.5959 | LR: 2.35e-04 | 4004.5ms/step | 15.0 samples/s
194
+ Step 26540/50462 | Loss: 2.5926 | LR: 2.35e-04 | 4014.6ms/step | 14.9 samples/s
195
+ Step 26550/50462 | Loss: 2.6270 | LR: 2.35e-04 | 4009.0ms/step | 15.0 samples/s
196
+ Step 26560/50462 | Loss: 2.5743 | LR: 2.35e-04 | 4003.2ms/step | 15.0 samples/s
197
+ Step 26570/50462 | Loss: 2.6094 | LR: 2.35e-04 | 4000.0ms/step | 15.0 samples/s
198
+ Step 26580/50462 | Loss: 2.5688 | LR: 2.35e-04 | 4015.5ms/step | 14.9 samples/s
199
+ Step 26590/50462 | Loss: 2.5596 | LR: 2.34e-04 | 4001.9ms/step | 15.0 samples/s
200
+ Step 26600/50462 | Loss: 2.6046 | LR: 2.34e-04 | 3998.3ms/step | 15.0 samples/s
201
+ Step 26610/50462 | Loss: 2.5984 | LR: 2.34e-04 | 4011.3ms/step | 15.0 samples/s
202
+ Step 26620/50462 | Loss: 2.5994 | LR: 2.34e-04 | 4010.7ms/step | 15.0 samples/s
203
+ Step 26630/50462 | Loss: 2.5994 | LR: 2.34e-04 | 4017.3ms/step | 14.9 samples/s
204
+ Step 26640/50462 | Loss: 2.5783 | LR: 2.34e-04 | 4006.5ms/step | 15.0 samples/s
205
+ Step 26650/50462 | Loss: 2.6084 | LR: 2.34e-04 | 4017.5ms/step | 14.9 samples/s
206
+ Step 26660/50462 | Loss: 2.6024 | LR: 2.33e-04 | 4007.2ms/step | 15.0 samples/s
207
+ Step 26670/50462 | Loss: 2.5304 | LR: 2.33e-04 | 4010.7ms/step | 15.0 samples/s
208
+ Step 26680/50462 | Loss: 2.6146 | LR: 2.33e-04 | 4579.8ms/step | 13.1 samples/s
209
+ Step 26690/50462 | Loss: 2.5709 | LR: 2.33e-04 | 4006.9ms/step | 15.0 samples/s
210
+ Step 26700/50462 | Loss: 2.5713 | LR: 2.33e-04 | 4014.6ms/step | 14.9 samples/s
211
+ Step 26710/50462 | Loss: 2.5760 | LR: 2.33e-04 | 4006.6ms/step | 15.0 samples/s
212
+ Step 26720/50462 | Loss: 2.5753 | LR: 2.32e-04 | 4012.9ms/step | 15.0 samples/s
213
+ Step 26730/50462 | Loss: 2.5968 | LR: 2.32e-04 | 4008.7ms/step | 15.0 samples/s
214
+ Step 26740/50462 | Loss: 2.5981 | LR: 2.32e-04 | 3997.3ms/step | 15.0 samples/s
215
+ Step 26750/50462 | Loss: 2.5618 | LR: 2.32e-04 | 4001.7ms/step | 15.0 samples/s
216
+ Validation at step 26750: Loss = 2.6021, Perplexity = 13.49
217
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-26750
218
+ Step 26760/50462 | Loss: 2.5784 | LR: 2.32e-04 | 5883.2ms/step | 10.2 samples/s
219
+ Step 26770/50462 | Loss: 2.5562 | LR: 2.32e-04 | 4007.5ms/step | 15.0 samples/s
220
+ Step 26780/50462 | Loss: 2.5895 | LR: 2.31e-04 | 3995.0ms/step | 15.0 samples/s
221
+ Step 26790/50462 | Loss: 2.6099 | LR: 2.31e-04 | 4007.4ms/step | 15.0 samples/s
222
+ Step 26800/50462 | Loss: 2.5785 | LR: 2.31e-04 | 4005.5ms/step | 15.0 samples/s
223
+ Step 26810/50462 | Loss: 2.5922 | LR: 2.31e-04 | 4006.7ms/step | 15.0 samples/s
224
+ Step 26820/50462 | Loss: 2.5990 | LR: 2.31e-04 | 4005.6ms/step | 15.0 samples/s
225
+ Step 26830/50462 | Loss: 2.5730 | LR: 2.31e-04 | 4001.7ms/step | 15.0 samples/s
226
+ Step 26840/50462 | Loss: 2.5676 | LR: 2.31e-04 | 4005.5ms/step | 15.0 samples/s
227
+ Step 26850/50462 | Loss: 2.5644 | LR: 2.30e-04 | 4015.5ms/step | 14.9 samples/s
228
+ Step 26860/50462 | Loss: 2.6272 | LR: 2.30e-04 | 4004.7ms/step | 15.0 samples/s
229
+ Step 26870/50462 | Loss: 2.5952 | LR: 2.30e-04 | 4014.7ms/step | 14.9 samples/s
230
+ Step 26880/50462 | Loss: 2.5862 | LR: 2.30e-04 | 3990.4ms/step | 15.0 samples/s
231
+ Step 26890/50462 | Loss: 2.5861 | LR: 2.30e-04 | 4005.7ms/step | 15.0 samples/s
232
+ Step 26900/50462 | Loss: 2.5933 | LR: 2.30e-04 | 3986.1ms/step | 15.1 samples/s
233
+ Step 26910/50462 | Loss: 2.5948 | LR: 2.29e-04 | 3998.3ms/step | 15.0 samples/s
234
+ Step 26920/50462 | Loss: 2.5595 | LR: 2.29e-04 | 4020.8ms/step | 14.9 samples/s
235
+ Step 26930/50462 | Loss: 2.5562 | LR: 2.29e-04 | 4005.7ms/step | 15.0 samples/s
236
+ Step 26940/50462 | Loss: 2.5914 | LR: 2.29e-04 | 3998.7ms/step | 15.0 samples/s
237
+ Step 26950/50462 | Loss: 2.5688 | LR: 2.29e-04 | 4019.0ms/step | 14.9 samples/s
238
+ Step 26960/50462 | Loss: 2.5958 | LR: 2.29e-04 | 4020.6ms/step | 14.9 samples/s
239
+ Step 26970/50462 | Loss: 2.5795 | LR: 2.28e-04 | 4011.3ms/step | 15.0 samples/s
240
+ Step 26980/50462 | Loss: 2.5710 | LR: 2.28e-04 | 4015.7ms/step | 14.9 samples/s
241
+ Step 26990/50462 | Loss: 2.5767 | LR: 2.28e-04 | 3993.9ms/step | 15.0 samples/s
242
+ Step 27000/50462 | Loss: 2.5847 | LR: 2.28e-04 | 4011.3ms/step | 15.0 samples/s
243
+ Validation at step 27000: Loss = 2.5992, Perplexity = 13.45
244
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-27000
245
+ Step 27010/50462 | Loss: 2.5624 | LR: 2.28e-04 | 5895.5ms/step | 10.2 samples/s
246
+ Step 27020/50462 | Loss: 2.5521 | LR: 2.28e-04 | 4010.3ms/step | 15.0 samples/s
247
+ Step 27030/50462 | Loss: 2.5952 | LR: 2.28e-04 | 3990.2ms/step | 15.0 samples/s
248
+ Step 27040/50462 | Loss: 2.5870 | LR: 2.27e-04 | 3998.5ms/step | 15.0 samples/s
249
+ Step 27050/50462 | Loss: 2.5833 | LR: 2.27e-04 | 4004.0ms/step | 15.0 samples/s
250
+ Step 27060/50462 | Loss: 2.5873 | LR: 2.27e-04 | 4008.3ms/step | 15.0 samples/s
251
+ Step 27070/50462 | Loss: 2.5696 | LR: 2.27e-04 | 4011.6ms/step | 15.0 samples/s
252
+ Step 27080/50462 | Loss: 2.5794 | LR: 2.27e-04 | 4005.2ms/step | 15.0 samples/s
253
+ Step 27090/50462 | Loss: 2.5960 | LR: 2.27e-04 | 4007.3ms/step | 15.0 samples/s
254
+ Step 27100/50462 | Loss: 2.5724 | LR: 2.26e-04 | 4003.8ms/step | 15.0 samples/s
255
+ Step 27110/50462 | Loss: 2.5812 | LR: 2.26e-04 | 4019.4ms/step | 14.9 samples/s
256
+ Step 27120/50462 | Loss: 2.5819 | LR: 2.26e-04 | 4011.2ms/step | 15.0 samples/s
257
+ Step 27130/50462 | Loss: 2.5860 | LR: 2.26e-04 | 4013.3ms/step | 15.0 samples/s
258
+ Step 27140/50462 | Loss: 2.5859 | LR: 2.26e-04 | 4000.8ms/step | 15.0 samples/s
259
+ Step 27150/50462 | Loss: 2.5593 | LR: 2.26e-04 | 4002.3ms/step | 15.0 samples/s
260
+ Step 27160/50462 | Loss: 2.5824 | LR: 2.25e-04 | 4015.2ms/step | 14.9 samples/s
261
+ Step 27170/50462 | Loss: 2.5691 | LR: 2.25e-04 | 4002.4ms/step | 15.0 samples/s
262
+ Step 27180/50462 | Loss: 2.5856 | LR: 2.25e-04 | 3998.7ms/step | 15.0 samples/s
263
+ Step 27190/50462 | Loss: 2.5932 | LR: 2.25e-04 | 4007.8ms/step | 15.0 samples/s
264
+ Step 27200/50462 | Loss: 2.5553 | LR: 2.25e-04 | 4004.2ms/step | 15.0 samples/s
265
+ Step 27210/50462 | Loss: 2.5560 | LR: 2.25e-04 | 4014.2ms/step | 14.9 samples/s
266
+ Step 27220/50462 | Loss: 2.5873 | LR: 2.25e-04 | 4016.1ms/step | 14.9 samples/s
267
+ Step 27230/50462 | Loss: 2.5729 | LR: 2.24e-04 | 4012.7ms/step | 15.0 samples/s
268
+ Step 27240/50462 | Loss: 2.5773 | LR: 2.24e-04 | 4002.4ms/step | 15.0 samples/s
269
+ Step 27250/50462 | Loss: 2.5795 | LR: 2.24e-04 | 4003.6ms/step | 15.0 samples/s
270
+ Validation at step 27250: Loss = 2.5992, Perplexity = 13.45
271
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-27250
272
+ Step 27260/50462 | Loss: 2.5288 | LR: 2.24e-04 | 5839.3ms/step | 10.3 samples/s
273
+ Step 27270/50462 | Loss: 2.5535 | LR: 2.24e-04 | 4008.9ms/step | 15.0 samples/s
274
+ Step 27280/50462 | Loss: 2.6165 | LR: 2.24e-04 | 4014.0ms/step | 14.9 samples/s
275
+ Step 27290/50462 | Loss: 2.5651 | LR: 2.23e-04 | 4008.2ms/step | 15.0 samples/s
276
+ Step 27300/50462 | Loss: 2.5612 | LR: 2.23e-04 | 4003.0ms/step | 15.0 samples/s
277
+ Step 27310/50462 | Loss: 2.5902 | LR: 2.23e-04 | 3991.2ms/step | 15.0 samples/s
278
+ Step 27320/50462 | Loss: 2.5794 | LR: 2.23e-04 | 4000.4ms/step | 15.0 samples/s
279
+ Step 27330/50462 | Loss: 2.5482 | LR: 2.23e-04 | 3983.2ms/step | 15.1 samples/s
280
+ Step 27340/50462 | Loss: 2.5846 | LR: 2.23e-04 | 3998.2ms/step | 15.0 samples/s
281
+ Step 27350/50462 | Loss: 2.5797 | LR: 2.23e-04 | 4003.6ms/step | 15.0 samples/s
282
+ Step 27360/50462 | Loss: 2.5544 | LR: 2.22e-04 | 3997.8ms/step | 15.0 samples/s
283
+ Step 27370/50462 | Loss: 2.6152 | LR: 2.22e-04 | 4017.1ms/step | 14.9 samples/s
284
+ Step 27380/50462 | Loss: 2.5583 | LR: 2.22e-04 | 4019.5ms/step | 14.9 samples/s
285
+ Step 27390/50462 | Loss: 2.5687 | LR: 2.22e-04 | 3997.0ms/step | 15.0 samples/s
286
+ Step 27400/50462 | Loss: 2.5816 | LR: 2.22e-04 | 4010.0ms/step | 15.0 samples/s
287
+ Step 27410/50462 | Loss: 2.5670 | LR: 2.22e-04 | 4009.4ms/step | 15.0 samples/s
288
+ Step 27420/50462 | Loss: 2.5594 | LR: 2.21e-04 | 4024.2ms/step | 14.9 samples/s
289
+ Step 27430/50462 | Loss: 2.5573 | LR: 2.21e-04 | 4011.0ms/step | 15.0 samples/s
290
+ Step 27440/50462 | Loss: 2.5756 | LR: 2.21e-04 | 4012.6ms/step | 15.0 samples/s
291
+ Step 27450/50462 | Loss: 2.5559 | LR: 2.21e-04 | 4015.7ms/step | 14.9 samples/s
292
+ Step 27460/50462 | Loss: 2.5533 | LR: 2.21e-04 | 4019.2ms/step | 14.9 samples/s
293
+ Step 27470/50462 | Loss: 2.5973 | LR: 2.21e-04 | 4006.1ms/step | 15.0 samples/s
294
+ Step 27480/50462 | Loss: 2.5765 | LR: 2.20e-04 | 4016.0ms/step | 14.9 samples/s
295
+ Step 27490/50462 | Loss: 2.5768 | LR: 2.20e-04 | 4016.9ms/step | 14.9 samples/s
296
+ Step 27500/50462 | Loss: 2.6068 | LR: 2.20e-04 | 4010.5ms/step | 15.0 samples/s
297
+ Validation at step 27500: Loss = 2.5974, Perplexity = 13.43
298
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-27500
299
+ Step 27510/50462 | Loss: 2.5780 | LR: 2.20e-04 | 5861.9ms/step | 10.2 samples/s
300
+ Step 27520/50462 | Loss: 2.5644 | LR: 2.20e-04 | 4005.0ms/step | 15.0 samples/s
301
+ Step 27530/50462 | Loss: 2.5793 | LR: 2.20e-04 | 4015.0ms/step | 14.9 samples/s
302
+ Step 27540/50462 | Loss: 2.5713 | LR: 2.20e-04 | 4009.4ms/step | 15.0 samples/s
303
+ Step 27550/50462 | Loss: 2.5615 | LR: 2.19e-04 | 4009.2ms/step | 15.0 samples/s
304
+ Step 27560/50462 | Loss: 2.6155 | LR: 2.19e-04 | 4015.3ms/step | 14.9 samples/s
305
+ Step 27570/50462 | Loss: 2.5759 | LR: 2.19e-04 | 4001.8ms/step | 15.0 samples/s
306
+ Step 27580/50462 | Loss: 2.5546 | LR: 2.19e-04 | 3998.3ms/step | 15.0 samples/s
307
+ Step 27590/50462 | Loss: 2.5781 | LR: 2.19e-04 | 4012.8ms/step | 15.0 samples/s
308
+ Step 27600/50462 | Loss: 2.5667 | LR: 2.19e-04 | 4011.0ms/step | 15.0 samples/s
309
+ Step 27610/50462 | Loss: 2.6037 | LR: 2.18e-04 | 4014.5ms/step | 14.9 samples/s
310
+ Step 27620/50462 | Loss: 2.5669 | LR: 2.18e-04 | 4004.2ms/step | 15.0 samples/s
311
+ Step 27630/50462 | Loss: 2.5614 | LR: 2.18e-04 | 4003.0ms/step | 15.0 samples/s
312
+ Step 27640/50462 | Loss: 2.5466 | LR: 2.18e-04 | 4015.4ms/step | 14.9 samples/s
313
+ Step 27650/50462 | Loss: 2.5726 | LR: 2.18e-04 | 4015.6ms/step | 14.9 samples/s
314
+ Step 27660/50462 | Loss: 2.5701 | LR: 2.18e-04 | 4012.6ms/step | 15.0 samples/s
315
+ Step 27670/50462 | Loss: 2.5745 | LR: 2.17e-04 | 3993.7ms/step | 15.0 samples/s
316
+ Step 27680/50462 | Loss: 2.5743 | LR: 2.17e-04 | 4004.5ms/step | 15.0 samples/s
317
+ Step 27690/50462 | Loss: 2.5834 | LR: 2.17e-04 | 4003.8ms/step | 15.0 samples/s
318
+ Step 27700/50462 | Loss: 2.5588 | LR: 2.17e-04 | 4002.1ms/step | 15.0 samples/s
319
+ Step 27710/50462 | Loss: 2.5688 | LR: 2.17e-04 | 4014.7ms/step | 14.9 samples/s
320
+ Step 27720/50462 | Loss: 2.5925 | LR: 2.17e-04 | 4004.3ms/step | 15.0 samples/s
321
+ Step 27730/50462 | Loss: 2.5488 | LR: 2.17e-04 | 4019.5ms/step | 14.9 samples/s
322
+ Step 27740/50462 | Loss: 2.5534 | LR: 2.16e-04 | 4006.9ms/step | 15.0 samples/s
323
+ Step 27750/50462 | Loss: 2.5913 | LR: 2.16e-04 | 4009.3ms/step | 15.0 samples/s
324
+ Validation at step 27750: Loss = 2.5953, Perplexity = 13.40
325
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-27750
326
+ Step 27760/50462 | Loss: 2.5951 | LR: 2.16e-04 | 5840.3ms/step | 10.3 samples/s
327
+ Step 27770/50462 | Loss: 2.5425 | LR: 2.16e-04 | 4010.4ms/step | 15.0 samples/s
328
+ Step 27780/50462 | Loss: 2.6086 | LR: 2.16e-04 | 4005.9ms/step | 15.0 samples/s
329
+ Step 27790/50462 | Loss: 2.5732 | LR: 2.16e-04 | 4022.7ms/step | 14.9 samples/s
330
+ Step 27800/50462 | Loss: 2.5568 | LR: 2.15e-04 | 4005.1ms/step | 15.0 samples/s
331
+ Step 27810/50462 | Loss: 2.5846 | LR: 2.15e-04 | 4001.1ms/step | 15.0 samples/s
332
+ Step 27820/50462 | Loss: 2.5673 | LR: 2.15e-04 | 3998.4ms/step | 15.0 samples/s
333
+ Step 27830/50462 | Loss: 2.5607 | LR: 2.15e-04 | 3996.8ms/step | 15.0 samples/s
334
+ Step 27840/50462 | Loss: 2.5557 | LR: 2.15e-04 | 4004.3ms/step | 15.0 samples/s
335
+ Step 27850/50462 | Loss: 2.5431 | LR: 2.15e-04 | 3997.8ms/step | 15.0 samples/s
336
+ Step 27860/50462 | Loss: 2.5941 | LR: 2.15e-04 | 4001.4ms/step | 15.0 samples/s
337
+ Step 27870/50462 | Loss: 2.5677 | LR: 2.14e-04 | 4001.7ms/step | 15.0 samples/s
338
+ Step 27880/50462 | Loss: 2.5614 | LR: 2.14e-04 | 4003.0ms/step | 15.0 samples/s
339
+ Step 27890/50462 | Loss: 2.5649 | LR: 2.14e-04 | 4014.0ms/step | 14.9 samples/s
340
+ Step 27900/50462 | Loss: 2.5661 | LR: 2.14e-04 | 4003.6ms/step | 15.0 samples/s
341
+ Step 27910/50462 | Loss: 2.5921 | LR: 2.14e-04 | 4012.8ms/step | 15.0 samples/s
342
+ Step 27920/50462 | Loss: 2.5863 | LR: 2.14e-04 | 4020.9ms/step | 14.9 samples/s
343
+ Step 27930/50462 | Loss: 2.5459 | LR: 2.13e-04 | 4013.5ms/step | 14.9 samples/s
344
+ Step 27940/50462 | Loss: 2.5811 | LR: 2.13e-04 | 4022.5ms/step | 14.9 samples/s
345
+ Step 27950/50462 | Loss: 2.5672 | LR: 2.13e-04 | 4029.0ms/step | 14.9 samples/s
346
+ Step 27960/50462 | Loss: 2.5672 | LR: 2.13e-04 | 4018.7ms/step | 14.9 samples/s
347
+ Step 27970/50462 | Loss: 2.5740 | LR: 2.13e-04 | 4034.9ms/step | 14.9 samples/s
348
+ Step 27980/50462 | Loss: 2.5634 | LR: 2.13e-04 | 4027.4ms/step | 14.9 samples/s
349
+ Step 27990/50462 | Loss: 2.5721 | LR: 2.12e-04 | 4015.7ms/step | 14.9 samples/s
350
+ Step 28000/50462 | Loss: 2.5381 | LR: 2.12e-04 | 4016.5ms/step | 14.9 samples/s
351
+ Validation at step 28000: Loss = 2.5933, Perplexity = 13.37
352
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-28000
353
+ Step 28010/50462 | Loss: 2.5490 | LR: 2.12e-04 | 5894.6ms/step | 10.2 samples/s
354
+ Step 28020/50462 | Loss: 2.5311 | LR: 2.12e-04 | 4004.6ms/step | 15.0 samples/s
355
+ Step 28030/50462 | Loss: 2.5633 | LR: 2.12e-04 | 4011.9ms/step | 15.0 samples/s
356
+ Step 28040/50462 | Loss: 2.5374 | LR: 2.12e-04 | 4011.1ms/step | 15.0 samples/s
357
+ Step 28050/50462 | Loss: 2.5810 | LR: 2.12e-04 | 4003.1ms/step | 15.0 samples/s
358
+ Step 28060/50462 | Loss: 2.5808 | LR: 2.11e-04 | 4009.9ms/step | 15.0 samples/s
359
+ Step 28070/50462 | Loss: 2.5684 | LR: 2.11e-04 | 3999.6ms/step | 15.0 samples/s
360
+ Step 28080/50462 | Loss: 2.5412 | LR: 2.11e-04 | 3997.5ms/step | 15.0 samples/s
361
+ Step 28090/50462 | Loss: 2.5767 | LR: 2.11e-04 | 4003.5ms/step | 15.0 samples/s
362
+ Step 28100/50462 | Loss: 2.5991 | LR: 2.11e-04 | 4009.9ms/step | 15.0 samples/s
363
+ Step 28110/50462 | Loss: 2.5731 | LR: 2.11e-04 | 3998.6ms/step | 15.0 samples/s
364
+ Step 28120/50462 | Loss: 2.5864 | LR: 2.10e-04 | 4004.2ms/step | 15.0 samples/s
365
+ Step 28130/50462 | Loss: 2.5934 | LR: 2.10e-04 | 4001.6ms/step | 15.0 samples/s
366
+ Step 28140/50462 | Loss: 2.6212 | LR: 2.10e-04 | 3998.4ms/step | 15.0 samples/s
367
+ Step 28150/50462 | Loss: 2.6062 | LR: 2.10e-04 | 4002.6ms/step | 15.0 samples/s
368
+ Step 28160/50462 | Loss: 2.5651 | LR: 2.10e-04 | 3999.5ms/step | 15.0 samples/s
369
+ Step 28170/50462 | Loss: 2.5805 | LR: 2.10e-04 | 4003.9ms/step | 15.0 samples/s
370
+ Step 28180/50462 | Loss: 2.5935 | LR: 2.10e-04 | 4016.4ms/step | 14.9 samples/s
371
+ Step 28190/50462 | Loss: 2.6050 | LR: 2.09e-04 | 4005.4ms/step | 15.0 samples/s
372
+ Step 28200/50462 | Loss: 2.5570 | LR: 2.09e-04 | 4002.3ms/step | 15.0 samples/s
373
+ Step 28210/50462 | Loss: 2.5839 | LR: 2.09e-04 | 4011.5ms/step | 15.0 samples/s
374
+ Step 28220/50462 | Loss: 2.5695 | LR: 2.09e-04 | 4012.0ms/step | 15.0 samples/s
375
+ Step 28230/50462 | Loss: 2.5700 | LR: 2.09e-04 | 4009.3ms/step | 15.0 samples/s
376
+ Step 28240/50462 | Loss: 2.6183 | LR: 2.09e-04 | 4005.4ms/step | 15.0 samples/s
377
+ Step 28250/50462 | Loss: 2.5520 | LR: 2.08e-04 | 4018.6ms/step | 14.9 samples/s
378
+ Validation at step 28250: Loss = 2.5917, Perplexity = 13.35
379
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-28250
380
+ Step 28260/50462 | Loss: 2.5770 | LR: 2.08e-04 | 5885.3ms/step | 10.2 samples/s
381
+ Step 28270/50462 | Loss: 2.5498 | LR: 2.08e-04 | 4017.2ms/step | 14.9 samples/s
382
+ Step 28280/50462 | Loss: 2.5914 | LR: 2.08e-04 | 4009.6ms/step | 15.0 samples/s
383
+ Step 28290/50462 | Loss: 2.5939 | LR: 2.08e-04 | 4004.7ms/step | 15.0 samples/s
384
+ Step 28300/50462 | Loss: 2.6334 | LR: 2.08e-04 | 4022.3ms/step | 14.9 samples/s
385
+ Step 28310/50462 | Loss: 2.6075 | LR: 2.07e-04 | 4008.0ms/step | 15.0 samples/s
386
+ Step 28320/50462 | Loss: 2.5609 | LR: 2.07e-04 | 4011.1ms/step | 15.0 samples/s
387
+ Step 28330/50462 | Loss: 2.5892 | LR: 2.07e-04 | 4006.7ms/step | 15.0 samples/s
388
+ Step 28340/50462 | Loss: 2.5694 | LR: 2.07e-04 | 4006.1ms/step | 15.0 samples/s
389
+ Step 28350/50462 | Loss: 2.5958 | LR: 2.07e-04 | 3993.1ms/step | 15.0 samples/s
390
+ Step 28360/50462 | Loss: 2.5802 | LR: 2.07e-04 | 4008.3ms/step | 15.0 samples/s
391
+ Step 28370/50462 | Loss: 2.5942 | LR: 2.07e-04 | 3988.8ms/step | 15.0 samples/s
392
+ Step 28380/50462 | Loss: 2.5433 | LR: 2.06e-04 | 4009.6ms/step | 15.0 samples/s
393
+ Step 28390/50462 | Loss: 2.5795 | LR: 2.06e-04 | 3999.4ms/step | 15.0 samples/s
394
+ Step 28400/50462 | Loss: 2.5751 | LR: 2.06e-04 | 4017.9ms/step | 14.9 samples/s
395
+ Step 28410/50462 | Loss: 2.5886 | LR: 2.06e-04 | 4007.0ms/step | 15.0 samples/s
396
+ Step 28420/50462 | Loss: 2.5385 | LR: 2.06e-04 | 4012.1ms/step | 15.0 samples/s
397
+ Step 28430/50462 | Loss: 2.5518 | LR: 2.06e-04 | 4010.0ms/step | 15.0 samples/s
398
+ Step 28440/50462 | Loss: 2.5702 | LR: 2.05e-04 | 3998.0ms/step | 15.0 samples/s
399
+ Step 28450/50462 | Loss: 2.5451 | LR: 2.05e-04 | 4002.5ms/step | 15.0 samples/s
400
+ Step 28460/50462 | Loss: 2.5790 | LR: 2.05e-04 | 4010.5ms/step | 15.0 samples/s
401
+ Step 28470/50462 | Loss: 2.5813 | LR: 2.05e-04 | 4009.9ms/step | 15.0 samples/s
402
+ Step 28480/50462 | Loss: 2.5294 | LR: 2.05e-04 | 4002.4ms/step | 15.0 samples/s
403
+ Step 28490/50462 | Loss: 2.5458 | LR: 2.05e-04 | 4005.4ms/step | 15.0 samples/s
404
+ Step 28500/50462 | Loss: 2.5619 | LR: 2.05e-04 | 4011.9ms/step | 15.0 samples/s
405
+ Validation at step 28500: Loss = 2.5907, Perplexity = 13.34
406
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-28500
407
+ Step 28510/50462 | Loss: 2.5414 | LR: 2.04e-04 | 5870.4ms/step | 10.2 samples/s
408
+ Step 28520/50462 | Loss: 2.5574 | LR: 2.04e-04 | 4006.7ms/step | 15.0 samples/s
409
+ Step 28530/50462 | Loss: 2.5582 | LR: 2.04e-04 | 3999.3ms/step | 15.0 samples/s
410
+ Step 28540/50462 | Loss: 2.5456 | LR: 2.04e-04 | 4014.6ms/step | 14.9 samples/s
411
+ Step 28550/50462 | Loss: 2.6087 | LR: 2.04e-04 | 4030.4ms/step | 14.9 samples/s
412
+ Step 28560/50462 | Loss: 2.5712 | LR: 2.04e-04 | 4014.0ms/step | 14.9 samples/s
413
+ Step 28570/50462 | Loss: 2.5804 | LR: 2.03e-04 | 4028.7ms/step | 14.9 samples/s
414
+ Step 28580/50462 | Loss: 2.5765 | LR: 2.03e-04 | 4002.1ms/step | 15.0 samples/s
415
+ Step 28590/50462 | Loss: 2.5983 | LR: 2.03e-04 | 4007.4ms/step | 15.0 samples/s
416
+ Step 28600/50462 | Loss: 2.5975 | LR: 2.03e-04 | 4007.6ms/step | 15.0 samples/s
417
+ Step 28610/50462 | Loss: 2.5641 | LR: 2.03e-04 | 4011.3ms/step | 15.0 samples/s
418
+ Step 28620/50462 | Loss: 2.5739 | LR: 2.03e-04 | 4000.8ms/step | 15.0 samples/s
419
+ Step 28630/50462 | Loss: 2.5955 | LR: 2.03e-04 | 4007.6ms/step | 15.0 samples/s
420
+ Step 28640/50462 | Loss: 2.5457 | LR: 2.02e-04 | 4002.4ms/step | 15.0 samples/s
421
+ Step 28650/50462 | Loss: 2.5545 | LR: 2.02e-04 | 4010.9ms/step | 15.0 samples/s
422
+ Step 28660/50462 | Loss: 2.6446 | LR: 2.02e-04 | 4015.7ms/step | 14.9 samples/s
423
+ Step 28670/50462 | Loss: 2.5769 | LR: 2.02e-04 | 4000.8ms/step | 15.0 samples/s
424
+ Step 28680/50462 | Loss: 2.5671 | LR: 2.02e-04 | 4003.6ms/step | 15.0 samples/s
425
+ Step 28690/50462 | Loss: 2.5597 | LR: 2.02e-04 | 3999.5ms/step | 15.0 samples/s
426
+ Step 28700/50462 | Loss: 2.5702 | LR: 2.01e-04 | 4006.2ms/step | 15.0 samples/s
427
+ Step 28710/50462 | Loss: 2.5655 | LR: 2.01e-04 | 4004.9ms/step | 15.0 samples/s
428
+ Step 28720/50462 | Loss: 2.5658 | LR: 2.01e-04 | 4011.5ms/step | 15.0 samples/s
429
+ Step 28730/50462 | Loss: 2.5536 | LR: 2.01e-04 | 4003.1ms/step | 15.0 samples/s
430
+ Step 28740/50462 | Loss: 2.5602 | LR: 2.01e-04 | 4012.9ms/step | 15.0 samples/s
431
+ Step 28750/50462 | Loss: 2.5693 | LR: 2.01e-04 | 4001.2ms/step | 15.0 samples/s
432
+ Validation at step 28750: Loss = 2.5895, Perplexity = 13.32
433
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-28750
434
+ Step 28760/50462 | Loss: 2.5592 | LR: 2.00e-04 | 5866.8ms/step | 10.2 samples/s
435
+ Step 28770/50462 | Loss: 2.5974 | LR: 2.00e-04 | 4006.1ms/step | 15.0 samples/s
436
+ Step 28780/50462 | Loss: 2.5860 | LR: 2.00e-04 | 4008.9ms/step | 15.0 samples/s
437
+ Step 28790/50462 | Loss: 2.5383 | LR: 2.00e-04 | 4008.2ms/step | 15.0 samples/s
438
+ Step 28800/50462 | Loss: 2.5707 | LR: 2.00e-04 | 4012.9ms/step | 15.0 samples/s
439
+ Step 28810/50462 | Loss: 2.5191 | LR: 2.00e-04 | 4005.3ms/step | 15.0 samples/s
440
+ Step 28820/50462 | Loss: 2.5949 | LR: 2.00e-04 | 4000.0ms/step | 15.0 samples/s
441
+ Step 28830/50462 | Loss: 2.5262 | LR: 1.99e-04 | 4002.3ms/step | 15.0 samples/s
442
+ Step 28840/50462 | Loss: 2.5327 | LR: 1.99e-04 | 4004.2ms/step | 15.0 samples/s
443
+ Step 28850/50462 | Loss: 2.5982 | LR: 1.99e-04 | 4000.7ms/step | 15.0 samples/s
444
+ Step 28860/50462 | Loss: 2.5656 | LR: 1.99e-04 | 4000.2ms/step | 15.0 samples/s
445
+ Step 28870/50462 | Loss: 2.5700 | LR: 1.99e-04 | 4008.4ms/step | 15.0 samples/s
446
+ Step 28880/50462 | Loss: 2.5216 | LR: 1.99e-04 | 4000.5ms/step | 15.0 samples/s
447
+ Step 28890/50462 | Loss: 2.5482 | LR: 1.98e-04 | 4013.8ms/step | 14.9 samples/s
448
+ Step 28900/50462 | Loss: 2.5608 | LR: 1.98e-04 | 4007.8ms/step | 15.0 samples/s
449
+ Step 28910/50462 | Loss: 2.5663 | LR: 1.98e-04 | 4005.6ms/step | 15.0 samples/s
450
+ Step 28920/50462 | Loss: 2.5897 | LR: 1.98e-04 | 4006.6ms/step | 15.0 samples/s
451
+ Step 28930/50462 | Loss: 2.5831 | LR: 1.98e-04 | 4012.4ms/step | 15.0 samples/s
452
+ Step 28940/50462 | Loss: 2.5523 | LR: 1.98e-04 | 3991.1ms/step | 15.0 samples/s
453
+ Step 28950/50462 | Loss: 2.5823 | LR: 1.98e-04 | 4002.7ms/step | 15.0 samples/s
454
+ Step 28960/50462 | Loss: 2.5692 | LR: 1.97e-04 | 4004.9ms/step | 15.0 samples/s
455
+ Step 28970/50462 | Loss: 2.5690 | LR: 1.97e-04 | 4007.8ms/step | 15.0 samples/s
456
+ Step 28980/50462 | Loss: 2.5435 | LR: 1.97e-04 | 4013.4ms/step | 14.9 samples/s
457
+ Step 28990/50462 | Loss: 2.5316 | LR: 1.97e-04 | 3998.6ms/step | 15.0 samples/s
458
+ Step 29000/50462 | Loss: 2.5825 | LR: 1.97e-04 | 3992.7ms/step | 15.0 samples/s
459
+ Validation at step 29000: Loss = 2.5891, Perplexity = 13.32
460
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-29000
461
+ Step 29010/50462 | Loss: 2.5562 | LR: 1.97e-04 | 5869.4ms/step | 10.2 samples/s
462
+ Step 29020/50462 | Loss: 2.5386 | LR: 1.96e-04 | 4005.9ms/step | 15.0 samples/s
463
+ Step 29030/50462 | Loss: 2.5817 | LR: 1.96e-04 | 4005.5ms/step | 15.0 samples/s
464
+ Step 29040/50462 | Loss: 2.5680 | LR: 1.96e-04 | 3999.7ms/step | 15.0 samples/s
465
+ Step 29050/50462 | Loss: 2.5984 | LR: 1.96e-04 | 4000.1ms/step | 15.0 samples/s
466
+ Step 29060/50462 | Loss: 2.5597 | LR: 1.96e-04 | 4006.0ms/step | 15.0 samples/s
467
+ Step 29070/50462 | Loss: 2.5676 | LR: 1.96e-04 | 4003.8ms/step | 15.0 samples/s
468
+ Step 29080/50462 | Loss: 2.5703 | LR: 1.96e-04 | 4000.9ms/step | 15.0 samples/s
469
+ Step 29090/50462 | Loss: 2.5794 | LR: 1.95e-04 | 4007.5ms/step | 15.0 samples/s
470
+ Step 29100/50462 | Loss: 2.5499 | LR: 1.95e-04 | 4002.6ms/step | 15.0 samples/s
471
+ Step 29110/50462 | Loss: 2.5474 | LR: 1.95e-04 | 4003.2ms/step | 15.0 samples/s
472
+ Step 29120/50462 | Loss: 2.5794 | LR: 1.95e-04 | 4016.3ms/step | 14.9 samples/s
473
+ Step 29130/50462 | Loss: 2.5724 | LR: 1.95e-04 | 4019.9ms/step | 14.9 samples/s
474
+ Step 29140/50462 | Loss: 2.5585 | LR: 1.95e-04 | 4005.7ms/step | 15.0 samples/s
475
+ Step 29150/50462 | Loss: 2.5533 | LR: 1.94e-04 | 4000.3ms/step | 15.0 samples/s
476
+ Step 29160/50462 | Loss: 2.5557 | LR: 1.94e-04 | 4012.1ms/step | 15.0 samples/s
477
+ Step 29170/50462 | Loss: 2.5831 | LR: 1.94e-04 | 4001.0ms/step | 15.0 samples/s
478
+ Step 29180/50462 | Loss: 2.5694 | LR: 1.94e-04 | 4003.3ms/step | 15.0 samples/s
479
+ Step 29190/50462 | Loss: 2.5534 | LR: 1.94e-04 | 4001.7ms/step | 15.0 samples/s
480
+ Step 29200/50462 | Loss: 2.5445 | LR: 1.94e-04 | 4008.4ms/step | 15.0 samples/s
481
+ Step 29210/50462 | Loss: 2.5859 | LR: 1.94e-04 | 3998.2ms/step | 15.0 samples/s
482
+ Step 29220/50462 | Loss: 2.5480 | LR: 1.93e-04 | 4018.2ms/step | 14.9 samples/s
483
+ Step 29230/50462 | Loss: 2.5323 | LR: 1.93e-04 | 4021.0ms/step | 14.9 samples/s
484
+ Step 29240/50462 | Loss: 2.5375 | LR: 1.93e-04 | 4000.4ms/step | 15.0 samples/s
485
+ Step 29250/50462 | Loss: 2.5777 | LR: 1.93e-04 | 4009.3ms/step | 15.0 samples/s
486
+ Validation at step 29250: Loss = 2.5859, Perplexity = 13.27
487
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-29250
488
+ Step 29260/50462 | Loss: 2.5489 | LR: 1.93e-04 | 5873.3ms/step | 10.2 samples/s
489
+ Step 29270/50462 | Loss: 2.5634 | LR: 1.93e-04 | 4014.9ms/step | 14.9 samples/s
490
+ Step 29280/50462 | Loss: 2.5499 | LR: 1.92e-04 | 4003.7ms/step | 15.0 samples/s
491
+ Step 29290/50462 | Loss: 2.5549 | LR: 1.92e-04 | 4014.3ms/step | 14.9 samples/s
492
+ Step 29300/50462 | Loss: 2.5512 | LR: 1.92e-04 | 4001.5ms/step | 15.0 samples/s
493
+ Step 29310/50462 | Loss: 2.5813 | LR: 1.92e-04 | 3991.9ms/step | 15.0 samples/s
494
+ Step 29320/50462 | Loss: 2.5834 | LR: 1.92e-04 | 3994.5ms/step | 15.0 samples/s
495
+ Step 29330/50462 | Loss: 2.5751 | LR: 1.92e-04 | 4006.5ms/step | 15.0 samples/s
496
+ Step 29340/50462 | Loss: 2.5664 | LR: 1.92e-04 | 3997.5ms/step | 15.0 samples/s
497
+ Step 29350/50462 | Loss: 2.5649 | LR: 1.91e-04 | 4003.8ms/step | 15.0 samples/s
498
+ Step 29360/50462 | Loss: 2.5791 | LR: 1.91e-04 | 4004.5ms/step | 15.0 samples/s
499
+ Step 29370/50462 | Loss: 2.5702 | LR: 1.91e-04 | 3996.9ms/step | 15.0 samples/s
500
+ Step 29380/50462 | Loss: 2.5543 | LR: 1.91e-04 | 4007.5ms/step | 15.0 samples/s
501
+ Step 29390/50462 | Loss: 2.5705 | LR: 1.91e-04 | 4006.1ms/step | 15.0 samples/s
502
+ Step 29400/50462 | Loss: 2.5745 | LR: 1.91e-04 | 3992.8ms/step | 15.0 samples/s
503
+ Step 29410/50462 | Loss: 2.5687 | LR: 1.90e-04 | 3993.1ms/step | 15.0 samples/s
504
+ Step 29420/50462 | Loss: 2.5918 | LR: 1.90e-04 | 4009.4ms/step | 15.0 samples/s
505
+ Step 29430/50462 | Loss: 2.5579 | LR: 1.90e-04 | 4004.4ms/step | 15.0 samples/s
506
+ Step 29440/50462 | Loss: 2.5625 | LR: 1.90e-04 | 3993.8ms/step | 15.0 samples/s
507
+ Step 29450/50462 | Loss: 2.5681 | LR: 1.90e-04 | 4017.3ms/step | 14.9 samples/s
508
+ Step 29460/50462 | Loss: 2.5599 | LR: 1.90e-04 | 4000.2ms/step | 15.0 samples/s
509
+ Step 29470/50462 | Loss: 2.5667 | LR: 1.90e-04 | 4003.5ms/step | 15.0 samples/s
510
+ Step 29480/50462 | Loss: 2.5556 | LR: 1.89e-04 | 4010.0ms/step | 15.0 samples/s
511
+ Step 29490/50462 | Loss: 2.5405 | LR: 1.89e-04 | 4003.3ms/step | 15.0 samples/s
512
+ Step 29500/50462 | Loss: 2.5687 | LR: 1.89e-04 | 3999.3ms/step | 15.0 samples/s
513
+ Validation at step 29500: Loss = 2.5844, Perplexity = 13.25
514
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-29500
515
+ Step 29510/50462 | Loss: 2.5743 | LR: 1.89e-04 | 5852.2ms/step | 10.3 samples/s
516
+ Step 29520/50462 | Loss: 2.5636 | LR: 1.89e-04 | 4017.4ms/step | 14.9 samples/s
517
+ Step 29530/50462 | Loss: 2.5431 | LR: 1.89e-04 | 4001.7ms/step | 15.0 samples/s
518
+ Step 29540/50462 | Loss: 2.5577 | LR: 1.88e-04 | 3995.3ms/step | 15.0 samples/s
519
+ Step 29550/50462 | Loss: 2.5452 | LR: 1.88e-04 | 4003.8ms/step | 15.0 samples/s
520
+ Step 29560/50462 | Loss: 2.5712 | LR: 1.88e-04 | 4009.4ms/step | 15.0 samples/s
521
+ Step 29570/50462 | Loss: 2.5476 | LR: 1.88e-04 | 4011.2ms/step | 15.0 samples/s
522
+ Step 29580/50462 | Loss: 2.5774 | LR: 1.88e-04 | 4002.4ms/step | 15.0 samples/s
523
+ Step 29590/50462 | Loss: 2.6001 | LR: 1.88e-04 | 4005.2ms/step | 15.0 samples/s
524
+ Step 29600/50462 | Loss: 2.5495 | LR: 1.88e-04 | 3991.3ms/step | 15.0 samples/s
525
+ Step 29610/50462 | Loss: 2.5557 | LR: 1.87e-04 | 4007.3ms/step | 15.0 samples/s
526
+ Step 29620/50462 | Loss: 2.5748 | LR: 1.87e-04 | 4004.3ms/step | 15.0 samples/s
527
+ Step 29630/50462 | Loss: 2.5855 | LR: 1.87e-04 | 4002.6ms/step | 15.0 samples/s
528
+ Step 29640/50462 | Loss: 2.5613 | LR: 1.87e-04 | 4000.3ms/step | 15.0 samples/s
529
+ Step 29650/50462 | Loss: 2.5708 | LR: 1.87e-04 | 3994.7ms/step | 15.0 samples/s
530
+ Step 29660/50462 | Loss: 2.5740 | LR: 1.87e-04 | 3997.5ms/step | 15.0 samples/s
531
+ Step 29670/50462 | Loss: 2.5639 | LR: 1.86e-04 | 3997.7ms/step | 15.0 samples/s
532
+ Step 29680/50462 | Loss: 2.5530 | LR: 1.86e-04 | 4003.8ms/step | 15.0 samples/s
533
+ Step 29690/50462 | Loss: 2.5658 | LR: 1.86e-04 | 4004.6ms/step | 15.0 samples/s
534
+ Step 29700/50462 | Loss: 2.5620 | LR: 1.86e-04 | 4008.5ms/step | 15.0 samples/s
535
+ Step 29710/50462 | Loss: 2.5647 | LR: 1.86e-04 | 3995.1ms/step | 15.0 samples/s
536
+ Step 29720/50462 | Loss: 2.5907 | LR: 1.86e-04 | 4014.5ms/step | 14.9 samples/s
537
+ Step 29730/50462 | Loss: 2.5404 | LR: 1.86e-04 | 4011.6ms/step | 15.0 samples/s
538
+ Step 29740/50462 | Loss: 2.5723 | LR: 1.85e-04 | 4012.1ms/step | 15.0 samples/s
539
+ Step 29750/50462 | Loss: 2.5618 | LR: 1.85e-04 | 4006.5ms/step | 15.0 samples/s
540
+ Validation at step 29750: Loss = 2.5834, Perplexity = 13.24
541
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-29750
542
+ Step 29760/50462 | Loss: 2.5587 | LR: 1.85e-04 | 5853.0ms/step | 10.3 samples/s
543
+ Step 29770/50462 | Loss: 2.5630 | LR: 1.85e-04 | 4004.5ms/step | 15.0 samples/s
544
+ Step 29780/50462 | Loss: 2.5772 | LR: 1.85e-04 | 3996.0ms/step | 15.0 samples/s
545
+ Step 29790/50462 | Loss: 2.5865 | LR: 1.85e-04 | 3999.8ms/step | 15.0 samples/s
546
+ Step 29800/50462 | Loss: 2.5858 | LR: 1.85e-04 | 4010.1ms/step | 15.0 samples/s
547
+ Step 29810/50462 | Loss: 2.5574 | LR: 1.84e-04 | 4001.6ms/step | 15.0 samples/s
548
+ Step 29820/50462 | Loss: 2.5411 | LR: 1.84e-04 | 4003.8ms/step | 15.0 samples/s
549
+ Step 29830/50462 | Loss: 2.5659 | LR: 1.84e-04 | 4001.9ms/step | 15.0 samples/s
550
+ Step 29840/50462 | Loss: 2.5662 | LR: 1.84e-04 | 4012.3ms/step | 15.0 samples/s
551
+ Step 29850/50462 | Loss: 2.5259 | LR: 1.84e-04 | 4009.5ms/step | 15.0 samples/s
552
+ Step 29860/50462 | Loss: 2.5950 | LR: 1.84e-04 | 4007.7ms/step | 15.0 samples/s
553
+ Step 29870/50462 | Loss: 2.5322 | LR: 1.83e-04 | 3994.1ms/step | 15.0 samples/s
554
+ Step 29880/50462 | Loss: 2.5846 | LR: 1.83e-04 | 3996.7ms/step | 15.0 samples/s
555
+ Step 29890/50462 | Loss: 2.6032 | LR: 1.83e-04 | 3995.2ms/step | 15.0 samples/s
556
+ Step 29900/50462 | Loss: 2.5548 | LR: 1.83e-04 | 4006.2ms/step | 15.0 samples/s
557
+ Step 29910/50462 | Loss: 2.5736 | LR: 1.83e-04 | 3996.3ms/step | 15.0 samples/s
558
+ Step 29920/50462 | Loss: 2.5823 | LR: 1.83e-04 | 4006.2ms/step | 15.0 samples/s
559
+ Step 29930/50462 | Loss: 2.5870 | LR: 1.83e-04 | 4002.7ms/step | 15.0 samples/s
560
+ Step 29940/50462 | Loss: 2.5801 | LR: 1.82e-04 | 4006.5ms/step | 15.0 samples/s
561
+ Step 29950/50462 | Loss: 2.5625 | LR: 1.82e-04 | 4016.4ms/step | 14.9 samples/s
562
+ Step 29960/50462 | Loss: 2.5707 | LR: 1.82e-04 | 4005.7ms/step | 15.0 samples/s
563
+ Step 29970/50462 | Loss: 2.5408 | LR: 1.82e-04 | 4021.1ms/step | 14.9 samples/s
564
+ Step 29980/50462 | Loss: 2.5530 | LR: 1.82e-04 | 3996.7ms/step | 15.0 samples/s
565
+ Step 29990/50462 | Loss: 2.5622 | LR: 1.82e-04 | 4002.1ms/step | 15.0 samples/s
566
+ Step 30000/50462 | Loss: 2.5579 | LR: 1.81e-04 | 4003.4ms/step | 15.0 samples/s
567
+ Validation at step 30000: Loss = 2.5832, Perplexity = 13.24
568
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-30000
569
+ Step 30010/50462 | Loss: 2.5898 | LR: 1.81e-04 | 5852.7ms/step | 10.3 samples/s
570
+ Step 30020/50462 | Loss: 2.5715 | LR: 1.81e-04 | 4015.9ms/step | 14.9 samples/s
571
+ Step 30030/50462 | Loss: 2.5664 | LR: 1.81e-04 | 4003.1ms/step | 15.0 samples/s
572
+ Step 30040/50462 | Loss: 2.5294 | LR: 1.81e-04 | 4006.7ms/step | 15.0 samples/s
573
+ Step 30050/50462 | Loss: 2.5646 | LR: 1.81e-04 | 4012.6ms/step | 15.0 samples/s
574
+ Step 30060/50462 | Loss: 2.5616 | LR: 1.81e-04 | 4012.7ms/step | 15.0 samples/s
575
+ Step 30070/50462 | Loss: 2.5653 | LR: 1.80e-04 | 4006.8ms/step | 15.0 samples/s
576
+ Step 30080/50462 | Loss: 2.5485 | LR: 1.80e-04 | 4005.0ms/step | 15.0 samples/s
577
+ Step 30090/50462 | Loss: 2.5562 | LR: 1.80e-04 | 4010.4ms/step | 15.0 samples/s
578
+ Step 30100/50462 | Loss: 2.5918 | LR: 1.80e-04 | 4006.0ms/step | 15.0 samples/s
579
+ Step 30110/50462 | Loss: 2.5876 | LR: 1.80e-04 | 3999.9ms/step | 15.0 samples/s
580
+ Step 30120/50462 | Loss: 2.5459 | LR: 1.80e-04 | 4007.6ms/step | 15.0 samples/s
581
+ Step 30130/50462 | Loss: 2.5566 | LR: 1.79e-04 | 4009.4ms/step | 15.0 samples/s
582
+ Step 30140/50462 | Loss: 2.5404 | LR: 1.79e-04 | 4022.0ms/step | 14.9 samples/s
583
+ Step 30150/50462 | Loss: 2.5443 | LR: 1.79e-04 | 4006.7ms/step | 15.0 samples/s
584
+ Step 30160/50462 | Loss: 2.5797 | LR: 1.79e-04 | 4004.0ms/step | 15.0 samples/s
585
+ Step 30170/50462 | Loss: 2.5887 | LR: 1.79e-04 | 3998.9ms/step | 15.0 samples/s
586
+ Step 30180/50462 | Loss: 2.5581 | LR: 1.79e-04 | 4003.6ms/step | 15.0 samples/s
587
+ Step 30190/50462 | Loss: 2.5326 | LR: 1.79e-04 | 4004.4ms/step | 15.0 samples/s
588
+ Step 30200/50462 | Loss: 2.5533 | LR: 1.78e-04 | 4006.7ms/step | 15.0 samples/s
589
+ Step 30210/50462 | Loss: 2.5682 | LR: 1.78e-04 | 4001.0ms/step | 15.0 samples/s
590
+ Step 30220/50462 | Loss: 2.5202 | LR: 1.78e-04 | 4004.0ms/step | 15.0 samples/s
591
+ Step 30230/50462 | Loss: 2.5500 | LR: 1.78e-04 | 3993.5ms/step | 15.0 samples/s
592
+ Step 30240/50462 | Loss: 2.5903 | LR: 1.78e-04 | 4010.8ms/step | 15.0 samples/s
593
+ Step 30250/50462 | Loss: 2.5532 | LR: 1.78e-04 | 4005.9ms/step | 15.0 samples/s
594
+ Validation at step 30250: Loss = 2.5810, Perplexity = 13.21
595
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-30250
596
+ Step 30260/50462 | Loss: 2.5700 | LR: 1.78e-04 | 5862.8ms/step | 10.2 samples/s
597
+ Step 30270/50462 | Loss: 2.5551 | LR: 1.77e-04 | 3996.2ms/step | 15.0 samples/s
598
+ Step 30280/50462 | Loss: 2.5504 | LR: 1.77e-04 | 4012.7ms/step | 15.0 samples/s
599
+ Step 30290/50462 | Loss: 2.5582 | LR: 1.77e-04 | 4014.0ms/step | 14.9 samples/s
600
+ Step 30300/50462 | Loss: 2.5591 | LR: 1.77e-04 | 4000.4ms/step | 15.0 samples/s
601
+ Step 30310/50462 | Loss: 2.5591 | LR: 1.77e-04 | 4007.8ms/step | 15.0 samples/s
602
+ Step 30320/50462 | Loss: 2.5626 | LR: 1.77e-04 | 4014.1ms/step | 14.9 samples/s
603
+ Step 30330/50462 | Loss: 2.5666 | LR: 1.76e-04 | 3997.4ms/step | 15.0 samples/s
604
+ Step 30340/50462 | Loss: 2.5596 | LR: 1.76e-04 | 4011.8ms/step | 15.0 samples/s
605
+ Step 30350/50462 | Loss: 2.5726 | LR: 1.76e-04 | 4016.6ms/step | 14.9 samples/s
606
+ Step 30360/50462 | Loss: 2.5712 | LR: 1.76e-04 | 4009.3ms/step | 15.0 samples/s
607
+ Step 30370/50462 | Loss: 2.5384 | LR: 1.76e-04 | 4020.0ms/step | 14.9 samples/s
608
+ Step 30380/50462 | Loss: 2.5522 | LR: 1.76e-04 | 4008.0ms/step | 15.0 samples/s
609
+ Step 30390/50462 | Loss: 2.5477 | LR: 1.76e-04 | 4016.0ms/step | 14.9 samples/s
610
+ Step 30400/50462 | Loss: 2.5332 | LR: 1.75e-04 | 4006.4ms/step | 15.0 samples/s
611
+ Step 30410/50462 | Loss: 2.5274 | LR: 1.75e-04 | 4002.5ms/step | 15.0 samples/s
612
+ Step 30420/50462 | Loss: 2.5765 | LR: 1.75e-04 | 4006.4ms/step | 15.0 samples/s
613
+ Step 30430/50462 | Loss: 2.5598 | LR: 1.75e-04 | 4014.6ms/step | 14.9 samples/s
614
+ Step 30440/50462 | Loss: 2.5764 | LR: 1.75e-04 | 4006.7ms/step | 15.0 samples/s
615
+ Step 30450/50462 | Loss: 2.5843 | LR: 1.75e-04 | 4017.0ms/step | 14.9 samples/s
616
+ Step 30460/50462 | Loss: 2.5639 | LR: 1.75e-04 | 4005.8ms/step | 15.0 samples/s
617
+ Step 30470/50462 | Loss: 2.5510 | LR: 1.74e-04 | 4002.6ms/step | 15.0 samples/s
618
+ Step 30480/50462 | Loss: 2.5552 | LR: 1.74e-04 | 4018.6ms/step | 14.9 samples/s
619
+ Step 30490/50462 | Loss: 2.5640 | LR: 1.74e-04 | 4012.3ms/step | 15.0 samples/s
620
+ Step 30500/50462 | Loss: 2.5819 | LR: 1.74e-04 | 4031.2ms/step | 14.9 samples/s
621
+ Validation at step 30500: Loss = 2.5793, Perplexity = 13.19
622
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-30500
623
+ Step 30510/50462 | Loss: 2.5373 | LR: 1.74e-04 | 5904.7ms/step | 10.2 samples/s
624
+ Step 30520/50462 | Loss: 2.5769 | LR: 1.74e-04 | 4021.2ms/step | 14.9 samples/s
625
+ Step 30530/50462 | Loss: 2.5310 | LR: 1.73e-04 | 4027.2ms/step | 14.9 samples/s
626
+ Step 30540/50462 | Loss: 2.5460 | LR: 1.73e-04 | 4018.7ms/step | 14.9 samples/s
627
+ Step 30550/50462 | Loss: 2.5802 | LR: 1.73e-04 | 4010.8ms/step | 15.0 samples/s
628
+ Step 30560/50462 | Loss: 2.5552 | LR: 1.73e-04 | 4018.1ms/step | 14.9 samples/s
629
+ Step 30570/50462 | Loss: 2.5901 | LR: 1.73e-04 | 4021.2ms/step | 14.9 samples/s
630
+ Step 30580/50462 | Loss: 2.5939 | LR: 1.73e-04 | 4012.2ms/step | 15.0 samples/s
631
+ Step 30590/50462 | Loss: 2.5396 | LR: 1.73e-04 | 4015.6ms/step | 14.9 samples/s
632
+ Step 30600/50462 | Loss: 2.5539 | LR: 1.72e-04 | 4019.5ms/step | 14.9 samples/s
633
+ Step 30610/50462 | Loss: 2.5610 | LR: 1.72e-04 | 4025.5ms/step | 14.9 samples/s
634
+ Step 30620/50462 | Loss: 2.5506 | LR: 1.72e-04 | 4017.3ms/step | 14.9 samples/s
635
+ Step 30630/50462 | Loss: 2.5696 | LR: 1.72e-04 | 4014.5ms/step | 14.9 samples/s
636
+ Step 30640/50462 | Loss: 2.5991 | LR: 1.72e-04 | 4013.0ms/step | 15.0 samples/s
637
+ Step 30650/50462 | Loss: 2.5531 | LR: 1.72e-04 | 3997.6ms/step | 15.0 samples/s
638
+ Step 30660/50462 | Loss: 2.5453 | LR: 1.72e-04 | 4012.7ms/step | 15.0 samples/s
639
+ Step 30670/50462 | Loss: 2.5743 | LR: 1.71e-04 | 3997.0ms/step | 15.0 samples/s
640
+ Step 30680/50462 | Loss: 2.5364 | LR: 1.71e-04 | 4004.3ms/step | 15.0 samples/s
641
+ Step 30690/50462 | Loss: 2.5624 | LR: 1.71e-04 | 4001.6ms/step | 15.0 samples/s
642
+ Step 30700/50462 | Loss: 2.5896 | LR: 1.71e-04 | 4008.3ms/step | 15.0 samples/s
643
+ Step 30710/50462 | Loss: 2.5467 | LR: 1.71e-04 | 4004.0ms/step | 15.0 samples/s
644
+ Step 30720/50462 | Loss: 2.5763 | LR: 1.71e-04 | 4004.2ms/step | 15.0 samples/s
645
+ Step 30730/50462 | Loss: 2.5569 | LR: 1.70e-04 | 3999.6ms/step | 15.0 samples/s
646
+ Step 30740/50462 | Loss: 2.5467 | LR: 1.70e-04 | 4016.9ms/step | 14.9 samples/s
647
+ Step 30750/50462 | Loss: 2.5480 | LR: 1.70e-04 | 4001.2ms/step | 15.0 samples/s
648
+ Validation at step 30750: Loss = 2.5788, Perplexity = 13.18
649
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-30750
650
+ Step 30760/50462 | Loss: 2.5538 | LR: 1.70e-04 | 5899.9ms/step | 10.2 samples/s
651
+ Step 30770/50462 | Loss: 2.5540 | LR: 1.70e-04 | 4001.1ms/step | 15.0 samples/s
652
+ Step 30780/50462 | Loss: 2.5516 | LR: 1.70e-04 | 4011.6ms/step | 15.0 samples/s
653
+ Step 30790/50462 | Loss: 2.5753 | LR: 1.70e-04 | 4005.6ms/step | 15.0 samples/s
654
+ Step 30800/50462 | Loss: 2.5549 | LR: 1.69e-04 | 4004.0ms/step | 15.0 samples/s
655
+ Step 30810/50462 | Loss: 2.5904 | LR: 1.69e-04 | 4010.8ms/step | 15.0 samples/s
656
+ Step 30820/50462 | Loss: 2.5860 | LR: 1.69e-04 | 3992.8ms/step | 15.0 samples/s
657
+ Step 30830/50462 | Loss: 2.5678 | LR: 1.69e-04 | 4005.5ms/step | 15.0 samples/s
658
+ Step 30840/50462 | Loss: 2.5455 | LR: 1.69e-04 | 3996.7ms/step | 15.0 samples/s
659
+ Step 30850/50462 | Loss: 2.5766 | LR: 1.69e-04 | 3995.5ms/step | 15.0 samples/s
660
+ Step 30860/50462 | Loss: 2.5614 | LR: 1.69e-04 | 4006.8ms/step | 15.0 samples/s
661
+ Step 30870/50462 | Loss: 2.5543 | LR: 1.68e-04 | 3988.2ms/step | 15.0 samples/s
662
+ Step 30880/50462 | Loss: 2.5550 | LR: 1.68e-04 | 3999.8ms/step | 15.0 samples/s
663
+ Step 30890/50462 | Loss: 2.5567 | LR: 1.68e-04 | 3996.6ms/step | 15.0 samples/s
664
+ Step 30900/50462 | Loss: 2.5522 | LR: 1.68e-04 | 3998.5ms/step | 15.0 samples/s
665
+ Step 30910/50462 | Loss: 2.5210 | LR: 1.68e-04 | 4025.7ms/step | 14.9 samples/s
666
+ Step 30920/50462 | Loss: 2.5636 | LR: 1.68e-04 | 4008.1ms/step | 15.0 samples/s
667
+ Step 30930/50462 | Loss: 2.5697 | LR: 1.67e-04 | 4009.0ms/step | 15.0 samples/s
668
+ Step 30940/50462 | Loss: 2.5319 | LR: 1.67e-04 | 3999.9ms/step | 15.0 samples/s
669
+ Step 30950/50462 | Loss: 2.5696 | LR: 1.67e-04 | 4001.9ms/step | 15.0 samples/s
670
+ Step 30960/50462 | Loss: 2.5869 | LR: 1.67e-04 | 4001.9ms/step | 15.0 samples/s
671
+ Step 30970/50462 | Loss: 2.5536 | LR: 1.67e-04 | 4010.4ms/step | 15.0 samples/s
672
+ Step 30980/50462 | Loss: 2.5406 | LR: 1.67e-04 | 3999.4ms/step | 15.0 samples/s
673
+ Step 30990/50462 | Loss: 2.5526 | LR: 1.67e-04 | 4005.8ms/step | 15.0 samples/s
674
+ Step 31000/50462 | Loss: 2.5669 | LR: 1.66e-04 | 4003.8ms/step | 15.0 samples/s
675
+ Validation at step 31000: Loss = 2.5774, Perplexity = 13.16
676
+ Saved checkpoint to /mloscratch/homes/navasard/sparse/checkpoints/l8k1p2b_mixed_seed72/run_20260827_071551/checkpoint-31000
epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/files/requirements.txt ADDED
@@ -0,0 +1,142 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ pip==26.0.1
2
+ nvidia-cusparselt-cu13==0.8.0
3
+ mpmath==1.3.0
4
+ cuda-toolkit==13.0.2
5
+ zstandard==0.25.0
6
+ xxhash==3.7.0
7
+ urllib3==2.6.3
8
+ typing_extensions==4.15.0
9
+ triton==3.6.0
10
+ tqdm==4.67.3
11
+ torchinfo==1.8.0
12
+ sympy==1.14.0
13
+ smmap==5.0.3
14
+ six==1.17.0
15
+ shellingham==1.5.4
16
+ setuptools==81.0.0
17
+ safetensors==0.7.0
18
+ regex==2026.4.4
19
+ PyYAML==6.0.3
20
+ Pygments==2.20.0
21
+ pyarrow==24.0.0
22
+ psutil==7.2.2
23
+ protobuf==7.34.1
24
+ propcache==0.4.1
25
+ platformdirs==4.9.6
26
+ packaging==26.2
27
+ nvidia-nvtx==13.0.85
28
+ nvidia-nvshmem-cu13==3.4.5
29
+ nvidia-nvjitlink==13.0.88
30
+ nvidia-nccl-cu13==2.28.9
31
+ nvidia-curand==10.4.0.35
32
+ nvidia-cufile==1.15.1.6
33
+ nvidia-cuda-runtime==13.0.96
34
+ nvidia-cuda-nvrtc==13.0.88
35
+ nvidia-cuda-cupti==13.0.85
36
+ nvidia-cublas==13.1.0.3
37
+ numpy==2.4.4
38
+ networkx==3.6.1
39
+ multidict==6.7.1
40
+ mdurl==0.1.2
41
+ MarkupSafe==3.0.3
42
+ idna==3.13
43
+ hf-xet==1.4.3
44
+ h11==0.16.0
45
+ fsspec==2026.2.0
46
+ frozenlist==1.8.0
47
+ filelock==3.29.0
48
+ dill==0.4.1
49
+ cuda-pathfinder==1.5.4
50
+ click==8.3.3
51
+ charset-normalizer==3.4.7
52
+ certifi==2026.4.22
53
+ attrs==26.1.0
54
+ annotated-types==0.7.0
55
+ annotated-doc==0.0.4
56
+ aiohappyeyeballs==2.6.1
57
+ yarl==1.23.0
58
+ typing-inspection==0.4.2
59
+ sentry-sdk==2.58.0
60
+ requests==2.33.1
61
+ python-dateutil==2.9.0.post0
62
+ pydantic_core==2.46.3
63
+ nvidia-cusparse==12.6.3.3
64
+ nvidia-cufft==12.0.0.61
65
+ nvidia-cudnn-cu13==9.19.0.56
66
+ multiprocess==0.70.19
67
+ markdown-it-py==4.0.0
68
+ Jinja2==3.1.6
69
+ httpcore==1.0.9
70
+ gitdb==4.0.12
71
+ cuda-bindings==13.2.0
72
+ anyio==4.13.0
73
+ aiosignal==1.4.0
74
+ rich==15.0.0
75
+ pydantic==2.13.3
76
+ pandas==3.0.2
77
+ nvidia-cusolver==12.0.4.66
78
+ httpx==0.28.1
79
+ GitPython==3.1.49
80
+ aiohttp==3.13.5
81
+ wandb==0.26.1
82
+ typer==0.25.1
83
+ torch==2.11.0
84
+ huggingface_hub==1.13.0
85
+ tokenizers==0.22.2
86
+ datasets==4.8.5
87
+ accelerate==1.13.0
88
+ transformers==5.7.0
89
+ word2number==1.1
90
+ sqlitedict==2.1.0
91
+ pytz==2026.2
92
+ threadpoolctl==3.6.0
93
+ tcolorpy==0.1.7
94
+ tabulate==0.10.0
95
+ scipy==1.17.1
96
+ portalocker==3.2.0
97
+ pathvalidate==3.3.1
98
+ narwhals==2.22.1
99
+ more-itertools==11.1.0
100
+ lxml==6.1.1
101
+ joblib==1.5.3
102
+ colorama==0.4.6
103
+ chardet==6.0.0.post1
104
+ absl-py==2.4.0
105
+ scikit-learn==1.9.0
106
+ sacrebleu==2.6.0
107
+ nltk==3.9.4
108
+ mbstrdecoder==1.1.5
109
+ typepy==1.3.5
110
+ rouge_score==0.1.2
111
+ DataProperty==1.1.1
112
+ tabledata==1.3.5
113
+ evaluate==0.4.6
114
+ pytablewriter==1.2.1
115
+ lm_eval==0.4.12
116
+ pluggy==1.6.0
117
+ iniconfig==2.3.0
118
+ pytest==9.1.0
119
+ py-spy==0.4.2
120
+ pillow==12.2.0
121
+ tifffile==2026.6.1
122
+ imagecodecs==2026.6.26
123
+ zipp==4.1.0
124
+ importlib_metadata==9.0.0
125
+ derivative==0.6.3
126
+ pysindy==2.1.0
127
+ pyparsing==3.3.2
128
+ kiwisolver==1.5.0
129
+ fonttools==4.63.0
130
+ cycler==0.12.1
131
+ contourpy==1.3.3
132
+ matplotlib==3.11.0
133
+ sortedcontainers==2.4.0
134
+ wcwidth==0.8.2
135
+ sigfig==1.3.19
136
+ importlib_resources==7.1.0
137
+ enum-tools==0.13.0
138
+ cramjam==2.11.0
139
+ python-snappy==0.7.3
140
+ compact-json==1.8.2
141
+ numbers-parser==4.18.5
142
+ entmax==1.3
epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/files/wandb-metadata.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "os": "Linux-6.8.0-124-generic-x86_64-with-glibc2.35",
3
+ "python": "CPython 3.13.13",
4
+ "startedAt": "2026-08-29T04:08:43.356438Z",
5
+ "args": [
6
+ "--config",
7
+ "configs/p2_1p2b_mixed_seed72_4gpu_autoresume_current.json"
8
+ ],
9
+ "program": "/mloscratch/homes/navasard/sparse/SparseAtt/train.py",
10
+ "codePath": "train.py",
11
+ "codePathLocal": "train.py",
12
+ "git": {
13
+ "remote": "https://github.com/rubentium/SparseAtt.git",
14
+ "commit": "8f862f749eb4d5c17f688eb1562235e3fe5b2550"
15
+ },
16
+ "email": "ruben.navasardyan@mail.utoronto.ca",
17
+ "root": "/mloscratch/homes/navasard/sparse/SparseAtt",
18
+ "host": "h100-workspace-4-gpu-3-0-15",
19
+ "executable": "/mloscratch/homes/navasard/conda/bin/python3.13",
20
+ "cpu_count": 96,
21
+ "cpu_count_logical": 192,
22
+ "gpu": "NVIDIA H100 80GB HBM3",
23
+ "gpu_count": 4,
24
+ "disk": {
25
+ "/": {
26
+ "total": "9374931091456",
27
+ "used": "6464774668288"
28
+ }
29
+ },
30
+ "memory": {
31
+ "total": "1623005732864"
32
+ },
33
+ "gpu_nvidia": [
34
+ {
35
+ "name": "NVIDIA H100 80GB HBM3",
36
+ "memoryTotal": "85520809984",
37
+ "cudaCores": 16896,
38
+ "architecture": "Hopper",
39
+ "uuid": "GPU-51f13867-a72c-7b65-ff60-661787f0c18a"
40
+ },
41
+ {
42
+ "name": "NVIDIA H100 80GB HBM3",
43
+ "memoryTotal": "85520809984",
44
+ "cudaCores": 16896,
45
+ "architecture": "Hopper",
46
+ "uuid": "GPU-bab1e315-c967-43cf-bdfb-cac1a8dfdfac"
47
+ },
48
+ {
49
+ "name": "NVIDIA H100 80GB HBM3",
50
+ "memoryTotal": "85520809984",
51
+ "cudaCores": 16896,
52
+ "architecture": "Hopper",
53
+ "uuid": "GPU-a2122da0-1020-9f78-7e6f-a1d5011f694b"
54
+ },
55
+ {
56
+ "name": "NVIDIA H100 80GB HBM3",
57
+ "memoryTotal": "85520809984",
58
+ "cudaCores": 16896,
59
+ "architecture": "Hopper",
60
+ "uuid": "GPU-f65cda91-ff87-9bff-fb3d-3e1b3f9f8cf0"
61
+ }
62
+ ],
63
+ "cudaVersion": "13.0",
64
+ "writerId": "dxobsacuhig64dndr1wef3nz7j82437w"
65
+ }
epfl_final/SparseAtt/wandb/run-20260829_040843-z1i60278/logs/debug-internal.log ADDED
The diff for this file is too large to render. See raw diff